Merge pull request #1 from Dainius-P/develop

Develop

.gitignore

@@ -102,3 +102,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+/config
+manage.py

.pypirc

@@ -0,0 +1,11 @@
+[distutils]
+index-servers=
+    pypi
+    testpypi
+
+[testpypi]
+repository: https://test.pypi.org/legacy/
+username: dainiusp
+
+[pypi]
+username: dainiusp

LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2010-2019 Google, Inc. http://angularjs.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

MANIFEST.in

@@ -0,0 +1,8 @@
+include LICENSE
+include requirements.txt
+include README.md
+recursive-include scrapyd_dash/static *
+recursive-include scrapyd_dash/templates *
+recursive-include scrapyd_dash/operations *
+
+recursive-exclude config *

README.md

@@ -1,2 +1,63 @@
-# scrapyd-dash
-Scrapyd Dashboard
+# Scrapyd Dashboard
+
+A dashboard specifically used for scrapyd servers to manipulate scrapy spiders (start, stop, check status).
+
+### Features
+
+- [x] Scrapyd server status
+- [x] Tasks (add, remove, list)
+- [x] Authentification (Using the Django authentication system)
+- [ ] Scheduled tasks (add, remove, list)
+- [ ] Multi user support
+- [ ] Task performance details
+
+### Demo
+
+[Demo website](https://scrapyddash.herokuapp.com/)
+
+Username: **demo**
+
+Password: **demo**
+
+### Installing
+
+1. Install the package
+
+```
+pip install scrapyd_dash
+```
+
+2. Add urls to your projects urls.py
+
+```
+from django.urls import include
+
+urlpatterns = [
+    ...
+    path('scrapyd_dash/', include('scrapyd_dash.urls')),
+]
+
+```
+
+
+### Built With
+
+* [Django](https://www.djangoproject.com/) - The web framework used
+* [ScrapyD](https://github.com/scrapy/scrapyd) - A service daemon to run Scrapy spiders
+
+### Authors
+
+* **Dainius Preimantas** - *Initial work* - [Dainius-P](https://github.com/Dainius-P)
+
+### Contributing
+
+1. Fork it
+2. Create your feature branch (```git checkout -b feature/fooBar```)
+3. Commit your changes (```git commit -am 'Add some fooBar'```)
+4. Push to the branch (```git push origin feature/fooBar```)
+5. Create a new Pull Request
+
+
+### License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

requirements.txt

@@ -0,0 +1,4 @@
+APScheduler==3.6.0
+Django==2.2.2
+djangorestframework==3.9.4
+requests==2.22.0

scrapyd_dash/__init__.py

@@ -0,0 +1,3 @@
+from django.apps import apps as django_apps
+
+default_app_config = 'scrapyd_dash.apps.ScrapydDashConfig'

scrapyd_dash/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

scrapyd_dash/apps.py

@@ -0,0 +1,10 @@
+from django.apps import AppConfig
+
+
+class ScrapydDashConfig(AppConfig):
+    name = 'scrapyd_dash'
+    verbose_name = "Scrapyd Dashboard"
+
+    def ready(self):
+        from .utils import updater
+        updater.start_updater()

scrapyd_dash/migrations/0001_initial.py

@@ -0,0 +1,110 @@
+# Generated by Django 2.2.2 on 2019-07-24 11:25
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ScheduledTasks',
+            fields=[
+                ('id', models.PositiveIntegerField(primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=256, null=True)),
+                ('create_datetime', models.DateTimeField(auto_now_add=True)),
+                ('update_datetime', models.DateTimeField(auto_now=True)),
+                ('project', models.CharField(max_length=256)),
+                ('spider', models.CharField(max_length=256)),
+                ('year', models.PositiveIntegerField()),
+                ('month', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(12)])),
+                ('day', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(32)])),
+                ('week', models.PositiveIntegerField()),
+                ('day_of_week', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(7)])),
+                ('hour', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(24)])),
+                ('minute', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(60)])),
+                ('second', models.PositiveIntegerField(validators=[django.core.validators.MaxValueValidator(60)])),
+                ('deleted', models.BooleanField(default=False)),
+            ],
+            options={
+                'db_table': 'scrapyd_dash_scheduled_tasks',
+            },
+        ),
+        migrations.CreateModel(
+            name='ScrapydProject',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=256)),
+            ],
+            options={
+                'db_table': 'scrapyd_dash_projects',
+            },
+        ),
+        migrations.CreateModel(
+            name='ScrapydServer',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('ip', models.CharField(max_length=128)),
+                ('port', models.CharField(max_length=32)),
+                ('node_name', models.CharField(max_length=256)),
+                ('status', models.CharField(max_length=64)),
+                ('status_message', models.CharField(max_length=512, null=True)),
+                ('pending_tasks', models.PositiveIntegerField(default=0)),
+                ('finished_tasks', models.PositiveIntegerField(default=0)),
+                ('running_tasks', models.PositiveIntegerField(default=0)),
+            ],
+            options={
+                'db_table': 'scrapyd_dash_servers',
+                'ordering': ['-status'],
+                'unique_together': {('ip', 'port')},
+            },
+        ),
+        migrations.CreateModel(
+            name='Task',
+            fields=[
+                ('id', models.CharField(max_length=64, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=256)),
+                ('spider', models.CharField(max_length=256)),
+                ('status', models.CharField(max_length=64)),
+                ('pages', models.PositiveIntegerField(null=True)),
+                ('items', models.PositiveIntegerField(null=True)),
+                ('pid', models.PositiveIntegerField(null=True)),
+                ('runtime', models.CharField(blank=True, max_length=64, null=True)),
+                ('start_datetime', models.DateTimeField(blank=True, null=True)),
+                ('finished_datetime', models.DateTimeField(blank=True, null=True)),
+                ('log_href', models.CharField(max_length=1024, null=True)),
+                ('items_href', models.CharField(max_length=1024, null=True)),
+                ('create_datetime', models.DateTimeField(auto_now_add=True)),
+                ('update_datetime', models.DateTimeField(auto_now=True)),
+                ('deleted', models.BooleanField(default=False)),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='scrapyd_dash.ScrapydProject')),
+                ('server', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='scrapyd_dash.ScrapydServer')),
+            ],
+            options={
+                'db_table': 'scrapyd_dash_tasks',
+                'ordering': ['-create_datetime'],
+            },
+        ),
+        migrations.CreateModel(
+            name='ScrapydProjectVersion',
+            fields=[
+                ('version', models.CharField(max_length=125, primary_key=True, serialize=False)),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='scrapyd_dash.ScrapydProject')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='scrapydproject',
+            name='server',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='scrapyd_dash.ScrapydServer'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='scrapydproject',
+            unique_together={('server', 'name')},
+        ),
+    ]

scrapyd_dash/models.py

@@ -0,0 +1,122 @@
+from django.db import models
+from django.core.validators import MaxValueValidator
+
+
+class ScrapydServer(models.Model):
+    ip = models.CharField(max_length=128, null=False, blank=False)
+    port = models.CharField(max_length=32, null=False, blank=False)
+    node_name = models.CharField(max_length=256)
+    status = models.CharField(max_length=64)
+    status_message = models.CharField(max_length=512, null=True)
+    pending_tasks = models.PositiveIntegerField(default=0)
+    finished_tasks = models.PositiveIntegerField(default=0)
+    running_tasks = models.PositiveIntegerField(default=0)
+
+    def __str__(self):
+        return "%s%s" % (self.ip.replace('.', ''),
+                         self.port.replace('.', ''))
+
+    class Meta:
+        unique_together = (("ip", "port"),)
+        ordering = ['-status']
+        db_table = 'scrapyd_dash_servers'
+
+class ScrapydProject(models.Model):
+    server = models.ForeignKey(ScrapydServer,
+                        on_delete=models.CASCADE,
+                        null=False,
+                        blank=False)
+    name = models.CharField(max_length=256, null=False)
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        unique_together = (("server", "name"),)
+        db_table = 'scrapyd_dash_projects'
+
+class ScrapydProjectVersion(models.Model):
+    version = models.CharField(primary_key=True, max_length=125)
+    project = models.ForeignKey(ScrapydProject,
+                                 on_delete=models.DO_NOTHING)
+
+    def __str__(self):
+        return self.version
+
+
+class Task(models.Model):
+    id = models.CharField(max_length=64, primary_key=True)
+    name = models.CharField(max_length=256, null=False, blank=False)
+
+    project = models.ForeignKey(ScrapydProject,
+                         on_delete=models.CASCADE,
+                         null=False,
+                         blank=False)
+    spider = models.CharField(max_length=256, null=False)
+    status = models.CharField(max_length=64, null=False)
+
+    server = models.ForeignKey(ScrapydServer,
+                        on_delete=models.CASCADE,
+                        null=False,
+                        blank=False)
+
+    pages = models.PositiveIntegerField(null=True)
+    items = models.PositiveIntegerField(null=True)
+    pid = models.PositiveIntegerField(null=True)
+    runtime = models.CharField(max_length=64, null=True, blank=True)
+    start_datetime = models.DateTimeField(null=True, blank=True)
+    finished_datetime = models.DateTimeField(null=True, blank=True)
+
+    log_href = models.CharField(max_length=1024, null=True)
+    items_href = models.CharField(max_length=1024, null=True)
+
+    create_datetime = models.DateTimeField(auto_now_add=True)
+    update_datetime = models.DateTimeField(auto_now=True)
+
+    deleted = models.BooleanField(null=False, default=False)
+    stopping = models.BooleanField(null=False, default=False)
+
+    """
+    Prints out start datetime in a custom format
+    """
+    def print_start(self):
+        return self.start_datetime.strftime('%Y-%m-%d %H:%M')
+
+    """
+    Prints out finished datetime in a custom format
+    """
+    def print_finish(self):
+        return self.finished_datetime.strftime('%Y-%m-%d %H:%M')
+
+    class Meta:
+        ordering = ['-create_datetime']
+        db_table = 'scrapyd_dash_tasks'
+
+class ScheduledTasks(models.Model):
+    id = models.PositiveIntegerField(primary_key=True)
+    name = models.CharField(max_length=256, null=True)
+    create_datetime = models.DateTimeField(auto_now_add=True)
+    update_datetime = models.DateTimeField(auto_now=True)
+
+    project = models.CharField(max_length=256, null=False)
+    spider = models.CharField(max_length=256, null=False)
+
+    year = models.PositiveIntegerField(null=False)
+    month = models.PositiveIntegerField(null=False,
+                                 validators=[MaxValueValidator(12)])
+    day = models.PositiveIntegerField(null=False,
+                               validators=[MaxValueValidator(32)])
+    week = models.PositiveIntegerField(null=False)
+    day_of_week = models.PositiveIntegerField(null=False,
+                                       validators=[MaxValueValidator(7)])
+    hour = models.PositiveIntegerField(null=False,
+                                validators=[MaxValueValidator(24)])
+    minute = models.PositiveIntegerField(null=False,
+                                  validators=[MaxValueValidator(60)])
+    second = models.PositiveIntegerField(null=False,
+                                  validators=[MaxValueValidator(60)])
+
+    deleted = models.BooleanField(null=False, default=False)
+
+    class Meta:
+        db_table = 'scrapyd_dash_scheduled_tasks'