From 5a3dd8ad13792a5eaa86e8caf44f7c68540f4406 Mon Sep 17 00:00:00 2001 From: Erhan Date: Wed, 1 Nov 2023 17:33:43 +0100 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=8E=89=20[#9]=20Default=20project=20s?= =?UTF-8?q?tructure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .babelrc | 5 + .dockerignore | 16 + .editorconfig | 14 + .github/ISSUE_TEMPLATE/bug_report.yml | 90 + .github/ISSUE_TEMPLATE/feature_request.yml | 38 + .github/ISSUE_TEMPLATE/prepare-release.md | 18 + .github/workflows/ci.yml | 300 +++ .github/workflows/code_quality.yml | 119 + .github/workflows/codeql-analysis.yml | 38 + .github/workflows/macos.yml | 36 + .gitignore | 48 +- .jshintrc | 93 + .nvmrc | 1 + CHANGELOG.rst | 11 + Dockerfile | 115 + INSTALL.rst | 318 +++ Jenkinsfile | 177 ++ README.md | 19 - README.rst | 46 + bin/celery_beat.sh | 14 + bin/celery_flower.sh | 2 + bin/celery_worker.sh | 20 + bin/compile_dependencies.sh | 46 + bin/docker_start.sh | 44 + bin/flake8_summary.py | 12 + bin/live2staging.sh | 70 + bin/live2staging_postimport.sql | 1 + bin/live2staging_settings.sh | 19 + bin/reset_sequences.sql | 16 + bootstrap.py | 139 + build/paths.js | 47 + docker-compose.yml | 41 + docker-init-db.sql | 4 + docs/coding_style/backend.rst | 66 + docs/coding_style/frontend.rst | 530 ++++ docs/coding_style/index.rst | 66 + docs/deployment/VHOSTS.rst | 80 + docs/general/index.rst | 59 + docs/index.rst | 18 + docs/install/dev.rst | 44 + docs/install/index.rst | 15 + docs/install/production.rst | 5 + docs/install/staging.rst | 5 + docs/testing.rst | 105 + dotenv.example | 10 + karma.conf.js | 57 + log/README.rst | 1 + db.sqlite3 => log/nginx/.gitkeep | 0 manage.py | 22 - netland/asgi.py | 16 - netland/settings.py | 137 - netland/urls.py | 24 - package.json | 56 + postcss.config.js | 37 + requirements.txt | 6 - requirements/base.in | 27 + requirements/base.txt | 98 + requirements/ci.txt | 253 ++ requirements/dev.in | 20 + requirements/dev.txt | 427 ++++ requirements/production.txt | 1 + requirements/staging.txt | 1 + requirements/test-tools.in | 13 + requirements/test.txt | 1 + setup.cfg | 21 + src/manage.py | 24 + {login => src/netland}/__init__.py | 0 src/netland/accounts/__init__.py | 1 + src/netland/accounts/admin.py | 34 + src/netland/accounts/apps.py | 5 + src/netland/accounts/backends.py | 18 + src/netland/accounts/forms.py | 48 + .../netland/accounts/management}/__init__.py | 0 .../accounts/management/commands}/__init__.py | 0 .../commands/createinitialsuperuser.py | 96 + src/netland/accounts/managers.py | 34 + .../accounts/migrations/0001_initial.py | 131 + .../netland/accounts}/migrations/__init__.py | 0 src/netland/accounts/models.py | 65 + .../netland/accounts/tests}/__init__.py | 0 src/netland/accounts/tests/factories.py | 24 + .../tests/test_createinitialsuperuser.py | 58 + .../tests/test_password_reset_view.py | 13 + .../accounts/tests/test_permission_limit.py | 137 + .../accounts/tests/test_user_manager.py | 22 + src/netland/accounts/utils.py | 58 + src/netland/accounts/views/__init__.py | 1 + src/netland/accounts/views/csrf.py | 17 + src/netland/accounts/views/password_reset.py | 10 + src/netland/conf/__init__.py | 4 + src/netland/conf/base.py | 465 ++++ src/netland/conf/dev.py | 116 + src/netland/conf/docker.py | 16 + src/netland/conf/jenkins.py | 84 + src/netland/conf/local_example.py | 17 + src/netland/conf/locale/README | 9 + .../conf/locale/nl/LC_MESSAGES/django.po | 146 ++ src/netland/conf/production.py | 74 + src/netland/conf/staging.py | 11 + src/netland/conf/test.py | 14 + src/netland/conf/utils.py | 47 + src/netland/js/components/index.js | 1 + src/netland/js/index.js | 3 + src/netland/js/views/index.js | 1 + .../jstests/components/dummy/dummy.spec.js | 7 + src/netland/jstests/index.js | 7 + .../netland/jstests/views/.gitkeep | 0 .../netland/login}/__init__.py | 0 {login => src/netland/login}/admin.py | 0 {login => src/netland/login}/apps.py | 2 +- .../netland/login/migrations}/__init__.py | 0 {login => src/netland/login}/models.py | 0 {login => src/netland/login}/tests.py | 0 {login => src/netland/login}/urls.py | 0 {login => src/netland/login}/views.py | 0 .../netland/movie}/__init__.py | 0 {movie => src/netland/movie}/admin.py | 0 {movie => src/netland/movie}/apps.py | 2 +- {movie => src/netland/movie}/forms.py | 0 .../netland/movie}/migrations/0001_initial.py | 0 src/netland/movie/migrations/__init__.py | 0 {movie => src/netland/movie}/models.py | 0 {movie => src/netland/movie}/tests.py | 0 {movie => src/netland/movie}/urls.py | 0 {movie => src/netland/movie}/views.py | 0 src/netland/scss/_settings.scss | 249 ++ src/netland/scss/admin/_admin_theme.scss | 237 ++ src/netland/scss/admin/_app_overrides.scss | 125 + src/netland/scss/admin/_vars.scss | 28 + src/netland/scss/admin/admin_overrides.scss | 2 + src/netland/scss/components/_index.scss | 1 + src/netland/scss/screen.scss | 3 + src/netland/scss/views/_index.scss | 1 + src/netland/serie/__init__.py | 0 {serie => src/netland/serie}/admin.py | 0 {serie => src/netland/serie}/apps.py | 2 +- {serie => src/netland/serie}/forms.py | 0 src/netland/serie/management/__init__.py | 0 .../serie/management/commands/__init__.py | 0 .../management/commands/series_rating.py | 0 .../netland/serie}/migrations/0001_initial.py | 0 src/netland/serie/migrations/__init__.py | 0 {serie => src/netland/serie}/models.py | 0 src/netland/serie/tests/__init__.py | 0 .../serie/tests}/test_management_command.py | 0 .../netland/serie/tests}/test_model.py | 0 .../netland/serie/tests}/test_views.py | 0 {serie => src/netland/serie}/urls.py | 0 {serie => src/netland/serie}/views.py | 4 +- src/netland/setup.py | 60 + src/netland/static/ico/favicon-32x32.png | Bin 0 -> 866 bytes src/netland/static/ico/favicon-96x96.png | Bin 0 -> 2322 bytes src/netland/static/ico/favicon.png | Bin 0 -> 494 bytes src/netland/static/ico/favicon.svg | 30 + .../static/img/admin/calendar-alt-regular.svg | 1 + .../static/img/admin/clock-regular.svg | 1 + src/netland/static/img/admin/edit-regular.svg | 1 + .../static/img/admin/plus-square-regular.svg | 1 + .../static/img/admin/trash-alt-regular.svg | 1 + src/netland/templates/403.html | 10 + src/netland/templates/404.html | 10 + src/netland/templates/500.html | 140 + src/netland/templates/503.html | 207 ++ src/netland/templates/account_blocked.html | 11 + src/netland/templates/admin/base_site.html | 56 + .../netland/templates}/createmovie.html | 0 .../netland/templates}/createserie.html | 0 .../netland/templates}/detailmovie.html | 0 .../netland/templates}/detailserie.html | 0 .../hijack/contrib/admin/button.html | 11 + src/netland/templates/icons/font-awesome.svg | 639 +++++ src/netland/templates/icons/icomoon.svg | 2259 +++++++++++++++++ src/netland/templates/icons/iconic.svg | 1 + .../netland/templates}/index.html | 0 src/netland/templates/master.html | 26 + .../templates}/registration/login.html | 0 src/netland/templates/samples/menu.html | 13 + src/netland/templates/samples/pager.html | 23 + src/netland/templates/samples/pagination.html | 12 + src/netland/templates/sniplates/form.html | 102 + .../netland/templates}/updatemovie.html | 0 .../netland/templates}/updateserie.html | 0 src/netland/urls.py | 57 + src/netland/utils/__init__.py | 1 + src/netland/utils/apps.py | 8 + src/netland/utils/checks.py | 79 + src/netland/utils/context_processors.py | 21 + src/netland/utils/django_two_factor_auth.py | 19 + src/netland/utils/management/__init__.py | 0 .../utils/management/commands/__init__.py | 0 .../utils/management/commands/clear_cache.py | 16 + src/netland/utils/migration_operations.py | 60 + src/netland/utils/mixins.py | 89 + src/netland/utils/pdf.py | 125 + src/netland/utils/templatetags/__init__.py | 0 src/netland/utils/templatetags/utils.py | 70 + src/netland/utils/tests/__init__.py | 0 src/netland/utils/tests/test_celery_beat.py | 22 + src/netland/utils/tests/test_validators.py | 94 + src/netland/utils/validators.py | 53 + src/netland/utils/views.py | 25 + {netland => src/netland}/wsgi.py | 7 +- webpack.config.js | 90 + 203 files changed, 10869 insertions(+), 254 deletions(-) create mode 100644 .babelrc create mode 100644 .dockerignore create mode 100644 .editorconfig create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/ISSUE_TEMPLATE/prepare-release.md create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/code_quality.yml create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/macos.yml create mode 100644 .jshintrc create mode 100644 .nvmrc create mode 100644 CHANGELOG.rst create mode 100644 Dockerfile create mode 100644 INSTALL.rst create mode 100644 Jenkinsfile delete mode 100644 README.md create mode 100644 README.rst create mode 100755 bin/celery_beat.sh create mode 100755 bin/celery_flower.sh create mode 100755 bin/celery_worker.sh create mode 100755 bin/compile_dependencies.sh create mode 100755 bin/docker_start.sh create mode 100755 bin/flake8_summary.py create mode 100644 bin/live2staging.sh create mode 100644 bin/live2staging_postimport.sql create mode 100644 bin/live2staging_settings.sh create mode 100644 bin/reset_sequences.sql create mode 100755 bootstrap.py create mode 100644 build/paths.js create mode 100644 docker-compose.yml create mode 100644 docker-init-db.sql create mode 100644 docs/coding_style/backend.rst create mode 100644 docs/coding_style/frontend.rst create mode 100644 docs/coding_style/index.rst create mode 100644 docs/deployment/VHOSTS.rst create mode 100644 docs/general/index.rst create mode 100644 docs/index.rst create mode 100644 docs/install/dev.rst create mode 100644 docs/install/index.rst create mode 100644 docs/install/production.rst create mode 100644 docs/install/staging.rst create mode 100644 docs/testing.rst create mode 100644 dotenv.example create mode 100644 karma.conf.js create mode 100644 log/README.rst rename db.sqlite3 => log/nginx/.gitkeep (100%) delete mode 100644 manage.py delete mode 100644 netland/asgi.py delete mode 100644 netland/settings.py delete mode 100644 netland/urls.py create mode 100644 package.json create mode 100644 postcss.config.js delete mode 100644 requirements.txt create mode 100644 requirements/base.in create mode 100644 requirements/base.txt create mode 100644 requirements/ci.txt create mode 100644 requirements/dev.in create mode 100644 requirements/dev.txt create mode 100644 requirements/production.txt create mode 100644 requirements/staging.txt create mode 100644 requirements/test-tools.in create mode 100644 requirements/test.txt create mode 100644 setup.cfg create mode 100755 src/manage.py rename {login => src/netland}/__init__.py (100%) create mode 100644 src/netland/accounts/__init__.py create mode 100644 src/netland/accounts/admin.py create mode 100644 src/netland/accounts/apps.py create mode 100644 src/netland/accounts/backends.py create mode 100644 src/netland/accounts/forms.py rename {login/migrations => src/netland/accounts/management}/__init__.py (100%) rename {movie => src/netland/accounts/management/commands}/__init__.py (100%) create mode 100644 src/netland/accounts/management/commands/createinitialsuperuser.py create mode 100644 src/netland/accounts/managers.py create mode 100644 src/netland/accounts/migrations/0001_initial.py rename {movie => src/netland/accounts}/migrations/__init__.py (100%) create mode 100644 src/netland/accounts/models.py rename {netland => src/netland/accounts/tests}/__init__.py (100%) create mode 100644 src/netland/accounts/tests/factories.py create mode 100644 src/netland/accounts/tests/test_createinitialsuperuser.py create mode 100644 src/netland/accounts/tests/test_password_reset_view.py create mode 100644 src/netland/accounts/tests/test_permission_limit.py create mode 100644 src/netland/accounts/tests/test_user_manager.py create mode 100644 src/netland/accounts/utils.py create mode 100644 src/netland/accounts/views/__init__.py create mode 100644 src/netland/accounts/views/csrf.py create mode 100644 src/netland/accounts/views/password_reset.py create mode 100644 src/netland/conf/__init__.py create mode 100644 src/netland/conf/base.py create mode 100644 src/netland/conf/dev.py create mode 100644 src/netland/conf/docker.py create mode 100644 src/netland/conf/jenkins.py create mode 100644 src/netland/conf/local_example.py create mode 100644 src/netland/conf/locale/README create mode 100644 src/netland/conf/locale/nl/LC_MESSAGES/django.po create mode 100644 src/netland/conf/production.py create mode 100644 src/netland/conf/staging.py create mode 100644 src/netland/conf/test.py create mode 100644 src/netland/conf/utils.py create mode 100644 src/netland/js/components/index.js create mode 100644 src/netland/js/index.js create mode 100644 src/netland/js/views/index.js create mode 100644 src/netland/jstests/components/dummy/dummy.spec.js create mode 100644 src/netland/jstests/index.js rename serie/__init__.py => src/netland/jstests/views/.gitkeep (100%) rename {serie/management => src/netland/login}/__init__.py (100%) rename {login => src/netland/login}/admin.py (100%) rename {login => src/netland/login}/apps.py (82%) rename {serie/management/commands => src/netland/login/migrations}/__init__.py (100%) rename {login => src/netland/login}/models.py (100%) rename {login => src/netland/login}/tests.py (100%) rename {login => src/netland/login}/urls.py (100%) rename {login => src/netland/login}/views.py (100%) rename {serie/migrations => src/netland/movie}/__init__.py (100%) rename {movie => src/netland/movie}/admin.py (100%) rename {movie => src/netland/movie}/apps.py (82%) rename {movie => src/netland/movie}/forms.py (100%) rename {movie => src/netland/movie}/migrations/0001_initial.py (100%) create mode 100644 src/netland/movie/migrations/__init__.py rename {movie => src/netland/movie}/models.py (100%) rename {movie => src/netland/movie}/tests.py (100%) rename {movie => src/netland/movie}/urls.py (100%) rename {movie => src/netland/movie}/views.py (100%) create mode 100644 src/netland/scss/_settings.scss create mode 100644 src/netland/scss/admin/_admin_theme.scss create mode 100644 src/netland/scss/admin/_app_overrides.scss create mode 100644 src/netland/scss/admin/_vars.scss create mode 100644 src/netland/scss/admin/admin_overrides.scss create mode 100644 src/netland/scss/components/_index.scss create mode 100644 src/netland/scss/screen.scss create mode 100644 src/netland/scss/views/_index.scss create mode 100644 src/netland/serie/__init__.py rename {serie => src/netland/serie}/admin.py (100%) rename {serie => src/netland/serie}/apps.py (82%) rename {serie => src/netland/serie}/forms.py (100%) create mode 100644 src/netland/serie/management/__init__.py create mode 100644 src/netland/serie/management/commands/__init__.py rename {serie => src/netland/serie}/management/commands/series_rating.py (100%) rename {serie => src/netland/serie}/migrations/0001_initial.py (100%) create mode 100644 src/netland/serie/migrations/__init__.py rename {serie => src/netland/serie}/models.py (100%) create mode 100644 src/netland/serie/tests/__init__.py rename {serie/test => src/netland/serie/tests}/test_management_command.py (100%) rename {serie/test => src/netland/serie/tests}/test_model.py (100%) rename {serie/test => src/netland/serie/tests}/test_views.py (100%) rename {serie => src/netland/serie}/urls.py (100%) rename {serie => src/netland/serie}/views.py (93%) create mode 100644 src/netland/setup.py create mode 100644 src/netland/static/ico/favicon-32x32.png create mode 100644 src/netland/static/ico/favicon-96x96.png create mode 100644 src/netland/static/ico/favicon.png create mode 100644 src/netland/static/ico/favicon.svg create mode 100644 src/netland/static/img/admin/calendar-alt-regular.svg create mode 100644 src/netland/static/img/admin/clock-regular.svg create mode 100644 src/netland/static/img/admin/edit-regular.svg create mode 100644 src/netland/static/img/admin/plus-square-regular.svg create mode 100644 src/netland/static/img/admin/trash-alt-regular.svg create mode 100644 src/netland/templates/403.html create mode 100644 src/netland/templates/404.html create mode 100644 src/netland/templates/500.html create mode 100644 src/netland/templates/503.html create mode 100644 src/netland/templates/account_blocked.html create mode 100644 src/netland/templates/admin/base_site.html rename {templates => src/netland/templates}/createmovie.html (100%) rename {templates => src/netland/templates}/createserie.html (100%) rename {templates => src/netland/templates}/detailmovie.html (100%) rename {templates => src/netland/templates}/detailserie.html (100%) create mode 100644 src/netland/templates/hijack/contrib/admin/button.html create mode 100644 src/netland/templates/icons/font-awesome.svg create mode 100644 src/netland/templates/icons/icomoon.svg create mode 100644 src/netland/templates/icons/iconic.svg rename {templates => src/netland/templates}/index.html (100%) create mode 100644 src/netland/templates/master.html rename {templates => src/netland/templates}/registration/login.html (100%) create mode 100644 src/netland/templates/samples/menu.html create mode 100644 src/netland/templates/samples/pager.html create mode 100644 src/netland/templates/samples/pagination.html create mode 100644 src/netland/templates/sniplates/form.html rename {templates => src/netland/templates}/updatemovie.html (100%) rename {templates => src/netland/templates}/updateserie.html (100%) create mode 100644 src/netland/urls.py create mode 100644 src/netland/utils/__init__.py create mode 100644 src/netland/utils/apps.py create mode 100644 src/netland/utils/checks.py create mode 100644 src/netland/utils/context_processors.py create mode 100644 src/netland/utils/django_two_factor_auth.py create mode 100644 src/netland/utils/management/__init__.py create mode 100644 src/netland/utils/management/commands/__init__.py create mode 100644 src/netland/utils/management/commands/clear_cache.py create mode 100644 src/netland/utils/migration_operations.py create mode 100644 src/netland/utils/mixins.py create mode 100644 src/netland/utils/pdf.py create mode 100644 src/netland/utils/templatetags/__init__.py create mode 100644 src/netland/utils/templatetags/utils.py create mode 100644 src/netland/utils/tests/__init__.py create mode 100644 src/netland/utils/tests/test_celery_beat.py create mode 100644 src/netland/utils/tests/test_validators.py create mode 100644 src/netland/utils/validators.py create mode 100644 src/netland/utils/views.py rename {netland => src/netland}/wsgi.py (67%) create mode 100644 webpack.config.js diff --git a/.babelrc b/.babelrc new file mode 100644 index 00000000..4857ee97 --- /dev/null +++ b/.babelrc @@ -0,0 +1,5 @@ +{ + "presets": [ + "@babel/preset-env", + ] +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..749abda8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +node_modules +Dockerfile* +docker-compose* +.dockerignore +.git +.vscode +env/ +local.py +/doc/_build/ + +/htmlcov +.coverage + +/static/ +/media/ +/private_media/ diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..d4de5fc9 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{scss,sass}] +indent_size = 2 + +[*.{yml,yaml}] +indent_size = 2 diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000..017bb639 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,90 @@ +name: Probleem melden / Bug report +description: Meldt een probleem om ons te helpen verbeteren / Create a report to help us improve +title: "Title here" +labels: ["bug", "triage"] +assignees: [] +body: + - type: input + id: product-version + attributes: + label: Product versie / Product version + description: Welke versie gebruikt u? / Which version do you use? + placeholder: "1.1.0" + validations: + required: true + - type: textarea + id: what-happened + attributes: + label: Omschrijf het probleem / Describe the bug + description: Een duidelijke omschrijving van het probleem (de "bug") / A clear and concise description of what the bug is. + validations: + required: true + - type: textarea + id: steps-to-reproduce + attributes: + label: Stappen om te reproduceren / Steps to reproduce + description: Stappen die leiden tot het probleem / Steps to reproduce the behavior + placeholder: | + 1. Go to '...' + 2. Click on '....' + 3. Scroll down to '....' + 4. See error + validations: + required: false + - type: textarea + id: expected-behaviour + attributes: + label: Verwacht gedrag / Expected behavior + description: Een duidelijke omschrijving van wat de verwachting is / A clear and concise description of what you expected to happen. + validations: + required: false + - type: dropdown + id: screen-resolution + attributes: + label: Screen resolution + options: + - "smaller" + - "1024 x 768" + - "1366 x 768" + - "1920 x 1080" + - "bigger" + - "unknown" + validations: + required: false + - type: dropdown + id: device + attributes: + label: Device + options: + - Desktop + - Mobile + - Tablet + - Anders / Other + validations: + required: false + - type: dropdown + id: os + attributes: + label: OS + options: + - Windows + - Linux + - MacOS + - iOS + - Android + - Anders / Other + validations: + required: false + - type: dropdown + id: browsers + attributes: + label: Browser + multiple: true + options: + - Edge + - Chrome + - Safari + - Firefox + - Anders / Other + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000..dd605f8f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,38 @@ +name: Verzoek tot verbetering / Feature request +description: Stel een idee voor om het project beter te maken / Suggest an idea for this project +title: "Title here" +labels: ["enhancement", "triage"] +assignees: [] +body: + - type: dropdown + id: theme + attributes: + label: Thema / Theme + options: + - "Frontend" + - "API" + - "Admin" + - "Other" + validations: + required: true + - type: textarea + id: description + attributes: + label: Omschrijving / Description + description: Omschrijf duidelijk het idee of de behoefte, eventueel aangevuld met een bepaalde oplossingsrichting / Clearly describe the idea or need, possibly supplemented with a specific solution direction + validations: + required: true + - type: textarea + id: added-value + attributes: + label: Added value / Toegevoegde waarde + description: Omschrijf de toegevoegde waarde voor de bedrijfsvoering of dienstverlening / Describe the added value for business operations or services + validations: + required: false + - type: textarea + id: remarks + attributes: + label: Aanvullende opmerkingen / Additional context + description: Voeg aanvullingen of mockups toe voor deze verbetering / Add any other context or screenshots about the feature request + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/prepare-release.md b/.github/ISSUE_TEMPLATE/prepare-release.md new file mode 100644 index 00000000..b6114e44 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/prepare-release.md @@ -0,0 +1,18 @@ +--- +name: Prepare release +about: Checklist for new releases +title: Prepare release x.y.z +labels: '' +--- + +- [ ] Resolve release blockers + - [ ] ... +- [ ] Check translations + - [ ] Backend + - [ ] Frontend +- [ ] Bump API version number + - [ ] Version bump + - [ ] Regenerate API spec + - [ ] Update READMEs with release dates + links +- [ ] Bump version number (including package-lock.json) +- [ ] Update changelog diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..8ed82f99 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,300 @@ +name: Run build pipeline + +# Run this workflow every time a new commit pushed to your repository +on: + push: + branches: + - main + - stable/* + tags: + - '*' + pull_request: + workflow_dispatch: + +env: + IMAGE_NAME: maykinmedia/netland + DJANGO_SETTINGS_MODULE: netland.conf.ci + DOCKER_BUILDKIT: '1' + +jobs: + tests: + name: Run the Django test suite + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:14 + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # Needed because the postgres container does not provide a healthcheck + options: + --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:6 + ports: + - 6379:6379 + steps: + - uses: actions/checkout@v3 + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1 + with: + python-version: '3.10.9' + optimize-postgres: 'yes' + pg-service: 'postgres' + setup-node: 'yes' + # apt-packages: 'gettext postgresql-client' # the default + # npm-ci-flags: '--legacy-peer-deps' -> preferably use a .npmrc file + + # Any additional services -> create docker-compose setups in a (new) `docker` + # subdirectory. + # - name: Start CI docker services + # run: | + # docker-compose -f docker-compose.ci.yml up -d + # working-directory: docker + + - name: Run tests + run: | + python src/manage.py compilemessages + python src/manage.py collectstatic --noinput --link + coverage run src/manage.py test src + env: + SECRET_KEY: dummy + DB_USER: postgres + DB_PASSWORD: '' + + - name: Publish coverage report + uses: codecov/codecov-action@v3 + + # API projects: implement the necessary scripts for this. You can take a look + # at open-forms for inspiration, using drf-spectacular. + # - name: Generate OAS + # run: ./bin/generate_oas.sh openapi.yaml + + # - name: Store generated OAS + # uses: actions/upload-artifact@v3 + # with: + # name: netland-oas + # path: openapi.yaml + # retention-days: 1 + + # docs: + # name: Build and check documentation + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v3 + # - uses: actions/setup-python@v4 + # with: + # python-version: '3.9' + + # # - name: Install OS dependencies + # # run: | + # # sudo apt-get update + # # sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl + + # - name: Install dependencies + # run: | + # pip install -r requirements/ci.txt + + # - name: Build and test docs + # working-directory: docs + # run: pytest check_sphinx.py -v --tb=auto + + docker_build: + name: Build Docker image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set tag + id: vars + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name (if present at all) + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "main" ] && VERSION=latest + + # PRs result in version 'merge' -> transform that into 'latest' + [ "$VERSION" == "merge" ] && VERSION=latest + + echo "tag=${VERSION}" >> $GITHUB_OUTPUT + echo "git_hash=${GITHUB_SHA}" >> $GITHUB_OUTPUT + + - name: Build the production Docker image + run: | + docker build . \ + --tag $IMAGE_NAME:$RELEASE_VERSION \ + --build-arg COMMIT_HASH=${{ steps.vars.outputs.git_hash }} \ + --build-arg RELEASE=${{ steps.vars.outputs.tag }} \ + env: + RELEASE_VERSION: ${{ steps.vars.outputs.tag }} + + - run: docker image save -o image.tar $IMAGE_NAME:${{ steps.vars.outputs.tag }} + - name: Store image artifact + uses: actions/upload-artifact@v3 + with: + name: docker-image + path: image.tar + retention-days: 1 + + # API projects: implement the necessary scripts for this. You can take a look + # at open-forms for inspiration, using drf-spectacular. + # oas-up-to-date: + # needs: tests + # name: Check for unexepected OAS changes + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - name: Download generated OAS + # uses: actions/download-artifact@v3 + # with: + # name: netland-oas + # - name: Check for OAS changes + # run: | + # diff openapi.yaml src/openapi.yaml + # - name: Write failure markdown + # if: ${{ failure() }} + # run: | + # echo 'Run the following command locally and commit the changes' >> $GITHUB_STEP_SUMMARY + # echo '' >> $GITHUB_STEP_SUMMARY + # echo '```bash' >> $GITHUB_STEP_SUMMARY + # echo './bin/generate_oas.sh' >> $GITHUB_STEP_SUMMARY + # echo '```' >> $GITHUB_STEP_SUMMARY + # + # oas-lint: + # needs: oas-up-to-date + # name: Validate OAS + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - name: Download generated OAS + # uses: actions/download-artifact@v3 + # with: + # name: netland-oas + # - name: Use Node.js + # uses: actions/setup-node@v3 + # with: + # node-version-file: '.nvmrc' + # - name: Install spectral + # run: npm install -g @stoplight/spectral@5.9.2 + # - name: Run OAS linter + # run: spectral lint ./openapi.yaml + + # oas-postman: + # needs: oas-up-to-date + # name: Generate Postman collection from OAS + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - name: Download generated OAS + # uses: actions/download-artifact@v3 + # with: + # name: netland-oas + # - name: Use Node.js + # uses: actions/setup-node@v3 + # with: + # node-version-file: '.nvmrc' + # - name: Install dependencies + # run: npm install -g openapi-to-postmanv2 + # - name: Create tests folder + # run: mkdir -p ./tests/postman + # - name: Generate Postman collection + # run: openapi2postmanv2 -s ./openapi.yaml -o ./tests/postman/collection.json --pretty + + # oas-generate-sdks: + # needs: oas-up-to-date + # name: Generate SDKs from OAS + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - name: Download generated OAS + # uses: actions/download-artifact@v3 + # with: + # name: netland-oas + # - name: Use Node.js + # uses: actions/setup-node@v3 + # with: + # node-version-file: '.nvmrc' + # - name: Install dependencies + # run: npm install -g @openapitools/openapi-generator-cli@2.4.2 + # - name: Validate schema + # run: openapi-generator-cli validate -i ./openapi.yaml + # - name: Generate Java client + # run: + # openapi-generator-cli generate -i ./openapi.yaml + # --global-property=modelTests=false,apiTests=false,modelDocs=false,apiDocs=false \ -o + # ./sdks/java -g java + # --additional-properties=dateLibrary=java8,java8=true,optionalProjectFile=false,optionalAssemblyInfo=false + # - name: Generate .NET Core client + # run: + # openapi-generator-cli generate -i ./openapi.yaml + # --global-property=modelTests=false,apiTests=false,modelDocs=false,apiDocs=false \ -o + # ./sdks/netcore -g csharp-netcore + # --additional-properties=optionalProjectFile=false,optionalAssemblyInfo=false + # - name: Generate .NET Full Framework client + # run: + # openapi-generator-cli generate -i ./openapi.yaml + # --global-property=modelTests=false,apiTests=false,modelDocs=false,apiDocs=false \ -o + # ./sdks/net -g csharp + # --additional-properties=optionalProjectFile=false,optionalAssemblyInfo=false + # - name: Generate Python client + # run: + # openapi-generator-cli generate -i ./openapi.yaml + # --global-property=modelTests=false,apiTests=false,modelDocs=false,apiDocs=false \ -o + # ./sdks/python -g python + # --additional-properties=optionalProjectFile=false,optionalAssemblyInfo=false+ + + docker_push: + needs: + - tests + - docker_build + # - oas-lint + # - oas-postman + # - oas-generate-sdks + + name: Push Docker image + runs-on: ubuntu-latest + if: github.event_name == 'push' # Exclude PRs + + steps: + # This will include the updated OAS (if updated) from the update-oas job. + - uses: actions/checkout@v3 + + - name: Download built image + uses: actions/download-artifact@v3 + with: + name: docker-image + + - name: Set tag + id: vars + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name (if present at all) + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "main" ] && VERSION=latest + + echo "tag=${VERSION}" >> $GITHUB_OUTPUT + + - name: Load image + run: | + docker image load -i image.tar + + - name: Log into registry + run: + echo "${{ secrets.DOCKER_TOKEN }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} + --password-stdin + + - name: Push the Docker image (production) + run: docker push $IMAGE_NAME:$RELEASE_VERSION + env: + RELEASE_VERSION: ${{ steps.vars.outputs.tag }} diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml new file mode 100644 index 00000000..26360c20 --- /dev/null +++ b/.github/workflows/code_quality.yml @@ -0,0 +1,119 @@ +name: Code quality checks + +# Run this workflow every time a new commit pushed to your repository +on: + push: + branches: + - main + - stable/* + tags: + paths: + - '**.py' + - '**.json' + - '**.yaml' + - '**.in' + pull_request: + paths: + - '**.py' + - '**.json' + - '**.yaml' + - '**.in' + workflow_dispatch: + +jobs: + isort: + name: Check import sorting + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1 + with: + python-version: '3.10' + setup-node: 'no' + - uses: isort/isort-action@v1.1.0 + with: + requirements-files: requirements/ci.txt + sort-paths: 'src docs bin' + configuration: '--check-only --diff' + + black: + name: Check code formatting with black + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1 + with: + python-version: '3.10' + setup-node: 'no' + - name: Run black + run: black --check src docs bin + + flake8: + name: Code style (flake8) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1 + with: + python-version: '3.10' + setup-node: 'no' + - name: Run flake8 + id: flake8 + run: | + flake8 src 2>&1 | tee flake8_output.txt + result_code=${PIPESTATUS[0]} + report="$(cat flake8_output.txt)" + report="${report//$'\n'/'%0A'}" # escape newlines + echo "FLAKE8_REPORT=${report}" >> $GITHUB_OUTPUT + exit $result_code + - name: Emit flake8 flake8 output + if: ${{ failure() }} + run: | + echo "${{ steps.flake8.outputs.FLAKE8_REPORT }}" + + echo 'flake8 found some issues' >> $GITHUB_STEP_SUMMARY + echo '' >> $GITHUB_STEP_SUMMARY + echo '| File | Line | Column | Issue |' >> $GITHUB_STEP_SUMMARY + echo '| :--- | ---- | ------ | :---- |' >> $GITHUB_STEP_SUMMARY + python ./bin/flake8_summary.py "${{ steps.flake8.outputs.FLAKE8_REPORT }}" >> $GITHUB_STEP_SUMMARY + + migrations: + name: Check for model changes not present in migrations + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:14 + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # Needed because the postgres container does not provide a healthcheck + options: + --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - uses: actions/checkout@v3 + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1 + with: + python-version: '3.10' + setup-node: 'no' + - name: Run makemigrations to check for missing migrations + run: | + src/manage.py makemigrations \ + --check \ + --dry-run + env: + DJANGO_SETTINGS_MODULE: "netland.conf.ci" + DEBUG: 'true' + SECRET_KEY: dummy + DB_USER: postgres + DB_NAME: postgres + DB_PASSWORD: '' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..95082a78 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,38 @@ +name: 'CodeQL' + +on: + push: + branches: + - main + - stable/* + schedule: + - cron: '32 20 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: ['javascript', 'python'] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 00000000..707a1f81 --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,36 @@ +name: Check MacOS compatibility + +# Run this workflow every time a new commit pushed to your repository +on: + push: + branches: + - main + - stable/* + tags: + - '*' + pull_request: + workflow_dispatch: + +jobs: + macos-deps: + name: Install dev dependencies on MacOS + runs-on: macos-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + cache-dependency-path: 'requirements/*.txt' + - name: Install OS-level packages + run: | + brew install pkg-config + - name: Install dependencies + run: | + export LIBRARY_PATH=$LIBRARY_PATH:/usr/local/opt/openssl/lib/ + pip install -r requirements/dev.txt \ + --use-pep517 \ + --use-feature=no-binary-enable-wheel-cache + env: + STATIC_DEPS: 'true' diff --git a/.gitignore b/.gitignore index db3af3a4..be2b1966 100644 --- a/.gitignore +++ b/.gitignore @@ -2,35 +2,39 @@ .hg .git .svn -.idea/* -.sass-cache/* +.idea/ +.vscode/ +.sass-cache/ *.db *.pyc +__pycache__ *~ +*.mo # Javascript libraries -src/bower_modules/* +/node_modules/ -# Environment -settings_local.py -/env/* -/staticfiles/* -/static/ -/media/ -/deployment/vars/staging-environment.json -/deployment/vars/production-environment.json -/environment.json +# Generated files +/docs/_build -# Logs -/log/ +# Coverage reports +/reports/ +/htmlcov/ +.coverage +# Environment +#/deployment/app.*.yml +#/deployment/hosts.* +local.py +/env/ +/venv/ +/media/ /static/ +/mail/ +/log/*.log* +/log/nginx/*.log* +/.env -# Code coverage -/htmlcov/* -.coveragerc -.coverage - -Pipfile -Pipfile.lock -.hypothesis +# Static files +src/netland/static/bundles/ +src/netland/fonts/ diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 00000000..493173c5 --- /dev/null +++ b/.jshintrc @@ -0,0 +1,93 @@ +{ + // JSHint Default Configuration File (as on JSHint website) + // See http://jshint.com/docs/ for more details + + "maxerr" : 50, // {int} Maximum error before stopping + + // Enforcing + "bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.) + "camelcase" : false, // true: Identifiers must be in camelCase + "curly" : true, // true: Require {} for every new block or scope + "eqeqeq" : true, // true: Require triple equals (===) for comparison + "forin" : true, // true: Require filtering for..in loops with obj.hasOwnProperty() + "freeze" : true, // true: prohibits overwriting prototypes of native objects such as Array, Date etc. + "immed" : false, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());` + "latedef" : false, // true: Require variables/functions to be defined before being used + "newcap" : false, // true: Require capitalization of all constructor functions e.g. `new F()` + "noarg" : true, // true: Prohibit use of `arguments.caller` and `arguments.callee` + "noempty" : true, // true: Prohibit use of empty blocks + "nonbsp" : true, // true: Prohibit "non-breaking whitespace" characters. + "nonew" : false, // true: Prohibit use of constructors for side-effects (without assignment) + "plusplus" : false, // true: Prohibit use of `++` and `--` + "quotmark" : "single", // Quotation mark consistency: + // false : do nothing (default) + // true : ensure whatever is used is consistent + // "single" : require single quotes + // "double" : require double quotes + "undef" : true, // true: Require all non-global variables to be declared (prevents global leaks) + "unused" : true, // Unused variables: + // true : all variables, last function parameter + // "vars" : all variables only + // "strict" : all variables, all function parameters + "strict" : false, // true: Requires all functions run in ES5 Strict Mode + "maxparams" : false, // {int} Max number of formal params allowed per function + "maxdepth" : 3, // {int} Max depth of nested blocks (within functions) + "maxstatements" : false, // {int} Max number statements per function + "maxcomplexity" : false, // {int} Max cyclomatic complexity per function + "maxlen" : false, // {int} Max number of characters per line + "varstmt" : false, // true: Disallow any var statements. Only `let` and `const` are allowed. + + // Relaxing + "asi" : false, // true: Tolerate Automatic Semicolon Insertion (no semicolons) + "boss" : false, // true: Tolerate assignments where comparisons would be expected + "debug" : false, // true: Allow debugger statements e.g. browser breakpoints. + "eqnull" : false, // true: Tolerate use of `== null` + "esversion" : 6, // {int} Specify the ECMAScript version to which the code must adhere. + "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features) + // (ex: `for each`, multiple try/catch, function expression�) + "evil" : false, // true: Tolerate use of `eval` and `new Function()` + "expr" : false, // true: Tolerate `ExpressionStatement` as Programs + "funcscope" : false, // true: Tolerate defining variables inside control statements + "globalstrict" : false, // true: Allow global "use strict" (also enables 'strict') + "iterator" : false, // true: Tolerate using the `__iterator__` property + "lastsemic" : false, // true: Tolerate omitting a semicolon for the last statement of a 1-line block + "laxbreak" : false, // true: Tolerate possibly unsafe line breakings + "laxcomma" : false, // true: Tolerate comma-first style coding + "loopfunc" : false, // true: Tolerate functions being defined in loops + "multistr" : false, // true: Tolerate multi-line strings + "noyield" : false, // true: Tolerate generator functions with no yield statement in them. + "notypeof" : false, // true: Tolerate invalid typeof operator values + "proto" : false, // true: Tolerate using the `__proto__` property + "scripturl" : false, // true: Tolerate script-targeted URLs + "shadow" : false, // true: Allows re-define variables later in code e.g. `var x=1; x=2;` + "sub" : false, // true: Tolerate using `[]` notation when it can still be expressed in dot notation + "supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;` + "validthis" : false, // true: Tolerate using this in a non-constructor function + + // Environments + "browser" : true, // Web Browser (window, document, etc) + "browserify" : false, // Browserify (node.js code in the browser) + "couch" : false, // CouchDB + "devel" : true, // Development/debugging (alert, confirm, etc) + "dojo" : false, // Dojo Toolkit + "jasmine" : false, // Jasmine + "jquery" : false, // jQuery + "mocha" : true, // Mocha + "mootools" : false, // MooTools + "node" : true, // Node.js + "nonstandard" : false, // Widely adopted globals (escape, unescape, etc) + "phantom" : false, // PhantomJS + "prototypejs" : false, // Prototype and Scriptaculous + "qunit" : false, // QUnit + "rhino" : false, // Rhino + "shelljs" : false, // ShellJS + "typed" : false, // Globals for typed array constructions + "worker" : false, // Web Workers + "wsh" : false, // Windows Scripting Host + "yui" : false, // Yahoo User Interface + + // Custom Globals + "globals": { + "google": true + } +} diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 00000000..3f430af8 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +v18 diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 00000000..712c5319 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,11 @@ +============== +Change history +============== + + +0.1.0 +===== + +** + +* Initial release. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..9260e436 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,115 @@ +# This is a multi-stage build file, which means a stage is used to build +# the backend (dependencies), the frontend stack and a final production +# stage re-using assets from the build stages. This keeps the final production +# image minimal in size. + +# Stage 1 - Backend build environment +# includes compilers and build tooling to create the environment +FROM python:3.10.9-slim-bullseye AS backend-build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config \ + build-essential \ + git \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +RUN mkdir /app/src + +# Ensure we use the latest version of pip +RUN pip install pip setuptools -U +COPY ./requirements /app/requirements +RUN pip install -r requirements/production.txt + + +# Stage 2 - Install frontend deps and build assets +FROM node:13-buster AS frontend-build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# copy configuration/build files +COPY ./build /app/build/ +COPY ./*.json ./*.js ./.babelrc /app/ + +# install WITH dev tooling +RUN npm ci + +# copy source code +COPY ./src /app/src + +# build frontend +RUN npm run build + + +# Stage 3 - Build docker image suitable for production +FROM python:3.10.9-slim-bullseye + +# Stage 3.1 - Set up the needed production dependencies +# install all the dependencies for GeoDjango +RUN apt-get update && apt-get install -y --no-install-recommends \ + procps \ + vim \ + mime-support \ + postgresql-client \ + gettext \ + # lxml deps + # libxslt \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY ./bin/docker_start.sh /start.sh +# Uncomment if you use celery +# COPY ./bin/celery_worker.sh /celery_worker.sh +# COPY ./bin/celery_beat.sh /celery_beat.sh +# COPY ./bin/celery_flower.sh /celery_flower.sh +RUN mkdir /app/log +RUN mkdir /app/media + +VOLUME ["/app/log", "/app/media"] + +# copy backend build deps +COPY --from=backend-build /usr/local/lib/python3.10 /usr/local/lib/python3.10 +COPY --from=backend-build /usr/local/bin/uwsgi /usr/local/bin/uwsgi +# Uncomment if you use celery +# COPY --from=backend-build /usr/local/bin/celery /usr/local/bin/celery +COPY --from=backend-build /app/src/ /app/src/ + +# copy frontend build statics +COPY --from=frontend-build /app/src/netland/static /app/src/netland/static + +# copy source code +COPY ./src /app/src + +RUN useradd -M -u 1000 maykin +RUN chown -R maykin:maykin /app + +# drop privileges +USER maykin + +ARG COMMIT_HASH +ARG RELEASE=latest + +ENV RELEASE=${RELEASE} \ + GIT_SHA=${COMMIT_HASH} \ + PYTHONUNBUFFERED=1 \ + DJANGO_SETTINGS_MODULE=netland.conf.docker + +ARG SECRET_KEY=dummy + +LABEL org.label-schema.vcs-ref=$COMMIT_HASH \ + org.label-schema.vcs-url="https://bitbucket.org/maykinmedia/netland" \ + org.label-schema.version=$RELEASE \ + org.label-schema.name="netland" + +# Run collectstatic and compilemessages, so the result is already included in +# the image +RUN python src/manage.py collectstatic --noinput \ + && python src/manage.py compilemessages + +EXPOSE 8000 +CMD ["/start.sh"] diff --git a/INSTALL.rst b/INSTALL.rst new file mode 100644 index 00000000..a1bcc92e --- /dev/null +++ b/INSTALL.rst @@ -0,0 +1,318 @@ +============ +Installation +============ + +The project is developed in Python using the `Django framework`_. There are 3 +sections below, focussing on developers, running the project using Docker and +hints for running the project in production. + +.. _Django framework: https://www.djangoproject.com/ + + +Development +=========== + + +Prerequisites +------------- + +You need the following libraries and/or programs: + +* `Python`_ 3.6 or above +* Python `Virtualenv`_ and `Pip`_ +* `PostgreSQL`_ 10 or above +* `Node.js`_ +* `npm`_ + +.. _Python: https://www.python.org/ +.. _Virtualenv: https://virtualenv.pypa.io/en/stable/ +.. _Pip: https://packaging.python.org/tutorials/installing-packages/#ensure-pip-setuptools-and-wheel-are-up-to-date +.. _PostgreSQL: https://www.postgresql.org +.. _Node.js: http://nodejs.org/ +.. _npm: https://www.npmjs.com/ + + +Getting started +--------------- + +Developers can follow the following steps to set up the project on their local +development machine. + +1. Navigate to the location where you want to place your project. + +2. Get the code: + + .. code-block:: bash + + $ git clone git@bitbucket.org:maykinmedia/netland.git + $ cd netland + +3. Install all required (backend) libraries. + **Tip:** You can use the ``bootstrap.py`` script to install the requirements + and set the proper settings in ``manage.py``. Or, perform the steps + manually: + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + $ pip install -r requirements/dev.txt + +4. Install all required (frontend) libraries and build static files. + + .. code-block:: bash + + $ npm install + $ npm run build + +5. Collect statics and create the initial database tables: + + .. code-block:: bash + + $ python src/manage.py collectstatic --link + $ python src/manage.py migrate + +6. Create a superuser to access the management interface: + + .. code-block:: bash + + $ python src/manage.py createsuperuser + +7. You can now run your installation and point your browser to the address + given by this command: + + .. code-block:: bash + + $ python src/manage.py runserver + +8. Create a .env file with database settings. See dotenv.example for an example. + + $ cp dotenv.example .env + + +**Note:** If you are making local, machine specific, changes, add them to +``src/netland/conf/local.py``. You can base this file on the +example file included in the same directory. + + +Update installation +------------------- + +When updating an existing installation: + +1. Activate the virtual environment: + + .. code-block:: bash + + $ cd netland + $ source env/bin/activate + +2. Update the code and libraries: + + .. code-block:: bash + + $ git pull + $ pip install -r requirements/dev.txt + $ npm install + $ npm run build + +3. Update the statics and database: + + .. code-block:: bash + + $ python src/manage.py collectstatic --link + $ python src/manage.py migrate + + +Testsuite +--------- + +To run the test suite: + +.. code-block:: bash + + $ python src/manage.py test netland + +Configuration via environment variables +--------------------------------------- + +A number of common settings/configurations can be modified by setting +environment variables. You can persist these in your ``local.py`` settings +file or as part of the ``(post)activate`` of your virtualenv. + +* ``SECRET_KEY``: the secret key to use. A default is set in ``dev.py`` + +* ``DB_NAME``: name of the database for the project. Defaults to ``netland``. +* ``DB_USER``: username to connect to the database with. Defaults to ``netland``. +* ``DB_PASSWORD``: password to use to connect to the database. Defaults to ``netland``. +* ``DB_HOST``: database host. Defaults to ``localhost`` +* ``DB_PORT``: database port. Defaults to ``5432``. + +* ``SENTRY_DSN``: the DSN of the project in Sentry. If set, enabled Sentry SDK as + logger and will send errors/logging to Sentry. If unset, Sentry SDK will be + disabled. + +Docker +====== + +The easiest way to get the project started is by using `Docker Compose`_. + +1. Clone or download the code from `Github`_ in a folder like + ``netland``: + + .. code-block:: bash + + $ git clone git@bitbucket.org:maykinmedia/netland.git + Cloning into 'netland'... + ... + + $ cd netland + +2. Start the database and web services: + + .. code-block:: bash + + $ docker-compose up -d + Starting netland_db_1 ... done + Starting netland_web_1 ... done + + It can take a while before everything is done. Even after starting the web + container, the database might still be migrating. You can always check the + status with: + + .. code-block:: bash + + $ docker logs -f netland_web_1 + +3. Create an admin user and load initial data. If different container names + are shown above, use the container name ending with ``_web_1``: + + .. code-block:: bash + + $ docker exec -it netland_web_1 /app/src/manage.py createsuperuser + Username: admin + ... + Superuser created successfully. + + $ docker exec -it netland_web_1 /app/src/manage.py loaddata admin_index groups + Installed 5 object(s) from 2 fixture(s) + +4. Point your browser to ``http://localhost:8000/`` to access the project's + management interface with the credentials used in step 3. + + If you are using ``Docker Machine``, you need to point your browser to the + Docker VM IP address. You can get the IP address by doing + ``docker-machine ls`` and point your browser to + ``http://:8000/`` instead (where the ```` is shown below the URL + column): + + .. code-block:: bash + + $ docker-machine ls + NAME ACTIVE DRIVER STATE URL + default * virtualbox Running tcp://: + +5. To shutdown the services, use ``docker-compose down`` and to clean up your + system you can run ``docker system prune``. + +.. _Docker Compose: https://docs.docker.com/compose/install/ +.. _Github: https://github.com/maykinmedia/netland/ + + +More Docker +----------- + +If you just want to run the project as a Docker container and connect to an +external database, you can build and run the ``Dockerfile`` and pass several +environment variables. See ``src/netland/conf/docker.py`` for +all settings. + +.. code-block:: bash + + $ docker build -t netland + $ docker run \ + -p 8000:8000 \ + -e DATABASE_USERNAME=... \ + -e DATABASE_PASSWORD=... \ + -e DATABASE_HOST=... \ + --name netland \ + netland + + $ docker exec -it netland /app/src/manage.py createsuperuser + +Building and publishing the image +--------------------------------- + +Using ``bin/release-docker-image``, you can easily build and tag the image. + +The script is based on git branches and tags - if you're on the ``master`` +branch and the current ``HEAD`` is tagged, the tag will be used as +``RELEASE_TAG`` and the image will be pushed. If you want to push the image +without a git tag, you can use the ``RELEASE_TAG`` envvar. + +The image will only be pushed if the ``JOB_NAME`` envvar is set. The image +will always be built, even if no envvar is set. The default release tag is +``latest``. + +Example usage: + +.. code-block:: bash + + JOB_NAME=publish RELEASE_TAG=dev ./bin/release-docker-image.sh + + +Staging and production +====================== + +Ansible is used to deploy test, staging and production servers. It is assumed +the target machine has a clean `Debian`_ installation. + +1. Make sure you have `Ansible`_ installed (globally or in the virtual + environment): + + .. code-block:: bash + + $ pip install ansible + +2. Navigate to the project directory, and install the Maykin deployment + submodule if you haven't already: + + .. code-block:: bash + + $ git submodule update --init + +3. Run the Ansible playbook to provision a clean Debian machine: + + .. code-block:: bash + + $ cd deployment + $ ansible-playbook .yml + +For more information, see the ``README`` file in the deployment directory. + +.. _Debian: https://www.debian.org/ +.. _Ansible: https://pypi.org/project/ansible/ + + +Settings +======== + +All settings for the project can be found in +``src/netland/conf``. +The file ``local.py`` overwrites settings from the base configuration. + + +Commands +======== + +Commands can be executed using: + +.. code-block:: bash + + $ python src/manage.py + +There are no specific commands for the project. See +`Django framework commands`_ for all default commands, or type +``python src/manage.py --help``. + +.. _Django framework commands: https://docs.djangoproject.com/en/dev/ref/django-admin/#available-commands diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..c35e9d03 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,177 @@ +#!groovy + +// Use "node {...}" to use any Jenkins server, or "node('master') {...}" to +// only run on the master node. +node { + // You can hardcode the settings here, or have it dynamically figured out + // in the build step. + def djangoSettings = null + def curDir = pwd() + def envDir = "${curDir}/env" + + stage ("Build") { + // Submit to Chucks will + chuckNorris() + + // Use the clean option that fits best in the project. + // Clean build when changing target + if (env.CHANGE_TARGET) { + // Clean workspace + // cleanWs() + + // Clean virtual environment + dir("env") { + deleteDir() + } + } + // Clean build when the previous build failed. + // cleanWs cleanWhenNotBuilt: false, cleanWhenSuccess: false, notFailBuild: true + + def installed = fileExists "${envDir}/bin/activate" + + checkout scm + + // Hard way of determining the Django settings path. + // Might break if you have multiple directories in src + if (!djangoSettings) { + djangoSettings = sh( + script: 'projectFolder=`cd src; ls -d */ | head -n 1`; echo "${projectFolder%?}.conf.jenkins"', + returnStdout: true + ) + } + + if (!installed) { + sh "virtualenv ${envDir} -p python3" + } + } + + stage ("Install backend requirements") { + sh """ + . ${envDir}/bin/activate + pip install pip --upgrade + pip install --exists-action=w -r requirements/ci.txt + deactivate + """ + } + + stage ("Install frontend requirements") { + sh """ + npm ci + npm run build --production --sourcemap + """ + + withEnv(["SECRET_KEY=test_key"]) { + sh """ + . ${envDir}/bin/activate + python src/manage.py collectstatic \ + --link \ + --noinput \ + --settings=${djangoSettings} + deactivate + """ + } + } + + stage ("Test backend") { + def testsError = null + def keepDbOption = "" + + if (!env.CHANGE_TARGET) { + keepDbOption = "--keepdb" + } + + withEnv(["SECRET_KEY=test_key","ELASTIC_APM_DISABLE_SEND=true"]) { + try { + sh """ + . ${envDir}/bin/activate + python src/manage.py jenkins \ + --project-apps-tests \ + --verbosity 2 \ + --noinput \ + --pep8-rcfile=.pep8 \ + --coverage-rcfile=.coveragerc \ + ${keepDbOption} \ + --enable-coverage \ + --settings=${djangoSettings} + deactivate + """ + } + catch(err) { + testsError = err + currentBuild.result = "FAILURE" + } + finally { + dir("media") { + deleteDir() + } + junit "reports/junit.xml" + + if (testsError) { + throw testsError + } + } + } + + withEnv(["SECRET_KEY=test_key"]) { + try { + sh "${envDir}/bin/isort --recursive --check-only --diff --quiet src > reports/isort.report" + } + catch(err) { + // Nothing... + } + } + } + + stage ("Test frontend") { + def testsError = null + + try { + sh "xvfb-run -a --server-args='-screen 0, 1920x1200x16' npm test" + } + catch(err) { + testsError = err + currentBuild.result = "FAILURE" + } + finally { + junit "reports/jstests/junit.xml" + + if (testsError) { + throw testsError + } + } + } + + stage ("Quality") { + step( + [ + $class: "CoberturaPublisher", + coberturaReportFile: "reports/coverage.xml" + ] + ) + step( + [ + $class: "WarningsPublisher", + parserConfigurations: [ + [ + parserName: "PyLint", + pattern: "reports/pylint.report", + unstableTotalAll: "10", + usePreviousBuildAsReference: true, + ], + [ + parserName: "Pep8", + pattern: "reports/pep8.report", + unstableTotalAll: "50", + usePreviousBuildAsReference: true, + ], + [ + parserName: "Dynamic", + pattern: "reports/isort.report", + unstableTotalAll: "10", + usePreviousBuildAsReference: true, + ], + ] + ] + ) + } +} diff --git a/README.md b/README.md deleted file mode 100644 index 91d36edc..00000000 --- a/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Netland -Netland is a website where you can see a table of information about series and movies. This data could you update and delete. I have worked with a MySQL database, the default database that Django uses is Sqlite. You can change your login details of your localhost phpmyadmin in settings.py. - -# Getting Started -With the following instructions you can get the project working on your local machine. - -# Prerequisites -`Python 3.10.5 and Django==4.1 or an higher version` - -# Installing -First of all you need to clone this repository to your local machine. After that you can paste the sql query's that I put in the sql folder in your localhost/phpmyadmin. - -## Activating the environment & Installing the requirements -Open any terminal in your main folder and activate the environment doing: `. env/Scripts/activate` for windows and `source env/bin/activate` for IOS. Install all the requirements from requirements.txt with the command `pip install -r requirements.txt`. Change your login credintials in the settings.py file. - -## Running the server -Run the server from your terminal with the command `python manage.py runserver`. - - diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..6c28dfd7 --- /dev/null +++ b/README.rst @@ -0,0 +1,46 @@ +================== +netland +================== + +:Version: 0.1.0 +:Source: https://bitbucket.org/maykinmedia/netland +:Keywords: ```` +:PythonVersion: 3.10 + +|build-status| |requirements| + +```` + +Developed by `Maykin Media B.V.`_ for ```` + + +Introduction +============ + +```` + + +Documentation +============= + +See ``INSTALL.rst`` for installation instructions, available settings and +commands. + + +References +========== + +* `Issues `_ +* `Code `_ + + +.. |build-status| image:: http://jenkins.maykin.nl/buildStatus/icon?job=bitbucket/netland/master + :alt: Build status + :target: http://jenkins.maykin.nl/job/netland + +.. |requirements| image:: https://requires.io/bitbucket/maykinmedia/netland/requirements.svg?branch=master + :target: https://requires.io/bitbucket/maykinmedia/netland/requirements/?branch=master + :alt: Requirements status + + +.. _Maykin Media B.V.: https://www.maykinmedia.nl diff --git a/bin/celery_beat.sh b/bin/celery_beat.sh new file mode 100755 index 00000000..590f581c --- /dev/null +++ b/bin/celery_beat.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +LOGLEVEL=${CELERY_LOGLEVEL:-INFO} + +mkdir -p celerybeat + +echo "Starting celery beat" +exec celery beat \ + --app netland \ + -l $LOGLEVEL \ + --workdir src \ + -s ../celerybeat/beat diff --git a/bin/celery_flower.sh b/bin/celery_flower.sh new file mode 100755 index 00000000..d576ec77 --- /dev/null +++ b/bin/celery_flower.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec celery flower --app netland --workdir src diff --git a/bin/celery_worker.sh b/bin/celery_worker.sh new file mode 100755 index 00000000..7ced3023 --- /dev/null +++ b/bin/celery_worker.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +LOGLEVEL=${CELERY_LOGLEVEL:-INFO} +CONCURRENCY=${CELERY_WORKER_CONCURRENCY:-1} + +QUEUE=${1:-${CELERY_WORKER_QUEUE:=celery}} +WORKER_NAME=${2:-${CELERY_WORKER_NAME:="${QUEUE}"@%n}} + +echo "Starting celery worker $WORKER_NAME with queue $QUEUE" +exec celery worker \ + --app netland \ + -Q $QUEUE \ + -n $WORKER_NAME \ + -l $LOGLEVEL \ + --workdir src \ + -O fair \ + -c $CONCURRENCY + diff --git a/bin/compile_dependencies.sh b/bin/compile_dependencies.sh new file mode 100755 index 00000000..4fd8d477 --- /dev/null +++ b/bin/compile_dependencies.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# +# Compile the dependencies for production, CI and development. +# +# Usage, in the root of the project: +# +# ./bin/compile_dependencies.sh +# +# Any extra flags/arguments passed to this wrapper script are passed down to pip-compile. +# E.g. to update a package: +# +# ./bin/compile_dependencies.sh --upgrade-package django + +set -ex + +toplevel=$(git rev-parse --show-toplevel) + +cd $toplevel + +export CUSTOM_COMPILE_COMMAND="./bin/compile_dependencies.sh" + +# Base (& prod) deps +pip-compile \ + --no-emit-index-url \ + --allow-unsafe \ + "$@" \ + requirements/base.in + +# Dependencies for testing +pip-compile \ + --no-emit-index-url \ + --allow-unsafe \ + --output-file requirements/ci.txt \ + "$@" \ + requirements/base.txt \ + requirements/test-tools.in + +# Dev depedencies - exact same set as CI + some extra tooling +pip-compile \ + --no-emit-index-url \ + --allow-unsafe \ + --output-file requirements/dev.txt \ + "$@" \ + requirements/ci.txt \ + requirements/dev.in diff --git a/bin/docker_start.sh b/bin/docker_start.sh new file mode 100755 index 00000000..5cdc4903 --- /dev/null +++ b/bin/docker_start.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +set -ex + +# Wait for the database container +# See: https://docs.docker.com/compose/startup-order/ +export PGHOST=${DB_HOST:-db} +export PGPORT=${DB_PORT:-5432} + +fixtures_dir=${FIXTURES_DIR:-/app/fixtures} + +uwsgi_port=${UWSGI_PORT:-8000} +uwsgi_processes=${UWSGI_PROCESSES:-4} +uwsgi_threads=${UWSGI_THREADS:-1} + +mountpoint=${SUBPATH:-/} + +until pg_isready; do + >&2 echo "Waiting for database connection..." + sleep 1 +done + +>&2 echo "Database is up." + +# Apply database migrations +>&2 echo "Apply database migrations" +python src/manage.py migrate + +# Start server +>&2 echo "Starting server" +exec uwsgi \ + --http :$uwsgi_port \ + --http-keepalive \ + --manage-script-name \ + --mount $mountpoint=netland.wsgi:application \ + --static-map /static=/app/static \ + --static-map /media=/app/media \ + --chdir src \ + --enable-threads \ + --processes $uwsgi_processes \ + --threads $uwsgi_threads \ + --post-buffering=8192 \ + --buffer-size=65535 + # processes & threads are needed for concurrency without nginx sitting inbetween diff --git a/bin/flake8_summary.py b/bin/flake8_summary.py new file mode 100755 index 00000000..7a05d5df --- /dev/null +++ b/bin/flake8_summary.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +import sys + + +def main(flake8_output: str): + for line in flake8_output.splitlines(): + file, line, column, error = line.split(":", 3) + print(f"| {file} | {line} | {column} | {error.strip()} |") + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/bin/live2staging.sh b/bin/live2staging.sh new file mode 100644 index 00000000..10cc52e1 --- /dev/null +++ b/bin/live2staging.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# This script should be run on staging. +source live2staging_settings.sh + +# Script starts here +TMP_FILE="$PGSQL_PROD_DB_NAME-livedb.sql" +TMP_MEDIA_FILE="$PGSQL_PROD_DB_NAME-media.tgz" + +if [ -f $TMP_FILE ]; then + echo -n "Create new dump? [y/n] " + read -n 1 yesno +else + yesno="y" +fi +echo + +if [ "$yesno" == "y" ]; then + echo "Creating dump of production database, please wait..." + + PGSQLDUMP_PROD_CMD_ARGS="--dbname=postgresql://$PGSQL_PROD_USERNAME:$PGSQL_PROD_PASSWORD@127.0.0.1:5432/$PGSQL_PROD_DB_NAME " + PGSQLDUMP_PROD_CMD="pg_dump $PGSQLDUMP_PROD_CMD_ARGS > /srv/sites/$TMP_FILE" + + ssh $PGSQL_PROD_HOST "$PGSQLDUMP_PROD_CMD" + scp $PGSQL_PROD_HOST:/srv/sites/$TMP_FILE $TMP_FILE +fi + +echo -n "Continue loading database? [y/n] " +read -n 1 cont +if [ "$cont" == "n" ]; then + echo + exit +fi +echo + +sudo supervisorctl stop $SUPERVISOR_GROUP:* +sudo service cron stop + +echo "Loading dump of production database into staging, please wait..." +sudo su postgres --command="dropdb $PGSQL_STAGING_DB_NAME" +sudo su postgres --command="createdb $PGSQL_STAGING_DB_NAME --owner=$PGSQL_STAGING_USERNAME" +PGSQL_STAGING_CMD="psql -q --dbname=postgresql://$PGSQL_STAGING_USERNAME:$PGSQL_STAGING_PASSWORD@127.0.0.1:5432/$PGSQL_STAGING_DB_NAME " +$PGSQL_STAGING_CMD < $TMP_FILE + +echo "Modifying data on staging to work with production data..." +$PGSQL_STAGING_CMD < $PWD/live2staging_postimport.sql + +echo "Copying media files from production to staging..." +rsync -au $PGSQL_PROD_HOST:$PROD_MEDIA_PATH/ $STAGING_MEDIA_PATH/ + +sudo chmod -R g+rw $STAGING_PROJECT_PATH + +cd $STAGING_PROJECT_PATH +source env/bin/activate + +env/bin/python src/manage.py migrate +env/bin/python src/manage.py locate_translation_files +sudo service cron start +sudo supervisorctl start $SUPERVISOR_GROUP:* + +cd $PWD + +NOW=$(date +"%Y-%m-%d") +FILE="$PGSQL_PROD_DB_NAME-live2staging-$NOW.sql" + +if [ "$yesno" == "y" ]; then + echo "Storing dump as $FILE..." + cp "$PWD/$TMP_FILE" "$FILE" + gzip $FILE +fi diff --git a/bin/live2staging_postimport.sql b/bin/live2staging_postimport.sql new file mode 100644 index 00000000..84d23301 --- /dev/null +++ b/bin/live2staging_postimport.sql @@ -0,0 +1 @@ +UPDATE accounts_user SET email=CONCAT('notify+', REPLACE(email,'@','__at__'), '@maykin.nl'), password='pbkdf2_sha256$12000$pZphUuHkCDke$x8KlvmVInqS1ru9sGZ1tkDStNkMh1xYU1VaG6SHtHMw='; diff --git a/bin/live2staging_settings.sh b/bin/live2staging_settings.sh new file mode 100644 index 00000000..6d2d092a --- /dev/null +++ b/bin/live2staging_settings.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Settings for live2staging.sh +PGSQL_STAGING_DB_NAME="" +PGSQL_STAGING_USERNAME="" +PGSQL_STAGING_PASSWORD="" + +PGSQL_PROD_DB_NAME="" +PGSQL_PROD_USERNAME="" +PGSQL_PROD_PASSWORD="" + +PGSQL_PROD_HOST="" # Example: john@server.nl + +# No trailing slash +PROD_MEDIA_PATH="" +STAGING_PROJECT_PATH="" +STAGING_MEDIA_PATH="$STAGING_PROJECT_PATH/media" + +SUPERVISOR_GROUP="" diff --git a/bin/reset_sequences.sql b/bin/reset_sequences.sql new file mode 100644 index 00000000..5c837874 --- /dev/null +++ b/bin/reset_sequences.sql @@ -0,0 +1,16 @@ +SELECT 'SELECT SETVAL(' || + quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) || + ', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' || + quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';' +FROM pg_class AS S, + pg_depend AS D, + pg_class AS T, + pg_attribute AS C, + pg_tables AS PGT +WHERE S.relkind = 'S' + AND S.oid = D.objid + AND D.refobjid = T.oid + AND D.refobjid = C.attrelid + AND D.refobjsubid = C.attnum + AND T.relname = PGT.tablename +ORDER BY S.relname; diff --git a/bootstrap.py b/bootstrap.py new file mode 100755 index 00000000..2c2e494f --- /dev/null +++ b/bootstrap.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# bootstrap.py +# Bootstrap and setup a virtualenv with the specified requirements.txt +import argparse +import os +import stat +import sys +from shutil import move +from subprocess import call +from tempfile import mkstemp + +description = """ +Set up my development environment for me! +""" + +project_name = 'netland' + +parser = argparse.ArgumentParser(description=description) +parser.add_argument('target', choices=['production', 'staging', 'test', 'jenkins', 'dev'], + help='production/staging/test/jenkins/dev') +parser.add_argument('--project', default=project_name, + help='Name of the project in your src directory, "%s" by default' % project_name) +parser.add_argument('--init', action='store_true', + help='Initialize a fresh "startproject" by pinning the requirements using pip-tools compile. ' + 'Automatically done if requirements/base.txt does not yet exist.') +parser.add_argument('--env', default='env', + help='Directory name for virtualenv, "env" by default') + +args = parser.parse_args() + + +def replace_or_append(file_path, search_val, replace_val): + file_handle, abs_path = mkstemp() + new_file = open(abs_path, 'w') + old_file = open(file_path, 'r') + found = False + for line in old_file: + if line.startswith(search_val): + new_file.write(replace_val) + found = True + else: + new_file.write(line) + if not found: + new_file.write("\n" + replace_val) + new_file.close() + os.close(file_handle) + old_file.close() + os.remove(file_path) + move(abs_path, file_path) + os.chmod(file_path, 436) + + +def replace_wsgi_settings(target): + path = os.path.join('src', project_name, 'wsgi.py') + replace_or_append( + path, 'os.environ.setdefault', + 'os.environ.setdefault("DJANGO_SETTINGS_MODULE", "%s.conf.%s")\n' % (project_name, target) + ) + + +def replace_manage_settings(target): + path = os.path.join('src', project_name, 'manage.py') + replace_or_append( + path, ' os.environ.setdefault', + ' os.environ.setdefault("DJANGO_SETTINGS_MODULE", "%s.conf.%s")\n' % (project_name, target) + ) + + +def append_settings_activate(project, target, env): + if os.name == 'posix': + path = '%s/bin/activate' % env + replace_or_append(path, 'export DJANGO_SETTINGS_MODULE=', + 'export DJANGO_SETTINGS_MODULE=\'%s.conf.%s\'\n' % + (project, target)) + elif os.name == 'nt': + path = '%s\\Scripts\\activate.bat' % env + replace_or_append(path, 'set DJANGO_SETTINGS_MODULE=', + 'set DJANGO_SETTINGS_MODULE=%s.conf.%s\n' % + (project, target)) + path = '%s\\Scripts\\deactivate.bat' % env + replace_or_append(path, 'set DJANGO_SETTINGS_MODULE=', + 'set DJANGO_SETTINGS_MODULE=\n') + + +def pip_compile_pin_requirements(virtualenv): + print('\n== Compiling base requirements ==\n') + if os.name == 'posix': + pip_path = os.path.join(virtualenv, 'bin', 'pip') + elif os.name == 'nt': + pip_path = os.path.join(virtualenv, 'Scripts', 'pip') + cmd_tpl = '{pip} install pip-tools'.format(pip=pip_path) + call(cmd_tpl, shell=True) + print('Error: Run `. env/bin/activate && ./bin/compile_dependencies.sh` to ensure you have requirements/base.txt and requirements/dev.txt') + print('After that rerun bootstrap.py') + sys.exit(1) + +def main(): + virtualenv = args.env + if not hasattr(sys, 'real_prefix'): + print('\n== Creating virtual environment ==\n') + call('virtualenv {0} --python=python3 --prompt="({1}-{2}) "'.format( + virtualenv, args.project, args.target + ), shell=True) + print('\n== Set "%s.conf.%s" as default settings ==\n' % (args.project, args.target)) + append_settings_activate(args.project, args.target, args.env) + + if os.name == 'posix': + # Make manage.py executable + st = os.stat('src/manage.py') + os.chmod('src/manage.py', st.st_mode | stat.S_IEXEC) + django_admin_symlink = os.path.join(virtualenv, 'bin', 'django') + if not os.path.exists(django_admin_symlink): + os.symlink('../../src/manage.py', django_admin_symlink) + + print('\n== Upgrading Pip ==\n') + if os.name == 'posix': + pip_path = os.path.join(virtualenv, 'bin', 'pip') + elif os.name == 'nt': + pip_path = os.path.join(virtualenv, 'Scripts', 'pip') + cmd_tpl = '{pip} install --upgrade pip'.format(pip=pip_path) + call(cmd_tpl, shell=True) + + if args.init \ + or not os.path.exists(os.path.join('requirements', 'base.txt')) \ + or not os.path.exists(os.path.join('requirements', 'dev.txt')): + pip_compile_pin_requirements(virtualenv) + + print('\n== Installing %s requirements ==\n' % args.target) + if os.name == 'posix': + os.environ['TMPDIR'] = '/var/tmp/' + pip_path = os.path.join(virtualenv, 'bin', 'pip') + cmd_tpl = '{pip} install -r requirements/{target}.txt' + elif os.name == 'nt': + pip_path = os.path.join(virtualenv, 'Scripts', 'pip') + cmd_tpl = '{pip} install -r requirements\\{target}.txt' + return call(cmd_tpl.format(pip=pip_path, target=args.target), shell=True) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/build/paths.js b/build/paths.js new file mode 100644 index 00000000..1a07ee45 --- /dev/null +++ b/build/paths.js @@ -0,0 +1,47 @@ +const fs = require('fs'); + + +/** Parses package.json */ +const pkg = JSON.parse(fs.readFileSync('./package.json', 'utf-8')); + +/** Src dir */ +const sourcesRoot = 'src/' + pkg.name + '/'; + +/** "Main" static dir */ +const staticRoot = sourcesRoot + 'static/'; + + +/** + * Application path configuration for use in frontend scripts + */ +module.exports = { + // Parsed package.json + package: pkg, + + // Path to the scss entry point + scssEntry: sourcesRoot + 'scss/screen.scss', + + // Path to the scss (sources) directory + scssSrcDir: sourcesRoot + 'scss/', + + // Path to the js entry point (source) + jsEntry: sourcesRoot + 'js/index.js', + + // Path to js (sources) + jsSrc: sourcesRoot + 'js/**/*.js', + + // Path to the js (sources) directory + jsSrcDir: sourcesRoot + 'js/', + + // Path to the (transpiled) js directory + jsDir: staticRoot + 'bundles/', + + // Path to js spec (test) files + jsSpec: sourcesRoot + 'jstests/**/*.spec.js', + + // Path to js spec (test) entry file + jsSpecEntry: sourcesRoot + 'jstests/index.js', + + // Path to js code coverage directory + coverageDir: 'reports/jstests/', +}; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..6e3045cd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,41 @@ +# Inspired by https://docs.docker.com/compose/django/ +version: '3' + +services: + db: + # NOTE: No persistance storage configured. + # See: https://hub.docker.com/_/postgres/ + image: postgres + # NOTE: this works for bitnami, not sure if this works for regular + # postgres image + volumes: + - ./docker-init-db.sql:/docker-entrypoint-initdb.d/init_db.sql + + web: + build: . + environment: + - DJANGO_SETTINGS_MODULE=netland.conf.docker + - SECRET_KEY=${SECRET_KEY:-django-insecure-a51_t&!%&+7a8km-21((u4hybyv6t_s5=lo=#u&3d-luc0z^)y} + ports: + - 8000:8000 + depends_on: + - db + +# See: src/netland/conf/docker.py +# Optional containers below: +# elasticsearch: +# # NOTE: No persistance storage configured. +# # See: https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html +# image: docker.elastic.co/elasticsearch/elasticsearch:6.2.4 +# container_name: elasticsearch +# environment: +# - discovery.type=single-node +# - cluster.routing.allocation.disk.threshold_enabled=false +# ports: +# - 9200:9200 +# redis: +# # NOTE: No persistance storage configured. +# # See: https://hub.docker.com/_/redis/ +# image: redis +# ports: +# - 6379:6379 diff --git a/docker-init-db.sql b/docker-init-db.sql new file mode 100644 index 00000000..c98654d5 --- /dev/null +++ b/docker-init-db.sql @@ -0,0 +1,4 @@ +CREATE USER {{ project_name|lower }}; +CREATE DATABASE {{ project_name|lower }}; +GRANT ALL PRIVILEGES ON DATABASE {{ project_name|lower }} TO {{ project_name|lower }}; + diff --git a/docs/coding_style/backend.rst b/docs/coding_style/backend.rst new file mode 100644 index 00000000..d2e4ec6d --- /dev/null +++ b/docs/coding_style/backend.rst @@ -0,0 +1,66 @@ +.. _coding_style_backend: + +===================== +Backend coding style +===================== + +The `django coding style`_ is the basis for this styleguide. Some sections dive +a bit deeper or put extra emphasis. + +Imports +======= + +In short: use `isort`_ to check your import ordering. The config file is in +``.isort.cfg``. + +Order and group your imports + +* Use relative imports for your django app +* Ordering: + - future + - standard libraries + - Django components + - third party libraries + - project imports + - local (app) imports + +Example: + +.. code-block:: + + from __future__ import absolute_import, unicode_literals + + import datetime + from datetime import timedelta + + import django.contrib.admin + + import netland.other_app.models + + from .models import SomeModel + +Naming +====== + +* Use plural form for apps. E.g.: ``accounts``, not ``account``. + +* Use singular form for model, view and form class + +Example: + +.. code-block:: + + from netland.accounts.models import Account + + class Idea(models.Model): + pass + + class IdeaForm(forms.ModelForm): + pass + + class IdeaDetailView(views.DetailView): + pass + + +.. _django coding style: https://docs.djangoproject.com/en/stable/internals/contributing/writing-code/coding-style/ +.. _isort: https://pypi.python.org/pypi/isort diff --git a/docs/coding_style/frontend.rst b/docs/coding_style/frontend.rst new file mode 100644 index 00000000..ce60719e --- /dev/null +++ b/docs/coding_style/frontend.rst @@ -0,0 +1,530 @@ +.. _coding_style_frontend: + +===================== +Frontend coding style +===================== + +Shortcuts: + +* :ref:`html` +* :ref:`sass` +* :ref:`javascript` + + +.. _html: + +HTML +==== + +Common +------ + +* Inline style is evil + + .. code-block:: html + +

+ Inline style cannot be cached.
+ Inline style is difficult to overwrite.
+ Inline style makes HTML less readable.
+ Inline style is harder to spot.
+

+ +* Inline script is evil (except Google Analytics) + + .. code-block:: html + + + +* Style your HTML, don't HTML your style (avoid adding divs for style) + + .. code-block:: html + +
+
+
+

... + +* Variables should be passed using data-attributes as well. They are no excuse for inline script. + + .. code-block:: html + +

...
+ + +Elements +-------- + +* Avoid the ``id`` attribute, unless there's a good reason. + + .. code-block:: html + +
+
+ + +
...
+ +* Use `semantic tags`_ like ``
``, ``