From c2f1ba286a15f90ca51c0082befbc4b5c46ea39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Sun, 3 Nov 2024 11:53:32 +0100 Subject: [PATCH] Init project --- .github/workflows/build.yml | 150 +++++ .github/workflows/code-style.yml | 123 ++++ .gitignore | 1 + .pylintrc.ini | 638 ++++++++++++++++++++ Dockerfile | 31 + README.md | 89 ++- dsw/application.yml | 71 +++ dsw/docker-compose.yml | 69 +++ dsw/seed_maker.env | 11 + example.env | 11 + pyproject.toml | 55 ++ requirements.txt | 26 + scripts/build-info.sh | 25 + scripts/run-dev.sh | 3 + setup.py | 3 + src/dsw_seed_maker/__init__.py | 4 + src/dsw_seed_maker/__main__.py | 3 + src/dsw_seed_maker/api.py | 68 +++ src/dsw_seed_maker/cli.py | 71 +++ src/dsw_seed_maker/comm/__init__.py | 4 + src/dsw_seed_maker/comm/db.py | 111 ++++ src/dsw_seed_maker/comm/s3.py | 106 ++++ src/dsw_seed_maker/config.py | 50 ++ src/dsw_seed_maker/consts.py | 20 + src/dsw_seed_maker/logic.py | 7 + src/dsw_seed_maker/models.py | 10 + src/dsw_seed_maker/static/css/style.css | 42 ++ src/dsw_seed_maker/static/js/script.js | 40 ++ src/dsw_seed_maker/templates/index.html.j2 | 25 + src/dsw_seed_maker/templates/layout.html.j2 | 30 + 30 files changed, 1896 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/code-style.yml create mode 100644 .pylintrc.ini create mode 100644 Dockerfile create mode 100644 dsw/application.yml create mode 100644 dsw/docker-compose.yml create mode 100644 dsw/seed_maker.env create mode 100644 example.env create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100755 scripts/build-info.sh create mode 100755 scripts/run-dev.sh create mode 100644 setup.py create mode 100644 src/dsw_seed_maker/__init__.py create mode 100644 src/dsw_seed_maker/__main__.py create mode 100644 src/dsw_seed_maker/api.py create mode 100644 src/dsw_seed_maker/cli.py create mode 100644 src/dsw_seed_maker/comm/__init__.py create mode 100644 src/dsw_seed_maker/comm/db.py create mode 100644 src/dsw_seed_maker/comm/s3.py create mode 100644 src/dsw_seed_maker/config.py create mode 100644 src/dsw_seed_maker/consts.py create mode 100644 src/dsw_seed_maker/logic.py create mode 100644 src/dsw_seed_maker/models.py create mode 100644 src/dsw_seed_maker/static/css/style.css create mode 100644 src/dsw_seed_maker/static/js/script.js create mode 100644 src/dsw_seed_maker/templates/index.html.j2 create mode 100644 src/dsw_seed_maker/templates/layout.html.j2 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..498328a --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,150 @@ +name: Build + +on: + push: + +jobs: + package: + name: Python Package + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.12 + cache: pip + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt + + - name: Prepare Python env + run: | + python -m pip install -U pip setuptools wheel + + - name: Create build info + run: | + bash scripts/build-info.sh + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Install package + run: | + pip install . + + - name: Build package sdist + run: | + python setup.py sdist + + - name: Build package bdist (wheel) + run: | + python setup.py bdist_wheel + + + docker: + name: Docker + runs-on: ubuntu-latest + + env: + PUBLIC_IMAGE_PREFIX: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_IMAGE_NAME: 'dsw-seed-maker' + DOCKER_META_CONTEXT: '.' + DOCKER_META_FILE: 'Dockerfile' + DOCKER_META_PLATFORMS: 'linux/amd64,linux/arm64' + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Create build info + run: | + bash scripts/build-info.sh + + # TEST DOCKER IMAGE BUILD + - name: Docker meta [test] + id: meta-test + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.PUBLIC_IMAGE_PREFIX }}/${{ env.DOCKER_IMAGE_NAME }} + tags: | + type=sha + + - name: Docker build [test] + uses: docker/build-push-action@v4 + with: + context: ${{ env.DOCKER_META_CONTEXT }} + file: ${{ env.DOCKER_META_FILE }} + platforms: ${{ env.DOCKER_META_PLATFORMS }} + push: false + tags: ${{ steps.meta-test.outputs.tags }} + labels: ${{ steps.meta-test.outputs.labels }} + + # PREPARE + - name: Docker login [docker.io] + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + # DEVELOPMENT IMAGES + - name: Docker meta [dev] + id: meta-dev + if: github.event_name != 'pull_request' + uses: docker/metadata-action@v4 + with: + images: | + ${{ secrets.DOCKER_HUB_USERNAME }}/${{ env.DOCKER_IMAGE_NAME }} + tags: | + type=ref,event=branch + + - name: Docker build+push [dev] + uses: docker/build-push-action@v4 + if: github.event_name != 'pull_request' && steps.meta-dev.outputs.tags != '' + with: + context: ${{ env.DOCKER_META_CONTEXT }} + file: ${{ env.DOCKER_META_FILE }} + platforms: ${{ env.DOCKER_META_PLATFORMS }} + push: true + tags: ${{ steps.meta-dev.outputs.tags }} + labels: ${{ steps.meta-dev.outputs.labels }} + + # PUBLIC IMAGES (latest, semver) + - name: Docker meta [public] + id: meta-public + if: github.event_name != 'pull_request' + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.PUBLIC_IMAGE_PREFIX }}/${{ env.DOCKER_IMAGE_NAME }} + tags: | + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} + + - name: Docker build+push [public] + uses: docker/build-push-action@v4 + if: github.event_name != 'pull_request' && steps.meta-public.outputs.tags != '' + with: + context: ${{ env.DOCKER_META_CONTEXT }} + file: ${{ env.DOCKER_META_FILE }} + platforms: ${{ env.DOCKER_META_PLATFORMS }} + push: true + tags: ${{ steps.meta-public.outputs.tags }} + labels: ${{ steps.meta-public.outputs.labels }} diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml new file mode 100644 index 0000000..4622016 --- /dev/null +++ b/.github/workflows/code-style.yml @@ -0,0 +1,123 @@ +name: Code Style + +on: + push: + +jobs: + # Flake 8 for basic code style checks + flake8: + name: Flake 8 + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + cache: pip + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt + + - name: Create build info + run: | + bash scripts/build-info.sh + + - name: Install Flake8 (5.0.4) + run: | + python -m pip install --upgrade pip + pip install flake8==5.0.4 + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Install package + run: | + pip install . + + - name: Lint with flake8 + run: | + flake8 src/dsw_seed_maker --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 src/dsw_seed_maker --count --max-complexity=12 --max-line-length=130 --statistics + + # Typing checks with MyPy + typing: + name: Typing + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + cache: pip + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt + + - name: Create build info + run: | + bash scripts/build-info.sh + + - name: Install MyPy (1.4.1) + run: | + python -m pip install --upgrade pip + pip install mypy==1.4.1 + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Install packages + run: | + pip install . + + - name: Check typing with MyPy + run: | + mypy --install-types --ignore-missing-imports --check-untyped-defs --non-interactive src/dsw_seed_maker + + # Pylint linting + pylint: + name: Pylint + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + cache: pip + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt + + - name: Create build info + run: | + bash scripts/build-info.sh + + - name: Install PyLint (3.2.5) + run: | + python -m pip install --upgrade pip + pip install pylint==3.2.5 + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Install packages + run: | + pip install . + + - name: Lint with PyLint + run: | + pylint --rcfile=.pylintrc.ini src/dsw_seed_maker \ No newline at end of file diff --git a/.gitignore b/.gitignore index 82f9275..059950c 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,4 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +.env diff --git a/.pylintrc.ini b/.pylintrc.ini new file mode 100644 index 0000000..54f50b1 --- /dev/null +++ b/.pylintrc.ini @@ -0,0 +1,638 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + db, + s3, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=10 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=1 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + missing-function-docstring, + missing-module-docstring, + missing-class-docstring, + broad-exception-caught + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=XXX +#notes=FIXME, +# XXX, +# TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..aec2f76 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM datastewardshipwizard/python-base:3.11-basic AS builder + +WORKDIR /app + +COPY . /app + +RUN python -m pip wheel --no-cache-dir --wheel-dir=/app/wheels -r /app/requirements.txt \ + && python -m pip wheel --no-cache-dir --no-deps --wheel-dir=/app/wheels /app + + +FROM datastewardshipwizard/python-base:3.11-basic + +ENV PATH="/home/user/.local/bin:$PATH" + +# Setup non-root user +USER user + +# Prepare dirs +WORKDIR /home/user + +# Install Python packages +COPY --from=builder --chown=user:user /app/wheels /home/user/wheels +RUN pip install --break-system-packages --user --no-cache --no-index /home/user/wheels/* \ + && rm -rf /home/user/wheels + +# Run +CMD ["uvicorn", "dsw_seed_maker:app", \ + "--proxy-headers", \ + "--forwarded-allow-ips=*", \ + "--host=0.0.0.0", \ + "--port=8000"] diff --git a/README.md b/README.md index f2dfa1f..5bf2a44 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,89 @@ -# dsw-seed-maker +# DSW Seed Maker + Maker for DSW data seeds (reproducible content packages) + +## Usage + +This application serves to create data seeding recipes for Data Stewardship Wizard, +more specifically its [Data Seeder](https://github.com/ds-wizard/engine-tools/tree/develop/packages/dsw-data-seeder) tool. + +### Installation + +You can install it locally with Python in a standard way (preferably to a prepared +[virtual environment](https://docs.python.org/3/library/venv.html)). + +```shell +git clone git@github.com:ds-wizard/dsw-seed-maker.git +cd dsw-seed-maker +pip install . +``` + +### Configuration + +All configuration is purely done via environment variables. For convenience, +it is recommended to use a `.env` file in a standard way (see [`example.env`](example.env)) +for details. + +### Command Line Interface (CLI) + +After installation, you can use the CLI: + +```shell +dsw-seed-maker --help +dsw-seed-maker --version +dsw-seed-maker make-seed --help +``` + +### Web Application + +A simple web application can be launched as this Python package contains +[WSGI](https://wsgi.readthedocs.io/en/latest/index.html)-compliant application +`app` exposed through the main module `dsw_seed_maker`, i.e. `dsw_seed_maker`. + +There is a simple way to run the web application: + +```shell +dsw-seed-maker run-web +``` + +## Development + +There are some specifics to develop + +### Environment + +1. Clone the repository (eventually switch branch) +2. Setup a virtual environment (`python -m venv env`) and activate it +3. Install the package as editable (`pip install -e .`) +4. Prepare your local DSW instance with DB and S3 (see [`dsw/docker-compose.yml`](dsw/docker-compose.yml)) +5. Create `.env` file (see [`example.env`](example.env)) +6. Run CLI or web app (`./scripts/run-dev.sh`), develop and try it out + +### Code Style and Best Practices + +- Use type annotations in Python (`mypy` is used in GitHub Actions) +- Comply with PEP8 and other code style conventions (checked in GitHub Actions using `flake8` and `pylint`) +- Name variables, classes, and functions as sensible human being so it is understandable for others +- Use descriptive commit messages (short message = 3-7 words, but you can add then description if suitable) + +### Resources + +Within Python scripts: + +- [Python (3.12+)](https://docs.python.org/3/) +- [Click](https://click.palletsprojects.com/en/stable/) +- [FastAPI](https://fastapi.tiangolo.com/) +- [Jinja](https://jinja.palletsprojects.com/en/stable/templates/) +- [Pydantic](https://docs.pydantic.dev/latest/) +- [Psycopg 3](https://www.psycopg.org/psycopg3/) + +For the web app simple frontend: + +- [Boostrap](https://getbootstrap.com/) +- [jQuery](https://jquery.com/) + +## License + +This project is licensed under the Apache License v2.0 - see the +[LICENSE](LICENSE) file for more details. + diff --git a/dsw/application.yml b/dsw/application.yml new file mode 100644 index 0000000..6763e0b --- /dev/null +++ b/dsw/application.yml @@ -0,0 +1,71 @@ +# (!!) Change the secret and RDA private key to your own +general: + clientUrl: http://localhost:8080/wizard + secret: b7zFNUhFYIFLmDSFKFkp0Bnp8Z3uWGnH + rsaPrivateKey: | + -----BEGIN RSA PRIVATE KEY----- + MIIJKAIBAAKCAgEAyMMyCZe+/oihq702T+Yb2qiqgpmSmH9krH5lX5ZlT94KTiIn + KjNsMg+SLjIWedU+d5R8uUxqXX50MAXYwfi49NDpSi1hDTfxw2O/J03P+SuzwDTU + LMWQjBCT0t1QZOD7ODhPL7+UvZsoujHQJEwJ32Hj99z8HXOJ+pEvLwlmv3te1Tpr + SK9XvzjobVBz+XJL+WkQYFTAnhJHuM7Xzz3wQsiF60vCMe8yFqAlppd+6rybLdl5 + aXn6o4kzpDUI1/izRHW093sOkhAP+Esch7VJpEfLDpqDu2DDml/hL3KXFevau6mt + sWJZ0qyswh1BrdtwKKm2eqzjqw+Q0sThJUA66r1ZzHljqt8E3OSJlFHUgIZXXx1o + G8DNzu5jcjy6T85YSzDg8upIgNgoT14kgYUWnZsIKiXvhAX9WdkSVAHOkM0ADN8P + PTKekgAO4YHbM1hsraGpXJVJf6AdAwYwC4burLrdwiERafXV1FEpcnaSBezjAs3P + PmFGc7tCUocKfxHY3fJqU2n/Ngw8cDxZaqbPR1K+RXPzAYN15PBdtO8VnubKZTBw + x0tPXxm7rz87duP2szLpjaIEvks77As22sOptyzYO39iA57Vs4+onxV/0J/quDTv + Nci4ZdWSPABoyqtoiswrud5hzr75Zo0t8FutkoI9yQXyJJbgHFhSy5A7o0ECAwEA + AQKCAgAPyRFJIc5ZsyKLH2CC3cP46GQnyq11IicnTPTplCi5I+tJg3Py5IIwWXsT + +Mo9oB++RUWdSD3gxg2arQsDDWx1i2uccn78DBcNVu1zyLF+lQoYJc89Cfe4m6ET + 5mTo2avHGt5XEcg/NFD6euozwGrSTp8fIRgNtMT3l6zPFP0oNyoYt/Qxjd/nYDEe + T0IfLfZ9UN6BxIg0pZlQFatwVx1qEtw8yQMm6/jTQJpVQ/otuNC2xS+/tGCZtTMD + SUw9twdZbavh/aICqBKpYjaHESEOMrrrlgiflhcrLzjEg00P4zCaqtfGZ8ZKQ1t+ + E9DTO7rQ5686bBemnp2zhNlzFV7MbGyXHxYUh7hbSHGJwGrSbgDCZEVOllcgALTF + RsUxivBAmkyytYlT79QqSUrMAicf8ys2jW0suHuxT/gNqnb4iGtzoGv3OVXORsPt + TlxfTHIo00yF9OdCYTWrsi2XFwBgaasc7eH4KKTMpFOSjIq3iu+zZiLGv20z04Xi + iSnG4rN/nZS6MYTt50XM1lZb4XE5SWw35Az/BVLeQ4T2yanrNnQr87mdmlrh/LHy + ARInp1rHCaUlodWQCS9loEGGVPxGaetSUPhPPt3xMEdFvjvYHvp/rPRQc5iVZOC7 + +Av8gctVL4t4qsU1aPVVwmyyvZhW0cPrEDd8ap+usAwYW/iY1QKCAQEA+pJuZz+O + tDkUP//p7uNRmc54yWLVMZ1IdrTeqNVRJyHVNvxCzYX3GC2UOQz/6IECf/eCP3Z/ + /IVnYXRsoGKNght7it24hoSGsLSQO6gCYa7Xlmf72VyBWCluE+9KlR4I8LmvfYrC + tYGCazL0qbHwXCBdw0oL1HvSEEGMPt/TpdDkMtyvi+OgmKM6BBiCVC64qVjFGtB1 + CRxchbHN56mETa63m9wC6C/mIl+veysC0nE745h5i+ocu0LLEpfGA6tSp3XyNii1 + fc6LwmA66JsamRqg5IUdj4/AnHvO/XnKyVlzIwLTSWaoGlBHjGHHYqJsoQk/SryX + bAYBcJsB+MYUewKCAQEAzRyKjOPLMvL7IJjbnxNrl3k7IVAAR6o5RgNuMQ7b+Fmb + dPwi6SQJOHKnAHlC3lqUyCQxoW5tOeLE1A2At2LSfNJh8M3tiAhgoamA5yD5x1MC + 1RZK9ADmnDS3ezS3xixHSJl8dPsmgB7M8usrrIVD7xcVbMc6SjkbWPJD3rsJZHXz + b1phu0avUx9kN2Y0yjOGCPSQSDMjY6bOMpJaDmRcKCq1+z22OtuxOYlFttdvRhxf + sODv+FYN+estm2okko5lAhfnZwApp8APmMxKRFzebNMKE6gTKQ+OvquSexx6jakz + PTj8h0aOUQ/sMU/aXWSNDzjE8WHvB/jKr4p1lG5QcwKCAQBHt/3No41u3B+FqfQF + Ih9LDKPZ0GYBC2mgQGiynkvaDX7/e/iK5tswmmsgYS4mGDbk9azcCUjcuDo81Jv6 + 0/p40PDtcETOi8QZc7UMvivgVbDzJdQaBLYoy8LsBcOtJqmcbSljn4HXfbgcgrsw + jvOjCsiLivgT21TmvjwsCtiDT2ThgfTk70QaLC4A2hsDiVPldLkF0cCN2QZKZffv + ZL+vlGCNRc23M26FTCnn8QmfvjVMEWpIt7t9hcKmEbblefXwZHvtMH2yAppSCkUS + KsLUK6D9h3RMkPKqV22Jc4CtXOLtlHNxQObAkVXJQntVKfQ5q3HoLE6S5q0m09ce + o2NdAoIBAEVbflpP+Pk8gsCuD63F8l7hJqaIF+Aj5oQPSuaZ4LYlDtVLIT726YtF + 8cTOPFdWB/N5kbLF1Xd6gBY4jqr0ZrrApU+5QrX6JzrjR2xgkTXekxmdFTFXUKPH + i7fExtPjSyN4i0Y/WvB9+c22yijnU2Z4xszmvYY1s1KrcVnSo0VuQSgJRvz91PFl + rwywqe8p5wwGLvfLGw9qa1VcecleFG+cqbbQUluqyE3tZj6CsOTjWVRFa7/s0YMO + DHj+Nmm8btl7XctNmchYwK1pfz+Iq8nevwyQjBIZaap99+VbZvGaInJ0X6gaqqss + mDDAkNV919kq2CXLePmYHmmnh0Tu3JMCggEBAL1wmSfX55tVCQRSVl7ahNpqGY7C + 7Syyd0z8vE3F+czsd8H0j5KGBgKSUbtiJDhB1BSLcIwaUczJFiFXpegkTXnSz1Id + 2H+riQDon8rVkdP8JP6ny5ncj4PQ5ZAE7gwMPdtjhY4WChfHMXGgyfubNKIGi0pP + U8QopElb2l9/YhTpSvXXHXLYAq8bz3Kye6jQqatoDwRCuHkPc0OUeT9NdtW+x/Y4 + cdU04pDO6DEYKYcD8vgSlEzVCNqb6TvFfRsfQXI5TSkKuX3fTR2wgLR6CxWIejSE + 32jnWA89Tg9wD6ACZo8On4IIVH4oeFuNi8mSb/M3pQ9hTtqIUqwUw+AxD+I= + -----END RSA PRIVATE KEY----- + +# (!!) Change default password +database: + connectionString: postgresql://postgres:postgres@postgres:15432/wizard + +# (!!) Change default password +s3: + url: http://host.docker.internal:9000 + username: minio + password: minioPassword + bucket: wizard + +# (!!) Configure SMTP +mail: + enabled: false diff --git a/dsw/docker-compose.yml b/dsw/docker-compose.yml new file mode 100644 index 0000000..5abae47 --- /dev/null +++ b/dsw/docker-compose.yml @@ -0,0 +1,69 @@ +services: + + dsw-server: + image: datastewardshipwizard/wizard-server:4.11 + platform: linux/amd64 + restart: always + ports: + - 127.0.0.1:3000:3000 + depends_on: + - postgres + - minio + volumes: + - ./application.yml:/app/config/application.yml:ro + extra_hosts: + - host.docker.internal:host-gateway + + dsw-client: + image: datastewardshipwizard/wizard-client:4.11 + restart: always + ports: + - 127.0.0.1:8080:8080 + environment: + API_URL: http://localhost:3000/wizard-api + + dsw-docworker: + image: datastewardshipwizard/document-worker:4.11 + depends_on: + - postgres + - minio + - dsw-server + volumes: + - ./application.yml:/app/config/application.yml:ro + extra_hosts: + - host.docker.internal:host-gateway + + postgres: + image: postgres:16.4 + restart: always + ports: + - 127.0.0.1:15432:5432 + environment: + POSTGRES_DB: wizard + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + volumes: + - db-data:/var/lib/postgresql/data + + minio: + image: minio/minio:RELEASE.2022-02-24T22-12-01Z + restart: always + command: server /data --console-address ":9001" + ports: + - 9000:9000 + - 9001:9001 + environment: + MINIO_ROOT_USER: minio + MINIO_ROOT_PASSWORD: minioPassword + volumes: + - s3-data:/data + + dsw-seed-maker: + image: datastewardshipwizard/dsw-seed-maker:develop + ports: + - 127.0.0.1:8000:8000 + env_file: seed_maker.env + +volumes: + db-data: + s3-data: diff --git a/dsw/seed_maker.env b/dsw/seed_maker.env new file mode 100644 index 0000000..8898eb1 --- /dev/null +++ b/dsw/seed_maker.env @@ -0,0 +1,11 @@ +API_ROOT_PATH='' + +DSW_DB_CONN_STR=postgresql://postgres:postgres@postgres:5432/wizard + +DSW_S3_URL=http://host.docker.internal:9000 +DSW_S3_USERNAME=minio +DSW_S3_PASSWORD=minioAdmin +DSW_S3_BUCKET=wizard +DSW_S3_REGION=eu-central-1 + +LOG_LEVEL=DEBUG diff --git a/example.env b/example.env new file mode 100644 index 0000000..159856f --- /dev/null +++ b/example.env @@ -0,0 +1,11 @@ +API_ROOT_PATH='' + +DSW_DB_CONN_STR=postgresql://postgres:postgres@localhost:15432/wizard + +DSW_S3_URL=http://localhost:9000 +DSW_S3_USERNAME=minio +DSW_S3_PASSWORD=minioAdmin +DSW_S3_BUCKET=wizard +DSW_S3_REGION=eu-central-1 + +LOG_LEVEL=DEBUG diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7c7b001 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,55 @@ +[build-system] +requires = ['setuptools'] +build-backend = 'setuptools.build_meta' + +[project] +name = 'dsw-seed-maker' +version = '0.1.0' +description = 'Maker for DSW data seeds (reproducible content packages)' +readme = 'README.md' +keywords = ['dsw', 'seed', 'data', 'reproducibility', 'SQL', 'S3'] +license = { file = 'LICENSE' } +authors = [ + { name = 'Marek Suchánek', email = 'marek.suchanek@ds-wizard.org' }, + { name = 'Jana Martínková', email = 'jana.martinkova@ds-wizard.org' }, + { name = 'Kryštof Komanec', email = 'krystof.komanec@ds-wizard.org' }, +] +classifiers = [ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.12', + 'Topic :: Text Processing', + 'Topic :: Utilities', +] +requires-python = '>=3.11, <4' +dependencies = [ + 'click', + 'fastapi', + 'jinja2', + 'minio', + 'psycopg[binary]', + 'python-dotenv', + 'tenacity', + 'uvicorn', +] + +[project.urls] +Homepage = 'https://ds-wizard.org' +Repository = 'https://github.com/ds-wizard/dsw-seed-maker' + +[project.scripts] +dsw-seed-maker = 'dsw_seed_maker:cli' + +[tool.setuptools] +zip-safe = false + +[tool.setuptools.packages.find] +namespaces = true +where = ['src'] + +[tool.setuptools.package-data] +'*' = ['*.css', '*.js', '*.j2'] + +[tool.distutils.bdist_wheel] +universal = true diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..bea7b51 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,26 @@ +annotated-types==0.7.0 +anyio==4.6.2.post1 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +certifi==2024.8.30 +cffi==1.17.1 +click==8.1.7 +fastapi==0.115.4 +h11==0.14.0 +idna==3.10 +Jinja2==3.1.4 +MarkupSafe==3.0.2 +minio==7.2.10 +psycopg==3.2.3 +psycopg-binary==3.2.3 +pycparser==2.22 +pycryptodome==3.21.0 +pydantic==2.9.2 +pydantic_core==2.23.4 +python-dotenv==1.0.1 +sniffio==1.3.1 +starlette==0.41.2 +tenacity==9.0.0 +typing_extensions==4.12.2 +urllib3==2.2.3 +uvicorn==0.32.0 diff --git a/scripts/build-info.sh b/scripts/build-info.sh new file mode 100755 index 0000000..3ce9439 --- /dev/null +++ b/scripts/build-info.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +# File with build info +BUILD_INFO_FILE=src/dsw_seed_maker/consts.py + +# Create version based on git tag or branch +branch=$(git rev-parse --abbrev-ref HEAD) +commit=$(git rev-parse --short HEAD) +version="$branch~$commit" +gittag=$(git tag -l --contains HEAD | head -n 1) +if test -n "$gittag" +then + version="$gittag~$commit" +fi + +# Get build timestamp +builtAt=$(date +"%Y-%m-%d %TZ") + +cat $BUILD_INFO_FILE +# Replace values +sed -i.bak "s#--BUILT_AT--#$version#" $BUILD_INFO_FILE && rm $BUILD_INFO_FILE".bak" +sed -i.bak "s#--VERSION--#$builtAt#" $BUILD_INFO_FILE && rm $BUILD_INFO_FILE".bak" + +cat $BUILD_INFO_FILE diff --git a/scripts/run-dev.sh b/scripts/run-dev.sh new file mode 100755 index 0000000..96cc507 --- /dev/null +++ b/scripts/run-dev.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +uvicorn dsw_seed_maker:app --reload --proxy-headers --forwarded-allow-ips=* --host 0.0.0.0 --port 8000 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b908cbe --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +import setuptools + +setuptools.setup() diff --git a/src/dsw_seed_maker/__init__.py b/src/dsw_seed_maker/__init__.py new file mode 100644 index 0000000..888a3a8 --- /dev/null +++ b/src/dsw_seed_maker/__init__.py @@ -0,0 +1,4 @@ +from .api import app +from .cli import cli + +__all__ = ['app', 'cli'] diff --git a/src/dsw_seed_maker/__main__.py b/src/dsw_seed_maker/__main__.py new file mode 100644 index 0000000..febeaef --- /dev/null +++ b/src/dsw_seed_maker/__main__.py @@ -0,0 +1,3 @@ +from .cli import cli + +cli(obj={}) diff --git a/src/dsw_seed_maker/api.py b/src/dsw_seed_maker/api.py new file mode 100644 index 0000000..c91133a --- /dev/null +++ b/src/dsw_seed_maker/api.py @@ -0,0 +1,68 @@ +import contextlib +import logging +import pathlib + +import fastapi +import fastapi.responses +import fastapi.staticfiles +import fastapi.templating + +from .config import Config +from .consts import NICE_NAME, VERSION +from .models import ExampleRequestDTO, ExampleResponseDTO +from .logic import example_logic + + +LOG = logging.getLogger('uvicorn.error') +ROOT = pathlib.Path(__file__).parent +STATIC_DIR = ROOT / 'static' +TEMPLATES_DIR = ROOT / 'templates' + + +@contextlib.asynccontextmanager +async def lifespan(fastapi_app: fastapi.FastAPI): + Config.apply_logging() + Config.check() + LOG.debug('Configured logging when starting app "%s"', repr(fastapi_app)) + yield + + +app = fastapi.FastAPI( + title=NICE_NAME, + version=VERSION, + lifespan=lifespan, + root_path=Config.API_ROOT_PATH, +) +app.mount('/static', fastapi.staticfiles.StaticFiles(directory=STATIC_DIR), name='static') +templates = fastapi.templating.Jinja2Templates(directory=TEMPLATES_DIR) + + +# HTML endpoints / controllers +@app.get('/', response_class=fastapi.responses.HTMLResponse) +async def get_example(request: fastapi.Request): + return templates.TemplateResponse( + name='index.html.j2', + request=request, + ) + + +# API/JSON endpoints / controllers +@app.post('/api/example', response_model=ExampleResponseDTO) +async def post_example(req_dto: ExampleRequestDTO, request: fastapi.Request): + LOG.debug('Handling example request...') + LOG.debug('Host: %s', request.client.host if request.client else 'unknown') + if req_dto.magic_code != 'example': + raise fastapi.HTTPException(status_code=400, detail='Magic code is not correct :-(') + return example_logic(req_dto) + + +@app.get('/api/resources', response_class=fastapi.responses.JSONResponse) +async def get_resources(): + # TODO: implement + return fastapi.responses.JSONResponse(content={'resources': []}) + + +@app.post('/api/seed-package', response_class=fastapi.responses.JSONResponse) +async def post_seed(): + # TODO: implement + return fastapi.responses.JSONResponse(content={'status': 'ok'}) diff --git a/src/dsw_seed_maker/cli.py b/src/dsw_seed_maker/cli.py new file mode 100644 index 0000000..7468f05 --- /dev/null +++ b/src/dsw_seed_maker/cli.py @@ -0,0 +1,71 @@ +import json +import pathlib + +import click + +from .config import Config +from .consts import DEFAULT_ENCODING, PACKAGE_VERSION + + +class AliasedGroup(click.Group): + + def get_command(self, ctx, cmd_name): + rv = click.Group.get_command(self, ctx, cmd_name) + if rv is not None: + return rv + matches = [x for x in self.list_commands(ctx) + if x.startswith(cmd_name)] + if not matches: + return None + if len(matches) == 1: + return click.Group.get_command(self, ctx, matches[0]) + return ctx.fail(f'Too many matches: {', '.join(sorted(matches))}') + + +@click.group(cls=AliasedGroup) +@click.version_option(version=PACKAGE_VERSION, message=f'%(prog)s v{PACKAGE_VERSION}') +def cli(): + Config.apply_logging() + + +@cli.command(help='Run the web application', name='run-web') +@click.option('-h', '--host', default='0.0.0.0', + help='Host address to bind to') +@click.option('-p', '--port', default=8000, + help='Port to bind to') +def run_web(host: str, port: int): + import uvicorn # pylint: disable=import-outside-toplevel + from .api import app # pylint: disable=import-outside-toplevel + Config.check() + uvicorn.run(app, host=host, port=port) + + +@cli.command(help='Example command', name='example') +def example(): + click.echo('Hello, world!') + + +@cli.command(help='List all available seed resources', name='list') +@click.option('-o', '--output', + type=click.File('w', encoding=DEFAULT_ENCODING), default='-', + help='Output file to write to (JSON)') +def list_resources(output_fp): + Config.check() + # TODO: Implement list command (do it in logic, import & use here) + output_fp.write(json.dumps({'resources': []})) + + +@cli.command(help='Create a seed package from input', name='make-seed') +@click.option('-i', '--input', + type=click.File('r', encoding=DEFAULT_ENCODING), default='-', + help='Input file to read from (JSON)') +@click.option('-o', '--output-dir', + type=click.Path(dir_okay=True, file_okay=False), default='-', + help='Output directory to write to') +def make_seed(input_fp, output_dir): + Config.check() + data = json.load(input_fp) + out_dir = pathlib.Path(output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + # TODO: Implement list command (do it in logic, import & use here) + print(data) diff --git a/src/dsw_seed_maker/comm/__init__.py b/src/dsw_seed_maker/comm/__init__.py new file mode 100644 index 0000000..34f2185 --- /dev/null +++ b/src/dsw_seed_maker/comm/__init__.py @@ -0,0 +1,4 @@ +from .db import Database, DatabaseConnection +from .s3 import S3Storage + +__all__ = ['Database', 'DatabaseConnection', 'S3Storage'] diff --git a/src/dsw_seed_maker/comm/db.py b/src/dsw_seed_maker/comm/db.py new file mode 100644 index 0000000..d8234fe --- /dev/null +++ b/src/dsw_seed_maker/comm/db.py @@ -0,0 +1,111 @@ +import logging +import psycopg +import psycopg.connection +import psycopg.conninfo +import psycopg.rows +import psycopg.types.json +import tenacity + + +LOG = logging.getLogger(__name__) + +RETRY_QUERY_MULTIPLIER = 0.5 +RETRY_QUERY_TRIES = 3 + +RETRY_CONNECT_MULTIPLIER = 0.2 +RETRY_CONNECT_TRIES = 10 + + +def wrap_json_data(data: dict): + return psycopg.types.json.Json(data) + + +class Database: + + def __init__(self, name: str, dsn: str, timeout: int = 30000, + autocommit: bool = False): + self._db = DatabaseConnection( + name=name, + dsn=dsn, + timeout=timeout, + autocommit=autocommit, + ) + self._db.connect() + + def __str__(self): + return f'DB[{self._db.name}]' + + @tenacity.retry( + reraise=True, + wait=tenacity.wait_exponential(multiplier=RETRY_QUERY_MULTIPLIER), + stop=tenacity.stop_after_attempt(RETRY_QUERY_TRIES), + before=tenacity.before_log(LOG, logging.DEBUG), + after=tenacity.after_log(LOG, logging.DEBUG), + ) + def execute_query(self, query: psycopg.connection.Query, **kwargs): + with self._db.new_cursor(use_dict=True) as cursor: + cursor.execute(query=query, params=kwargs) + + +class DatabaseConnection: + + def __init__(self, name: str, dsn: str, timeout: int, autocommit: bool): + self.name = name + self.dsn = psycopg.conninfo.make_conninfo( + conninfo=dsn, + connect_timeout=timeout, + ) + self.autocommit = autocommit + self._connection: psycopg.Connection | None = None + + @tenacity.retry( + reraise=True, + wait=tenacity.wait_exponential(multiplier=RETRY_CONNECT_MULTIPLIER), + stop=tenacity.stop_after_attempt(RETRY_CONNECT_TRIES), + before=tenacity.before_log(LOG, logging.DEBUG), + after=tenacity.after_log(LOG, logging.DEBUG), + ) + def _connect_db(self): + LOG.info('Creating connection to PostgreSQL database "%s"', self.name) + try: + connection: psycopg.Connection = psycopg.connect( + conninfo=self.dsn, + autocommit=self.autocommit, + ) + except Exception as e: + LOG.error('Failed to connect to PostgreSQL database "%s": %s', self.name, e) + raise e + # test connection + cursor = connection.cursor() + cursor.execute(query='SELECT 1;') + result = cursor.fetchone() + if result is None: + raise RuntimeError('Failed to verify DB connection') + LOG.debug('DB connection verified (result=%s)', result[0]) + cursor.close() + connection.commit() + self._connection = connection + + def connect(self): + if not self._connection or self._connection.closed != 0: + self._connect_db() + + @property + def connection(self): + self.connect() + return self._connection + + def new_cursor(self, use_dict: bool = False): + return self.connection.cursor( + row_factory=psycopg.rows.dict_row if use_dict else psycopg.rows.tuple_row, + ) + + def reset(self): + self.close() + self.connect() + + def close(self): + if self._connection: + LOG.info('Closing connection to PostgreSQL database "%s"', self.name) + self._connection.close() + self._connection = None diff --git a/src/dsw_seed_maker/comm/s3.py b/src/dsw_seed_maker/comm/s3.py new file mode 100644 index 0000000..1960643 --- /dev/null +++ b/src/dsw_seed_maker/comm/s3.py @@ -0,0 +1,106 @@ +import contextlib +import io +import logging +import pathlib +import tempfile + +import minio +import minio.error +import tenacity + +LOG = logging.getLogger(__name__) + +DOCUMENTS_DIR = 'documents' + +RETRY_S3_MULTIPLIER = 0.5 +RETRY_S3_TRIES = 3 + + +@contextlib.contextmanager +def temp_binary_file(data: bytes): + file = tempfile.TemporaryFile() + file.write(data) + file.seek(0) + yield file + file.close() + + +class S3Storage: + + def _get_endpoint(self): + parts = self._url.split('://', maxsplit=1) + return parts[0] if len(parts) == 1 else parts[1] + + def __init__(self, url: str, username: str, password: str, + bucket: str, region: str, multi_tenant: bool = True): + self.multi_tenant = multi_tenant + self._url = url + self._username = username + self._password = password + self._bucket = bucket + self._region = region + + self.client = minio.Minio( + endpoint=self._get_endpoint(), + access_key=self._username, + secret_key=self._password, + secure=self._url.startswith('https://'), + region=self._region, + ) + + def __str__(self): + return f'{self._url}/{self._bucket}' + + @tenacity.retry( + reraise=True, + wait=tenacity.wait_exponential(multiplier=RETRY_S3_MULTIPLIER), + stop=tenacity.stop_after_attempt(RETRY_S3_TRIES), + before=tenacity.before_log(LOG, logging.DEBUG), + after=tenacity.after_log(LOG, logging.DEBUG), + ) + def ensure_bucket(self): + found = self.client.bucket_exists(self._bucket) + if not found: + self.client.make_bucket(self._bucket) + + @tenacity.retry( + reraise=True, + wait=tenacity.wait_exponential(multiplier=RETRY_S3_MULTIPLIER), + stop=tenacity.stop_after_attempt(RETRY_S3_TRIES), + before=tenacity.before_log(LOG, logging.DEBUG), + after=tenacity.after_log(LOG, logging.DEBUG), + ) + def download_file(self, file_name: str, target_path: pathlib.Path) -> bool: + try: + self.client.fget_object( + bucket_name=self._bucket, + object_name=file_name, + file_path=str(target_path), + ) + except minio.error.S3Error as e: + if e.code != 'NoSuchKey': + raise e + return False + return True + + @tenacity.retry( + reraise=True, + wait=tenacity.wait_exponential(multiplier=RETRY_S3_MULTIPLIER), + stop=tenacity.stop_after_attempt(RETRY_S3_TRIES), + before=tenacity.before_log(LOG, logging.DEBUG), + after=tenacity.after_log(LOG, logging.DEBUG), + ) + def store_object(self, tenant_uuid: str, object_name: str, + content_type: str, data: bytes, + metadata: dict | None = None): + if self.multi_tenant: + object_name = f'{tenant_uuid}/{object_name}' + with io.BytesIO(data) as file: + self.client.put_object( + bucket_name=self._bucket, + object_name=object_name, + data=file, + length=len(data), + content_type=content_type, + metadata=metadata, + ) diff --git a/src/dsw_seed_maker/config.py b/src/dsw_seed_maker/config.py new file mode 100644 index 0000000..45889ff --- /dev/null +++ b/src/dsw_seed_maker/config.py @@ -0,0 +1,50 @@ +import logging +import os +import pathlib + +import dotenv + +from .consts import DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL + + +LOG = logging.getLogger(__name__) + +DOTENV_PATH = pathlib.Path('.env') +if DOTENV_PATH.exists(): + dotenv.load_dotenv(DOTENV_PATH) + + +class Config: + API_ROOT_PATH = os.getenv('API_ROOT_PATH', '') + + DSW_DB_CONN_STR = os.getenv('DSW_DB_CONN_STR') + + DSW_S3_URL = os.getenv('DSW_S3_URL') + DSW_S3_USERNAME = os.getenv('DSW_S3_USERNAME') + DSW_S3_PASSWORD = os.getenv('DSW_S3_PASSWORD') + DSW_S3_BUCKET = os.getenv('DSW_S3_BUCKET') + DSW_S3_REGION = os.getenv('DSW_S3_REGION', 'eu-central-1') + + LOG_LEVEL = os.getenv('LOG_LEVEL', DEFAULT_LOG_LEVEL) + LOG_FORMAT = os.getenv('LOG_FORMAT', DEFAULT_LOG_FORMAT) + + @classmethod + def check(cls): + if cls.DSW_DB_CONN_STR is None: + raise ValueError('DSW_DB_CONN_STR env variable is missing!') + if cls.DSW_S3_URL is None: + raise ValueError('DSW_S3_URL env variable is missing!') + if cls.DSW_S3_USERNAME is None: + raise ValueError('DSW_S3_USERNAME env variable is missing!') + if cls.DSW_S3_PASSWORD is None: + raise ValueError('DSW_S3_PASSWORD env variable is missing!') + if cls.DSW_S3_BUCKET is None: + raise ValueError('DSW_S3_BUCKET env variable is missing!') + + @classmethod + def apply_logging(cls): + logging.basicConfig( + level=cls.LOG_LEVEL, + format=cls.LOG_FORMAT, + ) + LOG.debug('Logging configured...') diff --git a/src/dsw_seed_maker/consts.py b/src/dsw_seed_maker/consts.py new file mode 100644 index 0000000..b2f2326 --- /dev/null +++ b/src/dsw_seed_maker/consts.py @@ -0,0 +1,20 @@ +PACKAGE_NAME = 'dsw_seed_maker' +NICE_NAME = 'DSW Seed Maker' +PACKAGE_VERSION = '0.1.0' + +_DEFAULT_BUILT_AT = 'BUILT_AT' +BUILT_AT = '--BUILT_AT--' +_DEFAULT_VERSION = 'VERSION' +VERSION = '--VERSION--' + +DEFAULT_ENCODING = 'utf-8' +DEFAULT_LOG_LEVEL = 'INFO' +DEFAULT_LOG_FORMAT = '%(asctime)s | %(levelname)s | %(module)s: %(message)s' + + +BUILD_INFO = { + 'name': NICE_NAME, + 'packageVersion': PACKAGE_VERSION, + 'version': VERSION if VERSION != f'--{_DEFAULT_VERSION}--' else 'unknown', + 'builtAt': BUILT_AT if BUILT_AT != f'--{_DEFAULT_BUILT_AT}--' else 'unknown', +} diff --git a/src/dsw_seed_maker/logic.py b/src/dsw_seed_maker/logic.py new file mode 100644 index 0000000..b86e8d7 --- /dev/null +++ b/src/dsw_seed_maker/logic.py @@ -0,0 +1,7 @@ +from .models import ExampleRequestDTO, ExampleResponseDTO + + +def example_logic(req_dto: ExampleRequestDTO) -> ExampleResponseDTO: + return ExampleResponseDTO( + message=req_dto.message.replace('server', 'client'), + ) diff --git a/src/dsw_seed_maker/models.py b/src/dsw_seed_maker/models.py new file mode 100644 index 0000000..126fb50 --- /dev/null +++ b/src/dsw_seed_maker/models.py @@ -0,0 +1,10 @@ +import pydantic + + +class ExampleRequestDTO(pydantic.BaseModel): + magic_code: str = pydantic.Field(alias='magicCode') + message: str = pydantic.Field(alias='message') + + +class ExampleResponseDTO(pydantic.BaseModel): + message: str = pydantic.Field(alias='message') diff --git a/src/dsw_seed_maker/static/css/style.css b/src/dsw_seed_maker/static/css/style.css new file mode 100644 index 0000000..fdcf458 --- /dev/null +++ b/src/dsw_seed_maker/static/css/style.css @@ -0,0 +1,42 @@ +h1 { + font-family: monospace; + text-align: center; + animation: colorChange 1s infinite; +} + +@keyframes colorChange { + 0% { color: red; } + 50% { color: blue; } + 100% { color: red; } +} + +.container { + padding: 1em; + margin: 1em auto; + border: 1px solid #ccc; + border-radius: 5px; + box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); + + background: linear-gradient(318deg, #f9c69c, #a5bfff); + background-size: 400% 400%; + + -webkit-animation: bgAnimation 7s ease infinite; + -moz-animation: bgAnimation 7s ease infinite; + animation: bgAnimation 7s ease infinite; +} + +@-webkit-keyframes bgAnimation { + 0%{background-position:0% 83%} + 50%{background-position:100% 18%} + 100%{background-position:0% 83%} +} +@-moz-keyframes bgAnimation { + 0%{background-position:0% 83%} + 50%{background-position:100% 18%} + 100%{background-position:0% 83%} +} +@keyframes bgAnimation { + 0%{background-position:0% 83%} + 50%{background-position:100% 18%} + 100%{background-position:0% 83%} +} diff --git a/src/dsw_seed_maker/static/js/script.js b/src/dsw_seed_maker/static/js/script.js new file mode 100644 index 0000000..7aa9aed --- /dev/null +++ b/src/dsw_seed_maker/static/js/script.js @@ -0,0 +1,40 @@ +console.log('script.js loaded') + +jQuery(document).ready(function($) { + const $btn_example = $('#btn-example') + const $input_example = $('#input-example') + + $btn_example.click(function() { + alert('Button clicked, sending AJAX request') + $btn_example.prop('disabled', true) // no clicking when processing + + const magicCode = $input_example.val() + console.log(`Sending magic code: ${magicCode}`) + + $.ajax({ + url: `${ROOT_PATH}/api/example`, + type: 'POST', + contentType: 'application/json', + data: JSON.stringify({ + magicCode: magicCode, + message: 'Hello, server!', + }), + success: function(response) { + console.log(response) + alert(`Got response from server: ${response.message}`) + // Here you can render something in the webpage (e.g. form elements) + + $btn_example.prop('disabled', false) // re-enable the button + }, + error: function(xhr, status, error) { + console.error(xhr, status, error) + alert(`Error: ${error}`) + // Here you can render an error message in the webpage + + $btn_example.prop('disabled', false) // re-enable the button + } + }) + }) +}) + +// classes and other functions to do something with the webpage and data diff --git a/src/dsw_seed_maker/templates/index.html.j2 b/src/dsw_seed_maker/templates/index.html.j2 new file mode 100644 index 0000000..77ea55f --- /dev/null +++ b/src/dsw_seed_maker/templates/index.html.j2 @@ -0,0 +1,25 @@ +{% extends 'layout.html.j2' %} + +{% block title %}DSW Seed Maker{% endblock %} + +{% block content %} +

DSW Seed Maker

+ +

There should be some shiny simple frontend created with Jinja, Bootstrap, and jQuery (+AJAX)...

+ +
+ +
+ + + This should be example but try other as well. +
+ +
+ + +{% endblock %} + +{% block scripts %} + {{ super() }} +{% endblock %} diff --git a/src/dsw_seed_maker/templates/layout.html.j2 b/src/dsw_seed_maker/templates/layout.html.j2 new file mode 100644 index 0000000..7f3f05e --- /dev/null +++ b/src/dsw_seed_maker/templates/layout.html.j2 @@ -0,0 +1,30 @@ + + + + + + + + {% block head %} + {% block title %}{% endblock %} + + + + + {% endblock %} + + +
+ {% block content %} + {% endblock %} +
+ + {% block scripts %} + + + + {% endblock %} + +