Wannadb Web

.dockerignore

@@ -0,0 +1,5 @@
+.git
+.gitignore
+LICENSE.md
+README.md
+venv/

.github/workflows/new_stable_version_manual.yml

@@ -0,0 +1,21 @@
+name: Create New Stable Version (Manual)
+
+on:
+    workflow_dispatch:
+
+jobs:
+    create_pull_request:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v2
+
+            - name: Create Pull Request
+              uses: peter-evans/create-pull-request@v3
+              with:
+                  token: ${{ secrets.GITHUB_TOKEN }}
+                  branch: stable
+                  base: main
+                  title: Automated New Stable Version
+                  body: This pull request was automatically created by the workflow and contains the latest stable version of the repository.

.github/workflows/new_stable_version_push.yml

@@ -0,0 +1,27 @@
+name: Create New Stable Version (Push)
+
+on:
+    push:
+        branches:
+            - main
+
+jobs:
+    create_pull_request:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v2
+
+            - name: Check commit message for [stable]
+              id: check_commit_message
+              run: echo ::set-output name=contains_stable::$(if grep -q "\[stable\]" <<< "$(git log --format=%B -n 1)"; then echo true; else echo false; fi)
+
+            - name: Create Pull Request
+              if: steps.check_commit_message.outputs.contains_stable == 'true'
+              uses: peter-evans/create-pull-request@v3
+              with:
+                  token: ${{ secrets.GITHUB_TOKEN }}
+                  branch: stable
+                  base: main
+                  title: Automated New Stable Version
+                  body: This pull request was automatically created by the workflow and contains the latest stable version of the repository.

.gitignore

@@ -4,3 +4,7 @@
 /.pytest_cache/
 
 /models/
+
+postgres/__pycache__
+flask_app/__pycache__
+1_:memory:

.pylintrc

@@ -0,0 +1,116 @@
+[MASTER]
+
+init-hook='import os, sys; sys.path.append(os.path.abspath(os.path.curdir))'
+ignore=tests
+load-plugins=
+jobs=4
+unsafe-load-any-extension=no
+extension-pkg-whitelist=
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+confidence=
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+disable=locally-disabled,too-few-public-methods,too-many-ancestors,useless-object-inheritance,useless-return,unnecessary-pass
+
+
+[REPORTS]
+output-format=text
+reports=yes
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+#msg-template=
+
+[LOGGING]
+logging-modules=logging
+
+[SIMILARITIES]
+min-similarity-lines=8
+ignore-comments=yes
+ignore-imports=no
+
+[FORMAT]
+max-line-length=100
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+single-line-if-stmt=no
+
+max-module-lines=1000
+indent-string='\t'
+
+[SPELLING]
+spelling-dict=
+spelling-ignore-words=
+spelling-private-dict-file=
+spelling-store-unknown-words=no
+
+
+[VARIABLES]
+init-import=no
+dummy-variables-rgx=(_[a-zA-Z0-9_]*?$)
+additional-builtins=
+callbacks=cb_,_cb,handle_,get,post,put,patch,delete,options
+redefining-builtins-modules=six.moves,future.builtins
+
+
+[TYPECHECK]
+ignore-mixin-members=yes
+ignored-modules=flask_sqlalchemy,app.extensions.flask_sqlalchemy
+ignored-classes=optparse.Values,thread._local,_thread._local
+generated-members=fget,query,begin,add,merge,delete,commit,rollback
+contextmanager-decorators=contextlib.contextmanager
+
+
+[MISCELLANEOUS]
+notes=FIXME,XXX,TODO
+
+
+[BASIC]
+good-names=i,j,k,ex,Run,_,log,api
+bad-names=foo,bar,baz,toto,tutu,tata
+
+
+[ELIF]
+max-nested-blocks=5
+
+
+[DESIGN]
+max-args=5
+ignored-argument-names=_.*
+max-bool-expr=5
+
+
+[IMPORTS]
+deprecated-modules=optparse
+import-graph=
+ext-import-graph=
+int-import-graph=
+known-standard-library=
+known-third-party=flask_restplus_patched
+analyse-fallback-blocks=no
+
+
+[CLASSES]
+defining-attr-methods=__init__,__new__,setUp
+valid-classmethod-first-arg=cls
+valid-metaclass-classmethod-first-arg=mcs
+exclude-protected=_asdict,_fields,_replace,_source,_make
+
+
+[EXCEPTIONS]
+overgeneral-exceptions=builtins.Exception

Dockerfile

@@ -0,0 +1,65 @@
+FROM python:3.9-slim-buster as build
+
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE 1
+
+RUN apt-get update \
+  # dependencies for building Python packages
+  && apt-get install -y build-essential \
+  # psycopg2 dependencies
+  && apt-get install -y libpq-dev \
+  # Additional dependencies
+  && apt-get install -y telnet netcat \
+  # cleaning up unused files
+  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /home/wannadb
+WORKDIR /home/wannadb
+
+# update Pip
+RUN pip install --upgrade pip
+
+# install torch
+RUN pip install --use-pep517 torch==1.10.0
+
+# Install dependencies
+COPY core-requirements.txt core-requirements.txt
+RUN pip install --use-pep517 -r core-requirements.txt
+COPY backend-requirements.txt backend-requirements.txt
+RUN pip install --use-pep517 -r backend-requirements.txt
+##################################
+##      do not change above     ##
+##      changes above cause     ##
+##      long loading times      ##
+##################################
+
+# Run tests
+RUN pip install --use-pep517 pytest
+#RUN pytest
+
+FROM build as worker
+
+FROM build as worker-prod
+
+#copy the rest
+COPY . .
+
+
+FROM build as dev 
+
+#CMD [ "python", "app.py" ]
+
+CMD ["mypy","--install-types", "--non-interactive"]
+
+CMD ["flask", "--app", "app", "--debug", "run","--host","0.0.0.0", "--port", "8000" ]
+
+
+FROM worker-prod as prod
+
+
+RUN chmod +x wannadb_web/entrypoint.sh
+
+# Define the entrypoint.sh
+CMD ["sh","./wannadb_web/entrypoint.sh"]
+

README.md

@@ -114,33 +114,31 @@ series = {SIGMOD '22}
 
 WannaDB is dually licensed under both AGPLv3 for the free usage by end users or the embedding in Open Source projects, and a commercial license for the integration in industrial projects and closed-source tool chains. More details can be found in [our licence agreement](LICENSE.md).
 
-
 ## Availability of Code & Datasets
 
 We publish the source code four our system as discussed in the papers here. Additionally, we publish code to reproduce our experiments in a separate repository (coming soon).
 
 Unfortunately, we cannot publish the datasets online due to copyright issues. We will send them via email on request to everyone interested and hope they can be of benefit for other research, too.
 
-
 ## Implementation details
 
-The core of WannaDB (extraction and matching) was previously developed by us under the name [ASET (Ad-hoc Structured Exploration of Text Collections)](https://link.tuda.systems/aset). To better reflect the whole application cycle vision we present with this paper, we switchted the name to WannaDB. 
+The core of WannaDB (extraction and matching) was previously developed by us under the name [ASET (Ad-hoc Structured Exploration of Text Collections)](https://link.tuda.systems/aset). To better reflect the whole application cycle vision we present with this paper, we switchted the name to WannaDB.
 
 ### Repository structure
 
 This repository is structured as follows:
 
-* `wannadb`, `wannadb_parsql`, and `wannadb_ui` contain the implementation of ASET and the GUI.
-* `scripts` contains helpers, like a stand-alone preprocessing script.
-* `tests` contains pytest tests.
+- `wannadb`, `wannadb_parsql`, and `wannadb_ui` contain the implementation of ASET and the GUI.
+- `scripts` contains helpers, like a stand-alone preprocessing script.
+- `tests` contains pytest tests.
 
 ### Architecture: Core
 
 The core implementation of WannaDB is in the `wannadb` package and implemented as a library. The implementation allows you to construct pipelines of different data processors that work with the data model and may involve user feedback.
 
 **Data model**
 
-`data` contains WannaDB's data model. The entities are `InformationNugget`s, `Attribute`s, `Document`s, and the `DocumentBase`. 
+`data` contains WannaDB's data model. The entities are `InformationNugget`s, `Attribute`s, `Document`s, and the `DocumentBase`.
 
 A nugget is an information piece obtained from a document. An attribute is a table column that gets
 populated with information from the documents. A document is a textual document, and the document base is a collection of documents and provides facilities for `BSON` serialization, consistency checks, and data access.
@@ -175,3 +173,33 @@ The `Statistics` object allows you to easily record information during runtime.
 ### Architecture: GUI
 
 The GUI implementation can be found in the `wannadb_ui` package. `wannadb_api.py` provides an asynchronous API for the `wannadb` library using PyQt's slots and signals mechanism. `main_window.py`, `document_base.py`, and `interactive_window.py` contain different parts of the user interface, and `common.py` contains base classes for some recurring user interface elements.
+
+---
+
+# Start the Web-Backend docker build
+
+to build/start the production
+
+```
+docker compose -f "docker-compose-prod.yaml" build
+docker compose -f "docker-compose-prod.yaml" up
+```
+
+for developers use
+
+```
+docker compose build
+docker compose up
+```
+
+the flask and other services start automaticly.
+for more information click [here](https://github.com/lw86ruwo/wannadbBackend/blob/main/WEBSERVER_STRUCTURE.md)
+
+so see all the routes and the structure of the webserver click [here](https://github.com/lw86ruwo/wannadbBackend/blob/main/ROUTES.md)  
+
+you can use `code` to attach the container and then work in docker
+
+git only works when you install gh and make gh auth
+then you can work as usual
+
+a docker rebuild is only necessary if dependencies have changed