Update import script (#45)

* Update importer script version * Refactor sample.env update envs * Update to work with changes in importer * Update to work with changes in importer * Update location type coding system url * Escape file path * Change default python interpreter path * Add logging to import endpoint * Increase concurent jobs to 5 * Update importer * Refactor a few logging rules * Remove notes * Update yarn version to v4.4 * Set version of node that should be used * Make bulk upload queue configurable * Update test * return on error if product or inventory list ids not provided * Update express-session * Add tests * Fix tests for the job module * Code cleanup and test regression fixes * Fix lint issues
onaio · Dec 20, 2024 · e5ec402 · e5ec402
1 parent 02e9349
commit e5ec402
Show file tree

Hide file tree

Showing 51 changed files with 5,461 additions and 3,966 deletions.
diff --git a/.env.sample b/.env.sample
@@ -1,12 +1,26 @@
 NODE_ENV=development
 
+# authentication 
 EXPRESS_OPENSRP_ACCESS_TOKEN_URL=https://reveal-stage.smartregister.org/opensrp/oauth/token
 EXPRESS_OPENSRP_AUTHORIZATION_URL=https://reveal-stage.smartregister.org/opensrp/oauth/authorize
+EXPRESS_KEYCLOAK_LOGOUT_URL=https://keycloak-stage.smartregister.org/auth/realms/reveal-stage/protocol/openid-connect/logout
 EXPRESS_OPENSRP_CALLBACK_URL=http://localhost:3000/oauth/callback/OpenSRP/
 EXPRESS_OPENSRP_OAUTH_STATE=opensrp
 EXPRESS_OPENSRP_CLIENT_ID=hunter2
 EXPRESS_OPENSRP_CLIENT_SECRET=hunter2
-EXPRESS_OPENSRP_SCOPES ="openid,profile"
+EXPRESS_OPENSRP_SCOPES=openid,profile
+EXPRESS_SERVER_LOGOUT_URL=http://localhost:3000/logout
+# optional -> kills opensrp web server session, for instance not needed when auth server is keycloak
+EXPRESS_OPENSRP_LOGOUT_URL=https://reveal-stage.smartregister.org/opensrp/logout.do
+EXPRESS_SESSION_LOGIN_URL=/login
+EXPRESS_FRONTEND_OPENSRP_CALLBACK_URL=http://localhost:3000/fe/oauth/callback/opensrp
+EXPRESS_FRONTEND_LOGIN_URL=/fe/login
+EXPRESS_ALLOW_TOKEN_RENEWAL=true
+# time in seconds 3*60*60 = 10800
+EXPRESS_MAXIMUM_SESSION_LIFE_TIME=10800
+
+
+EXPRESS_OPENSRP_SERVER_URL=http://localhost:8081/fhir
 
 EXPRESS_PORT=3000
 EXPRESS_SESSION_NAME=reveal-session
@@ -16,19 +30,6 @@ EXPRESS_REACT_BUILD_PATH=/home/mosh/ona/reveal-frontend/build
 EXPRESS_SESSION_FILESTORE_PATH=/tmp/express-sessions
 EXPRESS_PRELOADED_STATE_FILE=/tmp/revealState.json
 
-EXPRESS_SESSION_LOGIN_URL=/login
-EXPRESS_FRONTEND_OPENSRP_CALLBACK_URL=http://localhost:3000/fe/oauth/callback/opensrp
-EXPRESS_FRONTEND_LOGIN_URL=/fe/login
-
-EXPRESS_ALLOW_TOKEN_RENEWAL=true
-# time in seconds 3*60*60 = 10800
-EXPRESS_MAXIMUM_SESSION_LIFE_TIME=10800
-
-EXPRESS_SERVER_LOGOUT_URL=http://localhost:3000/logout
-# optional -> kills opensrp web server session, for instance not needed when auth server is keycloak
-EXPRESS_OPENSRP_LOGOUT_URL=https://reveal-stage.smartregister.org/opensrp/logout.do
-EXPRESS_KEYCLOAK_LOGOUT_URL=https://keycloak-stage.smartregister.org/auth/realms/reveal-stage/protocol/openid-connect/logout
-
 EXPRESS_MAXIMUM_LOGS_FILE_SIZE=5242880 # 5MB
 EXPRESS_MAXIMUM_LOG_FILES_NUMBER=5
 EXPRESS_LOGS_FILE_PATH=/home/.express/reveal-express-server.log
@@ -45,4 +46,5 @@ EXPRESS_REDIS_SENTINEL_CONFIG='{"name":"master","sentinelUsername":"u_name","sen
 
 # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/report-to. `Map<string, stringifiedJson>`.
 EXPRESS_RESPONSE_HEADERS='{"Report-To":"{ \"group\": \"csp-endpoint\", \"max_age\": 10886400, \"endpoints\": [{ \"url\": \"https://example.com/endpoint\" }] }", "Access-Control-Allow-Headers": "GET"}'
-
+EXPRESS_PYTHON_INTERPRETER_PATH=/usr/bin/python
+EXPRESS_BULK_UPLOAD_REDIS_QUEUE="fhir-import"
diff --git a/.gitignore b/.gitignore
@@ -31,3 +31,12 @@ yarn-error.log*
 # logs
 logs
 *.log
+
+# yarn
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
diff --git a/.nvmrc b/.nvmrc
@@ -0,0 +1 @@
+v20.16.0
diff --git a/build/importer/.gitignore b/build/importer/.gitignore
@@ -1,41 +1,11 @@
 config.py
+importer.log
 
-sampleData
-.idea
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
 
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
@@ -55,76 +25,11 @@ coverage.xml
 .pytest_cache/
 cover/
 
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
-.pdm.toml
-.pdm-python
-.pdm-build/
+.python-version
 
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
 
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
 # Environments
 .env
 .venv
@@ -134,33 +39,9 @@ ENV/
 env.bak/
 venv.bak/
 
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/