diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..dc71603 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.github/ + +.coverage/ +.coverage +coverage + +venv/ +.env diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml new file mode 100644 index 0000000..a1be27b --- /dev/null +++ b/.github/workflows/production.yml @@ -0,0 +1,12 @@ +name: Production CI/CD Pipeline + +on: + push: + branches: + - main + +jobs: + ci: + uses: TogetherCrew/operations/.github/workflows/ci.yml@main + secrets: + CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} \ No newline at end of file diff --git a/.github/workflows/start.staging.yml b/.github/workflows/start.staging.yml new file mode 100644 index 0000000..842e3bd --- /dev/null +++ b/.github/workflows/start.staging.yml @@ -0,0 +1,9 @@ +name: Staging CI/CD Pipeline + +on: pull_request + +jobs: + ci: + uses: TogetherCrew/operations/.github/workflows/ci.yml@main + secrets: + CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f206ca2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,173 @@ +.DS_Store +.idea +*.log +tmp/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ +# +# Emacs +.org + +interactions/credentials.py +interactions/temp.ipynb +test*.ipynb +analyzer/rndao_analyzer/analysis/credentials.py \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..96b74ae --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +# It's recommended that we use `bullseye` for Python (alpine isn't suitable as it conflcts with numpy) +FROM python:3.10-bullseye AS base +WORKDIR /project +COPY . . +RUN pip3 install -r requirements.txt + +FROM base AS test +RUN python3 -m coverage run -m pytest tests +CMD ["python3", "-m", "coverage", "lcov" ,"-o", "coverage/lcov.info"] + +FROM base AS prod +CMD ["python3", "server.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..c7845d4 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# python-service \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..4cc7912 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,30 @@ +version: "3.9" + +services: + redis: + image: "redis:7.0.12-alpine" + ports: + - 6379:6379 + mongo: + image: "mongo:6.0.8" + ports: + - 27017:27017 + environment: + - MONGO_INITDB_ROOT_USERNAME=root + - MONGO_INITDB_ROOT_PASSWORD=pass + rabbitmq: + image: "rabbitmq:3-management-alpine" + ports: + - 5672:5672 + environment: + - RABBITMQ_DEFAULT_USER=root + - RABBITMQ_DEFAULT_PASS=pass + neo4j: + image: "neo4j:5.9.0" + ports: + - 7687:7687 + - 7474:7474 + environment: + - NEO4J_AUTH=neo4j/password + - NEO4J_PLUGINS=["apoc", "graph-data-science"] + - NEO4J_dbms_security_procedures_unrestricted=apoc.*,gds.* diff --git a/docker-compose.example.yml b/docker-compose.example.yml new file mode 100644 index 0000000..0eccf2e --- /dev/null +++ b/docker-compose.example.yml @@ -0,0 +1,14 @@ +version: "3.9" + +services: + server: + build: + context: . + target: prod + dockerfile: Dockerfile + worker: + build: + context: . + target: prod + dockerfile: Dockerfile + command: python3 worker.py diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..0f4af40 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,70 @@ +version: "3.9" + +services: + app: + build: + context: . + target: test + dockerfile: Dockerfile + environment: + - PORT=3000 + - MONGODB_HOST=mongo + - MONGODB_PORT=27017 + - MONGODB_USER=root + - MONGODB_PASS=pass + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_PASSWORD=password + - RABBIT_HOST=rabbitmq + - RABBIT_PORT=5672 + - RABBIT_USER=root + - RABBIT_PASSWORD=pass + - NEO4J_PROTOCOL=bolt + - NEO4J_HOST=neo4j + - NEO4J_PORT=7687 + - NEO4J_USER=neo4j + - NEO4J_PASSWORD=password + - NEO4J_DB=neo4j + - SAGA_DB_NAME=Saga + - SAGA_DB_COLLECTION=saga + - SENTRY_DSN=sample_dsn + - SENTRY_ENV=local + volumes: + - ./coverage:/project/coverage + networks: + - python_service_network + depends_on: + - redis + - mongo + - neo4j + - rabbitmq + redis: + image: "redis:7.0.12-alpine" + networks: + - python_service_network + mongo: + image: "mongo:6.0.8" + environment: + - MONGO_INITDB_ROOT_USERNAME=root + - MONGO_INITDB_ROOT_PASSWORD=pass + networks: + - python_service_network + rabbitmq: + image: "rabbitmq:3-management-alpine" + environment: + - RABBITMQ_DEFAULT_USER=root + - RABBITMQ_DEFAULT_PASS=pass + networks: + - python_service_network + neo4j: + image: "neo4j:5.9.0" + environment: + - NEO4J_AUTH=neo4j/password + - NEO4J_PLUGINS=["apoc", "graph-data-science"] + - NEO4J_dbms_security_procedures_unrestricted=apoc.*,gds.* + networks: + - python_service_network + +networks: + python_service_network: + driver: bridge diff --git a/mongodb_connection.py b/mongodb_connection.py new file mode 100644 index 0000000..b14768e --- /dev/null +++ b/mongodb_connection.py @@ -0,0 +1,15 @@ +import os + +from dotenv import load_dotenv +from pymongo import MongoClient + +load_dotenv() + +host = os.getenv("DB_HOST") +port = os.getenv("DB_PORT") +user = os.getenv("DB_USER") +password = os.getenv("DB_PASSWORD") +db_name = os.getenv("DB_NAME") + +mongodb_connection = f"mongodb://{user}:{password}@{host}:{port}/{db_name}" +client = MongoClient(host=mongodb_connection) diff --git a/neo4j_connection.py b/neo4j_connection.py new file mode 100644 index 0000000..af67016 --- /dev/null +++ b/neo4j_connection.py @@ -0,0 +1,30 @@ +import os + +from dotenv import load_dotenv +from neo4j import GraphDatabase + +load_dotenv() + +user = os.getenv("NEO4J_USER") +protocol = os.getenv("NEO4J_PROTOCOL") +password = os.getenv("NEO4J_PASSWORD") +host = os.getenv("NEO4J_HOST") +port = os.getenv("NEO4J_PORT") +db_name = os.getenv("NEO4J_DB") + +neo4j_url = f"{protocol}://{host}:{port}" +neo4j_auth = (user, password) + +driver = GraphDatabase.driver(neo4j_url, auth=neo4j_auth) +with GraphDatabase.driver(neo4j_url, auth=neo4j_auth) as driver: + driver.verify_connectivity() + + +def read(cypher, database="neo4j"): + records, summary, keys = driver.execute_query(cypher, database_=database) + return records, summary, keys + + +def write(cypher, database="neo4j"): + records, summary, keys = driver.execute_query(cypher, database_=database) + return summary diff --git a/rabbitmq_connection.py b/rabbitmq_connection.py new file mode 100644 index 0000000..5089ed1 --- /dev/null +++ b/rabbitmq_connection.py @@ -0,0 +1,16 @@ +import os + +from dotenv import load_dotenv +from tc_messageBroker import RabbitMQ + +load_dotenv() + +if __name__ == "__main__": + broker_url = os.getenv("RABBIT_HOST") + port = os.getenv("RABBIT_PORT") + username = os.getenv("RABBIT_USER") + password = os.getenv("RABBIT_PASSWORD") + + rabbit_mq = RabbitMQ( + broker_url=broker_url, port=port, username=username, password=password + ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5f41820 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,33 @@ +async-timeout==4.0.2 +attrs==22.2.0 +cffi==1.15.1 +click==8.1.6 +coverage==7.2.5 +cryptography==41.0.2 +dnspython==2.2.1 +exceptiongroup==1.1.0 +iniconfig==2.0.0 +jsonschema==4.17.3 +neo4j==5.10.0 +networkx==3.1 +numpy==1.24.1 +packaging==23.0 +pandas==2.0.3 +pika==1.3.2 +pluggy==1.0.0 +pycparser==2.21 +pymongo==4.3.3 +pyrsistent==0.19.3 +pytest==7.2.0 +pytest-cov==4.0.0 +python-dateutil==2.8.2 +python-dotenv==0.21.1 +pytz==2022.7.1 +redis==4.6.0 +rq==1.15.1 +ruff==0.0.265 +six==1.16.0 +tc-messageBroker==1.4.0 +tomli==2.0.1 +types-pyOpenSSL==23.2.0.2 +tzdata==2023.3 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_sample.py b/tests/test_sample.py new file mode 100644 index 0000000..5b4adb4 --- /dev/null +++ b/tests/test_sample.py @@ -0,0 +1,2 @@ +def test_sample(): + assert 3 != 5 diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..5920945 --- /dev/null +++ b/worker.py @@ -0,0 +1,37 @@ +import logging +import os + +import redis +from dotenv import load_dotenv +from rq import Worker + +load_dotenv() + + +def worker_exception_handler(job, exc_type, exc_value, traceback): + logging.error(" ========= RQ Exception =========") + logging.error(f"JOB: {job}") + logging.error(f"exc_type: {exc_type}") + logging.error(f"exc_value: {exc_value}") + logging.error(f"traceback: {traceback}") + + +if __name__ == "__main__": + host = os.getenv("REDIS_HOST", "") + port = int(os.getenv("REDIS_PORT", 6379)) + password = os.getenv("REDIS_PASSWORD") + + logging.basicConfig() + logging.getLogger().setLevel(logging.INFO) + + redis_connection = redis.Redis(host=host, port=port, password=password) + worker = Worker( + queues=["default"], + connection=redis_connection, + exception_handlers=worker_exception_handler, + ) + try: + worker.work(with_scheduler=True, max_jobs=1) + except KeyboardInterrupt: + worker.clean_registries() + worker.stop_scheduler()