Alembic

README.md

@@ -118,61 +118,51 @@ The version that `asdf` will use for the current project is declared in `.tool-v
 2. Follow the instructions in the files to fill in the values
 
 
-### Run Nur via shell
+### Run Top-Assist
 
 ````
-git clone https://github.com/MDGrey33/Nur.git
-cd Nur
+git clone https://github.com/toptal/top-assist
+cd top-assist
 poetry install
-poetry run python main.py
+PYTHONPATH=. poetry run python main.py
 ````
 
-`poetry run python ./main.py` (for the menu operations)
+`PYTHONPATH=. poetry run python main.py` (for the menu operations)
 
-`poetry run python ./api/endpoint.py` (API / uvicorn web server)
+`PYTHONPATH=. poetry run python api/endpoint.py` (API / uvicorn web server)
 
 Refer to the ./documentation/slack_app_manifest_installation_guide.md for slack bot setup
 
-`poetry run python ./slack/bot.py` (slack bot stream listener)
+`PYTHONPATH=. poetry run python slack/bot.py` (slack bot stream listener)
 
-### Run Nur via Docker
+### Run DBs
 
-We will use docker compose to build 3 containers of Nur. Each one is used to run a separate part of the app:
-* `nur-manager` will create a container allowing us to use `main.py` functionality. It is meant to be interactive (read further for details).
-* `nur-web` starts the web application.
-* `nur-slack` starts the slack integration script.
+Top Assist uses two databases: `postgresql` and `chroma`.  
+Please be sure that you set all the required environment variables in the `.env` file in `# Database` section.
+To run the databases, you can use docker-compose
 
-First off, run the composer script:
 ```
-git clone https://github.com/MDGrey33/Nur.git
-cd Nur
 docker composer up
+poetry run alembic upgrade head     # to setup PostgreSQL database
 ```
 
-Upon successful completion, you have three containers running, all of them sharing a common volume mounted at `/content` within the containers.
 
-`nur-web` and `nur-slack` should be fully operational already. If you wish to run commands in Nur's cli interface, you will need to attach a shell to the `nur-manager` container in interactive mode, and enter the management environment:
+### DB Migrations
 
+1) Import the model in `alembic/env.py` and run:
 ```
-./bin/manage
+poetry run alembic revision --autogenerate -m "Add my new model"
 ```
+Alembic will detect the changes in the models/tables and create a new migration file.  
+Be aware that it can remove empty indexes/fields.  
+If you need to have control over the migration you can remove the `--autogenerate` flag and write the migration manually.  
 
-If the above works, you should see Nur's manager command prompt - the same thing you should see when running `./main.py` locally.
 
-Bear in mind that the Dockerized version uses a shared volume named `nur_shared_content`. This is bootstrapped during the first installation, but will persist until it is manually removed.
-
-## Add a new data model using Alembic
-
-
-```warp-runnable-command
-alembic revision -m "create account table"
-# Import the model in /nurai/migrations/env.py
-# Create a new migration
-poetry run alembic revision --autogenerate -m "Initialize the db"
-# Apply the migration
-poetry run alembic upgrade head
-```
+2) Apply the migration:  
+`poetry run alembic upgrade head`
 
+3) To rollback last N migrations run:  
+`poetry run alembic downgrade -N`
 
 
 ## Network traffic

alembic.ini

@@ -60,8 +60,8 @@ version_path_separator = os  # Use os.pathsep. Default configuration used for ne
 # are written from script.py.mako
 # output_encoding = utf-8
 
-# Custom: Add DB URL
-sqlalchemy.url = configuration.DB_URL
+# Custom: Replaced in alembic/env.py
+sqlalchemy.url = driver://user:pass@localhost/dbname
 
 
 [post_write_hooks]

alembic/README.md

@@ -0,0 +1,8 @@
+### Alembic documentation
+https://alembic.sqlalchemy.org/en/latest/
+
+### Cookbook
+https://alembic.sqlalchemy.org/en/latest/cookbook.html#rudimental-schema-level-multi-tenancy-for-postgresql-databases
+
+Notes:
+- we set `DB_URL` in `.env` file and rewrite it in `alembic/env.py` file (`alembic.ini` file is not used of security reasons)

alembic/env.py

@@ -2,21 +2,33 @@
 
 from sqlalchemy import engine_from_config
 from sqlalchemy import pool
+from configuration import DB_URL
+from database.database import Base
+
+# Import all models to ensure they are registered with the metadata
+from models.page_data import PageData
+from models.bookmarked_conversation import BookmarkedConversation
+from models.qa_interaction import QAInteraction
+from models.quiz_question import QuizQuestion
+from models.space_info import SpaceInfo
+from models.user_score import UserScore
 
 from alembic import context
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
 
+# Custom: Change DB URL to the one in the configuration
+config.set_main_option("sqlalchemy.url", DB_URL)
+
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)
 
 # add your model's MetaData object here
 # for 'autogenerate' support
-from database.database import Base
 target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,

alembic/versions/2024_04_12_1354-713caec2649e_initial_migration.py

@@ -0,0 +1,30 @@
+"""Initial migration
+
+Revision ID: 713caec2649e
+Revises: 
+Create Date: 2024-04-12 13:54:12.213839
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '713caec2649e'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

alembic/versions/2024_04_12_1357-4ae5c037da0f_add_page_data_table.py

@@ -0,0 +1,43 @@
+"""Add page_data table
+
+Revision ID: 4ae5c037da0f
+Revises: 713caec2649e
+Create Date: 2024-04-12 13:57:09.418241
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '4ae5c037da0f'
+down_revision: Union[str, None] = '713caec2649e'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('page_data',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('page_id', sa.String(), nullable=True),
+    sa.Column('space_key', sa.String(), nullable=True),
+    sa.Column('title', sa.String(), nullable=True),
+    sa.Column('author', sa.String(), nullable=True),
+    sa.Column('createdDate', sa.DateTime(), nullable=True),
+    sa.Column('lastUpdated', sa.DateTime(), nullable=True),
+    sa.Column('content', sa.Text(), nullable=True),
+    sa.Column('comments', sa.Text(), nullable=True),
+    sa.Column('last_embedded', sa.DateTime(), nullable=True),
+    sa.Column('embed', sa.Text(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('page_data')
+    # ### end Alembic commands ###

alembic/versions/2024_04_12_1358-c3363bdf61fa_add_bookmarked_conversation_table.py

@@ -0,0 +1,38 @@
+"""Add bookmarked_conversation table
+
+Revision ID: c3363bdf61fa
+Revises: 4ae5c037da0f
+Create Date: 2024-04-12 13:58:12.798686
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'c3363bdf61fa'
+down_revision: Union[str, None] = '4ae5c037da0f'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('bookmarked_conversations',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('title', sa.Text(), nullable=True),
+    sa.Column('body', sa.Text(), nullable=True),
+    sa.Column('thread_id', sa.String(), nullable=True),
+    sa.Column('bookmarked_on_slack', sa.DateTime(), nullable=True),
+    sa.Column('posted_on_confluence', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('bookmarked_conversations')
+    # ### end Alembic commands ###

alembic/versions/2024_04_12_1358-ce3fb8b3d191_add_qa_interaction_table.py

@@ -0,0 +1,44 @@
+"""Add qa_interaction table
+
+Revision ID: ce3fb8b3d191
+Revises: c3363bdf61fa
+Create Date: 2024-04-12 13:58:46.916722
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'ce3fb8b3d191'
+down_revision: Union[str, None] = 'c3363bdf61fa'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('qa_interactions',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('question_text', sa.Text(), nullable=True),
+    sa.Column('thread_id', sa.String(), nullable=True),
+    sa.Column('assistant_thread_id', sa.String(), nullable=True),
+    sa.Column('answer_text', sa.Text(), nullable=True),
+    sa.Column('channel_id', sa.String(), nullable=True),
+    sa.Column('slack_user_id', sa.String(), nullable=True),
+    sa.Column('question_timestamp', sa.DateTime(), nullable=True),
+    sa.Column('answer_timestamp', sa.DateTime(), nullable=True),
+    sa.Column('comments', sa.Text(), nullable=True),
+    sa.Column('last_embedded', sa.DateTime(), nullable=True),
+    sa.Column('embed', sa.Text(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('qa_interactions')
+    # ### end Alembic commands ###

alembic/versions/2024_04_12_1359-0278f7c21985_add_quiz_question_table.py

@@ -0,0 +1,38 @@
+"""Add quiz_question table
+
+Revision ID: 0278f7c21985
+Revises: ce3fb8b3d191
+Create Date: 2024-04-12 13:59:10.846548
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '0278f7c21985'
+down_revision: Union[str, None] = 'ce3fb8b3d191'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('quiz_questions',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('question_text', sa.Text(), nullable=True),
+    sa.Column('thread_id', sa.String(), nullable=True),
+    sa.Column('summary', sa.Text(), nullable=True),
+    sa.Column('posted_on_slack', sa.DateTime(), nullable=True),
+    sa.Column('posted_on_confluence', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('quiz_questions')
+    # ### end Alembic commands ###

alembic/versions/2024_04_12_1359-dccb3577feb5_add_space_info_table.py

@@ -0,0 +1,36 @@
+"""Add space_info table
+
+Revision ID: dccb3577feb5
+Revises: 0278f7c21985
+Create Date: 2024-04-12 13:59:35.003367
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'dccb3577feb5'
+down_revision: Union[str, None] = '0278f7c21985'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('space_info',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('space_key', sa.String(), nullable=False),
+    sa.Column('space_name', sa.String(), nullable=False),
+    sa.Column('last_import_date', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('space_info')
+    # ### end Alembic commands ###