Migrationer

README.md

@@ -11,9 +11,21 @@ A day starts at 5:00 and ends at 5:00 the next day. This means that an end date
 ### Priority
 The default priority is 0. If a PR has its priority set to 1, it will be the only PR shown until its end date (useful for pubs etc.).
 
+### Migrations
+In order to enable changes to the database schema there is the `src/migrations` folder, files in this folder named like `07-myname.py` (<number>-<name>.py) are migrations which are used to migrate to new database schemas.
+
+Files in the migrations folder should define a function `upgrade()` which updates from the previous schema to the new one. Please thoroughly test that your migration works properly locally before pushing any remote changes.
+
+Concretely, when you make changes to `src/data.py`, you must also create a new migration, with number following the last existing one, which runs database operations migrating from the old format to the new one.
+
+The database tracks its current migration version and runs all new migrations on application startup.
+
+## Running locally
+In order to aid running locally there is a shell script `dev.sh`, running `./dev.sh setup` sets up a python virtualenv with all the dependencies, running `./dev.sh run` will run the application on port `8080`, running `./dev.sh shell` will run bash with the virtual env tools in path.
+
 ## Running in Docker
 The provided sample compose file should work out of the box provided a
-`SECRET_KEY` env-variable. Aditional variables can be found in `src/config.py`.
+`SECRET_KEY` env-variable. Additional variables can be found in `src/config.py`.
 
-At first launc the database is populated with a user _admin_ with the password
+At first launch the database is populated with a user _admin_ with the password
 _pass_. It is suggested you change this immediately.

dev.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -e
+printf 'Changing Working Directory to %s\n' "${0%/*}"
+cd "${0%/*}"
+
+name="${0##*/}"
+
+setup() {
+  virtualenv venv
+  ./venv/bin/pip install -r requirements.txt
+}
+
+run() {
+  cd src
+  SECRET_KEY="${SECRET_KEY:-default}" ../venv/bin/uwsgi --enable-threads --http-socket :"${PORT:-8080}" --module tv:app
+}
+
+shell() {
+  export PATH="$PWD/venv/bin:$PATH" 
+  bash
+}
+
+usage() {
+  cat <<EOF
+USAGE: $name [CMD]
+  where CMD is one of:
+  - setup: create a python virtualenv
+  - run:   run the server locally on port 8080
+  - shell: open a shell with virtualenv in path
+  - help:  shows this
+EOF
+}
+
+case "$1" in
+  setup)
+    setup
+    ;;
+  run)
+    run
+    ;;
+  shell)
+    shell
+    ;;
+  *)
+    usage
+    ;;
+esac

src/data.py

@@ -5,9 +5,9 @@
 
 class User(UserMixin, db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    username = db.Column(db.String(64), index=True, unique=True)
-    password_hash = db.Column(db.String(128))
-    role = db.Column(db.String(32))
+    username = db.Column(db.String(), index=True, unique=True)
+    password_hash = db.Column(db.String())
+    role = db.Column(db.String())
 
     def set_password(self, password):
       self.password_hash = generate_password_hash(password)
@@ -18,14 +18,13 @@ def check_password(self, password):
 
 class PR(db.Model):
     id = db.Column(db.Integer, primary_key=True)
-    desc = db.Column(db.String(128))
-    file_name = db.Column(db.String(128))
+    desc = db.Column(db.String())
+    file_name = db.Column(db.String())
     start_date = db.Column(db.DateTime, index=True, default=datetime.utcnow)
     end_date = db.Column(db.DateTime, index=True)
-    owner = db.Column(db.String(64))
+    owner = db.Column(db.String())
     user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
     priority = db.Column(db.Integer, default=0)
-
 
 def fix_date(start_date, end_date, priority):
 

src/migrate.py

@@ -0,0 +1,52 @@
+from tv import app, db
+import os
+import re
+import importlib
+
+def ensure_migration_table():
+    with app.app_context():
+        table = db.session.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='migration'")
+        if len(table.fetchall()) != 1:
+            print("Creating migration table")
+            db.session.execute("CREATE TABLE migration ("
+                "id INTEGER PRIMARY KEY CHECK (id = 0),"
+                "version INTEGER"
+            ")")
+            db.session.execute("INSERT INTO migration (id, version) VALUES (0, 0)")
+        db.session.commit()
+
+def get_migration_version():
+    ensure_migration_table()
+    with app.app_context():
+        version = db.session.execute("SELECT version FROM migration").fetchall()[0][0];
+        db.session.commit()
+        return version
+
+def set_migration_version(version):
+    with app.app_context():
+        db.session.execute(f"UPDATE MIGRATION SET version = :version", { 'version': version });
+        db.session.commit()
+
+def do_migrations():
+    # only consider files of form N-name.py where N is a number and name is non-whitespace
+    pyfiles = filter(lambda x: re.match(r"^(\d+)-\S+\.py$", x), os.listdir("migrations"))
+    # group number with filepath
+    pyfiles = [(f, int(f.split("-")[0])) for f in pyfiles]
+    # sort by number
+    pyfiles = sorted(pyfiles, key=lambda x: x[1])
+    print("========================================")
+    print("=          Running Migrations          =")
+    print("========================================")
+    curr_version = get_migration_version()
+    for path, n in pyfiles:
+        if n <= curr_version:
+            print(f"Skipping migration {path}")
+            continue
+
+        print(f"Running migration {path}")
+        mod = importlib.import_module(f"migrations.{os.path.splitext(path)[0]}")
+        mod.upgrade()
+        set_migration_version(n)
+    print("========================================")
+    print("=        Done Running Migrations       =")
+    print("========================================")

src/migrations/01-string-length.py

@@ -0,0 +1,25 @@
+from tv import app, db
+
+def retype(table, column, newtype):
+    # sql injection is my passion :) [its ok cause the inputs are just hardcoded constants]
+    db.session.execute(f"ALTER TABLE {table} ADD COLUMN {column}_new {newtype}")
+    db.session.execute(f"UPDATE {table} SET {column}_new = {column}")
+    db.session.execute(f"ALTER TABLE {table} DROP COLUMN {column}")
+    db.session.execute(f"ALTER TABLE {table} RENAME COLUMN {column}_NEW TO {column}")
+
+
+def upgrade():
+    with app.app_context():
+        db.session.execute('DROP INDEX ix_user_username')
+
+        retype("user", "username", "VARCHAR")
+        retype("user", "password_hash", "VARCHAR")
+        retype("user", "role", "VARCHAR")
+
+        db.session.execute('CREATE UNIQUE INDEX ix_user_username ON user (username)')
+
+        retype("pr", "desc", "VARCHAR")
+        retype("pr", "file_name", "VARCHAR")
+        retype("pr", "owner", "VARCHAR")
+
+        db.session.commit()

src/tv.py

@@ -27,6 +27,9 @@
         print("Data base does not exist, creating a new one")
         create_db()
 
+import migrate
+migrate.do_migrations()
+
 from users import users_page
 app.register_blueprint(users_page)