fix: migrations to make postgresql compatible.

common/djangoapps/split_modulestore_django/migrations/0001_initial.py

@@ -1,15 +1,40 @@
 # Generated by Django 2.2.20 on 2021-05-07 18:29, manually modified to make "course_id" column case sensitive
 
 from django.conf import settings
-from django.db import migrations, models
+from django.db import migrations, models, connection
 import django.db.models.deletion
 import opaque_keys.edx.django.models
 import simple_history.models
 
 
+def generate_split_module_sql(db_engine):
+    if 'mysql' in db_engine:
+        return 'ALTER TABLE split_modulestore_django_splitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL UNIQUE;'
+    elif 'postgresql' in db_engine:
+        return """
+            ALTER TABLE split_modulestore_django_splitmodulestorecourseindex
+            ALTER COLUMN course_id TYPE VARCHAR(255),
+            ALTER COLUMN course_id SET NOT NULL;
+
+            ALTER TABLE split_modulestore_django_splitmodulestorecourseindex
+            ADD CONSTRAINT course_id_unique UNIQUE (course_id);
+        """
+
+
+def generate_split_history_module_sql(db_engine):
+    if 'mysql' in db_engine:
+        return 'ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;'
+    elif 'postgresql' in db_engine:
+        return """
+            ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex
+            ALTER COLUMN course_id TYPE VARCHAR(255),
+            ALTER COLUMN course_id SET NOT NULL,
+            ALTER COLUMN course_id SET DATA TYPE VARCHAR(255) COLLATE "C";
+        """
 class Migration(migrations.Migration):
 
     initial = True
+    db_engine = connection.settings_dict['ENGINE']
 
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
@@ -65,11 +90,11 @@ class Migration(migrations.Migration):
         # Custom code: Convert columns to utf8_bin because we want to allow
         # case-sensitive comparisons for CourseKeys, which were case-sensitive in MongoDB
         migrations.RunSQL(
-            'ALTER TABLE split_modulestore_django_splitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL UNIQUE;',
+            generate_split_module_sql(db_engine),
             reverse_sql=migrations.RunSQL.noop,
         ),
         migrations.RunSQL(
-            'ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;',
+            generate_split_history_module_sql(db_engine),
             reverse_sql=migrations.RunSQL.noop,
         ),
     ]

lms/djangoapps/commerce/migrations/0001_data__add_ecommerce_service_user.py

@@ -1,22 +1,24 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import User
-from django.db import migrations, models
+from django.db import migrations, models, transaction
 
 USERNAME = settings.ECOMMERCE_SERVICE_WORKER_USERNAME
 EMAIL = USERNAME + '@fake.email'
 
 def forwards(apps, schema_editor):
     """Add the service user."""
     User = get_user_model()
-    user, created = User.objects.get_or_create(username=USERNAME, email=EMAIL)
-    if created:
-        user.set_unusable_password()
-        user.save()
+    with transaction.atomic():
+        user, created = User.objects.get_or_create(username=USERNAME, email=EMAIL)
+        if created:
+            user.set_unusable_password()
+            user.save()
 
 def backwards(apps, schema_editor):
     """Remove the service user."""
-    User.objects.get(username=USERNAME, email=EMAIL).delete()
+    with transaction.atomic():
+        User.objects.get(username=USERNAME, email=EMAIL).delete()
 
 class Migration(migrations.Migration):
 

lms/djangoapps/courseware/fields.py

@@ -1,36 +1,3 @@
 """
 Custom fields
 """
-
-
-from django.db.models.fields import AutoField
-
-
-class UnsignedBigIntAutoField(AutoField):
-    """
-    An unsigned 8-byte integer for auto-incrementing primary keys.
-    """
-    def db_type(self, connection):
-        if connection.settings_dict['ENGINE'] == 'django.db.backends.mysql':
-            return "bigint UNSIGNED AUTO_INCREMENT"
-        elif connection.settings_dict['ENGINE'] == 'django.db.backends.sqlite3':
-            # Sqlite will only auto-increment the ROWID column. Any INTEGER PRIMARY KEY column
-            # is an alias for that (https://www.sqlite.org/autoinc.html). An unsigned integer
-            # isn't an alias for ROWID, so we have to give up on the unsigned part.
-            return "integer"
-        elif connection.settings_dict['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
-            # Pg's bigserial is implicitly unsigned (doesn't allow negative numbers) and
-            # goes 1-9.2x10^18
-            return "BIGSERIAL"
-        else:
-            return None
-
-    def rel_db_type(self, connection):
-        if connection.settings_dict['ENGINE'] == 'django.db.backends.mysql':
-            return "bigint UNSIGNED"
-        elif connection.settings_dict['ENGINE'] == 'django.db.backends.sqlite3':
-            return "integer"
-        elif connection.settings_dict['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
-            return "BIGSERIAL"
-        else:
-            return None

lms/djangoapps/courseware/migrations/0001_initial.py

@@ -53,7 +53,7 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='StudentModule',
             fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('id', models.BigAutoField(primary_key=True, serialize=False, verbose_name='ID')),
                 ('module_type', models.CharField(default='problem', max_length=32, db_index=True, choices=[('problem', 'problem'), ('video', 'video'), ('html', 'html'), ('course', 'course'), ('chapter', 'Section'), ('sequential', 'Subsection'), ('library_content', 'Library Content')])),
                 ('module_state_key', UsageKeyField(max_length=255, db_column='module_id', db_index=True)),
                 ('course_id', CourseKeyField(max_length=255, db_index=True)),

lms/djangoapps/courseware/migrations/0008_move_idde_to_edx_when.py

@@ -33,5 +33,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(move_overrides_to_edx_when)
+        migrations.RunPython(move_overrides_to_edx_when, reverse_code=migrations.RunPython.noop)
     ]

lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py

@@ -1,45 +1,56 @@
 # Generated by Django 1.11.23 on 2019-08-28 15:50
 
-
 import lms.djangoapps.courseware.fields
 
 from django.conf import settings
-from django.db import migrations
+from django.db import migrations, models
 from django.db.migrations import AlterField
 
 class CsmBigInt(AlterField):
     '''
     Subclass AlterField migration class to split SQL between two different databases
-    We can't use the normal AlterField migration operation because Django generate and routes migrations at the model
+    We can't use the normal AlterField migration operation because Django generates and routes migrations at the model
     level and the coursewarehistoryextended_studentmodulehistoryextended table is in a different database
     '''
     def database_forwards(self, app_label, schema_editor, from_state, to_state):
         if hasattr(schema_editor.connection, 'is_in_memory_db') and schema_editor.connection.is_in_memory_db():
             # sqlite3 doesn't support 'MODIFY', so skipping during tests
             return
+
         to_model = to_state.apps.get_model(app_label, self.model_name)
+
         if schema_editor.connection.alias == 'student_module_history':
             if settings.FEATURES["ENABLE_CSMH_EXTENDED"]:
-              schema_editor.execute("ALTER TABLE `coursewarehistoryextended_studentmodulehistoryextended` MODIFY `student_module_id` bigint UNSIGNED NOT NULL;")
+                if schema_editor.connection.vendor == 'mysql':
+                    schema_editor.execute("ALTER TABLE `coursewarehistoryextended_studentmodulehistoryextended` MODIFY `student_module_id` bigint UNSIGNED NOT NULL;")
+                elif schema_editor.connection.vendor == 'postgresql':
+                    schema_editor.execute("ALTER TABLE coursewarehistoryextended_studentmodulehistoryextended ALTER COLUMN student_module_id TYPE bigint;")
         elif self.allow_migrate_model(schema_editor.connection.alias, to_model):
-            schema_editor.execute("ALTER TABLE `courseware_studentmodule` MODIFY `id` bigint UNSIGNED AUTO_INCREMENT NOT NULL;")
+            if schema_editor.connection.vendor == 'postgresql':
+                # For PostgreSQL
+                schema_editor.execute("ALTER TABLE courseware_studentmodule ALTER COLUMN id SET DATA TYPE bigint;")
+                schema_editor.execute("ALTER TABLE courseware_studentmodule ALTER COLUMN id SET NOT NULL;")
+            else:
+                # For MySQL
+                schema_editor.execute("ALTER TABLE `courseware_studentmodule` MODIFY `id` bigint UNSIGNED AUTO_INCREMENT NOT NULL;")
 
     def database_backwards(self, app_label, schema_editor, from_state, to_state):
-        # Make backwards migration a no-op, app will still work if column is wider than expected
+        # Make backwards migration a no-op; app will still work if column is wider than expected
         pass
 
 class Migration(migrations.Migration):
 
     dependencies = [
         ('courseware', '0010_auto_20190709_1559'),
     ]
+
     if settings.FEATURES["ENABLE_CSMH_EXTENDED"]:
         dependencies.append(('coursewarehistoryextended', '0002_force_studentmodule_index'))
 
     operations = [
         CsmBigInt(
             model_name='studentmodule',
             name='id',
-            field=lms.djangoapps.courseware.fields.UnsignedBigIntAutoField(primary_key=True, serialize=False),
+            field=models.BigAutoField(primary_key=True, serialize=False, verbose_name='ID'),
         )
     ]

lms/djangoapps/courseware/models.py

@@ -28,7 +28,6 @@
 from edx_django_utils.cache.utils import RequestCache
 from model_utils.models import TimeStampedModel
 from opaque_keys.edx.django.models import BlockTypeKeyField, CourseKeyField, LearningContextKeyField, UsageKeyField
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
 
 from openedx.core.djangolib.markup import HTML
 
@@ -86,7 +85,7 @@ class StudentModule(models.Model):
     """
     objects = ChunkingManager()
 
-    id = UnsignedBigIntAutoField(primary_key=True)  # pylint: disable=invalid-name
+    id = models.BigAutoField(verbose_name='ID', primary_key=True)  # pylint: disable=invalid-name
 
     ## The XBlock/XModule type (e.g. "problem")
     module_type = models.CharField(max_length=32, db_index=True)

lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py

@@ -2,7 +2,6 @@
 
 from django.db import migrations, models
 import django.db.models.deletion
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
 from django.conf import settings
 
 def bump_pk_start(apps, schema_editor):
@@ -45,7 +44,7 @@ class Migration(migrations.Migration):
                 ('state', models.TextField(null=True, blank=True)),
                 ('grade', models.FloatField(null=True, blank=True)),
                 ('max_grade', models.FloatField(null=True, blank=True)),
-                ('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)),
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
                 ('student_module', models.ForeignKey(to='courseware.StudentModule', on_delete=django.db.models.deletion.DO_NOTHING, db_constraint=False)),
             ],
             options={

lms/djangoapps/coursewarehistoryextended/models.py

@@ -19,7 +19,6 @@
 from django.dispatch import receiver
 
 from lms.djangoapps.courseware.models import BaseStudentModuleHistory, StudentModule
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
 
 
 class StudentModuleHistoryExtended(BaseStudentModuleHistory):
@@ -35,7 +34,7 @@ class Meta:
         get_latest_by = "created"
         index_together = ['student_module']
 
-    id = UnsignedBigIntAutoField(primary_key=True)  # pylint: disable=invalid-name
+    id = models.BigAutoField(primary_key=True, serialize=False)  # pylint: disable=invalid-name
 
     student_module = models.ForeignKey(StudentModule, db_index=True, db_constraint=False, on_delete=models.DO_NOTHING)
 

lms/djangoapps/grades/migrations/0001_initial.py

@@ -3,8 +3,6 @@
 from django.db import migrations, models
 from opaque_keys.edx.django.models import CourseKeyField, UsageKeyField
 
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
-
 
 class Migration(migrations.Migration):
 
@@ -17,7 +15,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
                 ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
-                ('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)),
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
                 ('user_id', models.IntegerField()),
                 ('course_id', CourseKeyField(max_length=255)),
                 ('usage_key', UsageKeyField(max_length=255)),

lms/djangoapps/grades/migrations/0006_persistent_course_grades.py

@@ -3,8 +3,6 @@
 from django.db import migrations, models
 from opaque_keys.edx.django.models import CourseKeyField
 
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
-
 
 class Migration(migrations.Migration):
 
@@ -18,7 +16,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
                 ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
-                ('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)),
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
                 ('user_id', models.IntegerField(db_index=True)),
                 ('course_id', CourseKeyField(max_length=255)),
                 ('course_edited_timestamp', models.DateTimeField(verbose_name='Last content edit timestamp')),

lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py

@@ -1,14 +1,13 @@
 # Generated by Django 1.11.20 on 2019-06-05 13:59
 
 
-from django.conf import settings
-from django.db import migrations, models
 import django.db.models.deletion
 import simple_history.models
+from django.conf import settings
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
         ('grades', '0014_persistentsubsectiongradeoverridehistory'),
@@ -28,15 +27,24 @@ class Migration(migrations.Migration):
                 ('history_id', models.AutoField(primary_key=True, serialize=False)),
                 ('history_date', models.DateTimeField()),
                 ('history_change_reason', models.CharField(max_length=100, null=True)),
-                ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
-                ('grade', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='grades.PersistentSubsectionGrade')),
-                ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+                ('history_type',
+                 models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
+                ('history_user',
+                 models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+',
+                                   to=settings.AUTH_USER_MODEL)),
             ],
-            options={
-                'ordering': ('-history_date', '-history_id'),
-                'get_latest_by': 'history_date',
-                'verbose_name': 'historical persistent subsection grade override',
-            },
-            bases=(simple_history.models.HistoricalChanges, models.Model),
+        options = {
+        'ordering': ('-history_date', '-history_id'),
+        'get_latest_by': 'history_date',
+        'verbose_name': 'historical persistent subsection grade override',
+    },
+    bases = (simple_history.models.HistoricalChanges, models.Model),
+    ),
+        migrations.AddField(
+            model_name='historicalpersistentsubsectiongradeoverride',
+            name='grade',
+            field=models.ForeignKey(blank=True, db_constraint=False, null=True,
+                                    on_delete=django.db.models.deletion.DO_NOTHING, related_name='+',
+                                    to='grades.PersistentSubsectionGrade'),
         ),
     ]

lms/djangoapps/grades/models.py

@@ -26,7 +26,6 @@
 from opaque_keys.edx.keys import CourseKey, UsageKey
 from simple_history.models import HistoricalRecords
 
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
 from lms.djangoapps.grades import events  # lint-amnesty, pylint: disable=unused-import
 from openedx.core.lib.cache_utils import get_cache
 from lms.djangoapps.grades.signals.signals import (
@@ -321,7 +320,7 @@ class Meta:
         ]
 
     # primary key will need to be large for this table
-    id = UnsignedBigIntAutoField(primary_key=True)  # pylint: disable=invalid-name
+    id = models.BigAutoField(primary_key=True)  # pylint: disable=invalid-name
 
     user_id = models.IntegerField(blank=False)
     course_id = CourseKeyField(blank=False, max_length=255)
@@ -581,7 +580,7 @@ class Meta:
         ]
 
     # primary key will need to be large for this table
-    id = UnsignedBigIntAutoField(primary_key=True)  # pylint: disable=invalid-name
+    id = models.BigAutoField(primary_key=True)  # pylint: disable=invalid-name
     user_id = models.IntegerField(blank=False, db_index=True)
     course_id = CourseKeyField(blank=False, max_length=255)