feat: Updated devstack mysql 5.7 to 8

Makefile

@@ -8,8 +8,8 @@
 #     make dev.attach.credentials
 #     make dev.pull.registrar+studio
 #     make dev.up.lms
-#     make dev.up.without-deps.lms+forum+discovery+mysql57+elasticsearch+memcached
-#     make dev.restart-container.mysql57+lms
+#     make dev.up.without-deps.lms+forum+discovery+mysql80+elasticsearch+memcached
+#     make dev.restart-container.mysql80+lms
 
 # There are also "prefix-form" targets, which are simply an alternate way to spell
 # the 'dev.' targets.
@@ -230,17 +230,17 @@ impl-dev.provision.%: dev.check-memory ## Provision specified services.
 dev.provision.%: ## Provision specified services.
 	@scripts/send_metrics.py wrap "dev.provision.$*"
 
-dev.backup: dev.up.mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Write all data volumes to the host.
-	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql57) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql57.tar.gz /var/lib/mysql
+dev.backup: dev.up.mysql80+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Write all data volumes to the host.
+	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql80) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql80.tar.gz /var/lib/mysql
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch7) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch7.tar.gz /usr/share/elasticsearch/data
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch710) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch710.tar.gz /usr/share/elasticsearch/data
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.opensearch12) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/opensearch12.tar.gz /usr/share/opensearch/data
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.coursegraph) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/coursegraph.tar.gz /data
 
-dev.restore: dev.up.mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Restore all data volumes from the host. WILL OVERWRITE ALL EXISTING DATA!
-	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql57) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql57.tar.gz
+dev.restore: dev.up.mysql80+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Restore all data volumes from the host. WILL OVERWRITE ALL EXISTING DATA!
+	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql80) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql80.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz
 	docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch7) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch7.tar.gz
@@ -272,7 +272,7 @@ dev.migrate.%: ## Run migrations on a service.
 dev.drop-db: _expects-database.dev.drop-db
 
 dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mysql container.
-	docker-compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\""
+	docker-compose exec -T mysql80 bash -c "mysql --execute=\"DROP DATABASE $*;\""
 
 
 ########################################################################################
@@ -465,10 +465,10 @@ dev.shell.%: ## Run a shell on the specified service's container.
 	docker-compose exec $* /bin/bash
 
 dev.dbshell:
-	docker-compose exec mysql57 bash -c "mysql"
+	docker-compose exec mysql80 bash -c "mysql"
 
 dev.dbshell.%: ## Run a SQL shell on the given database.
-	docker-compose exec mysql57 bash -c "mysql $*"
+	docker-compose exec mysql80 bash -c "mysql $*"
 
 # List of Makefile targets to run static asset generation, in the form dev.static.$(service)
 # Services will only have their asset generation added here

configuration_files/ecommerce.yml

@@ -25,7 +25,7 @@ DATABASES:
         ATOMIC_REQUESTS: true
         CONN_MAX_AGE: 60
         ENGINE: django.db.backends.mysql
-        HOST: edx.devstack.mysql57
+        HOST: edx.devstack.mysql80
         NAME: ecommerce
         OPTIONS:
             connect_timeout: 10

configuration_files/xqueue.yml

@@ -5,7 +5,7 @@ DATABASES:
         ATOMIC_REQUESTS: true
         CONN_MAX_AGE: 0
         ENGINE: django.db.backends.mysql
-        HOST: edx.devstack.mysql57
+        HOST: edx.devstack.mysql80
         NAME: xqueue
         OPTIONS: {}
         PASSWORD: password

docker-compose.yml

@@ -237,22 +237,22 @@ services:
     volumes:
       - mongo_data:/data/db
 
-  mysql57:
+  mysql80:
     command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci
-    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql57"
-    hostname: mysql57.devstack.edx
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql80"
+    hostname: mysql80.devstack.edx
     environment:
       MYSQL_ROOT_PASSWORD: ""
       MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
-    image: edxops/mysql:5.7
+    image: mysql:latest
     networks:
       default:
         aliases:
-          - edx.devstack.mysql57
+          - edx.devstack.mysql80
     ports:
       - "3506:3306"
     volumes:
-      - mysql57_data:/var/lib/mysql
+      - mysql80_data:/var/lib/mysql
 
   redis:
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.redis"
@@ -309,13 +309,13 @@ services:
     depends_on:
       - lms
       - memcached
-      - mysql57
+      - mysql80
     # Allows attachment to the credentials service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
       CACHE_LOCATION: edx.devstack.memcached:11211
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       SOCIAL_AUTH_EDX_OIDC_URL_ROOT: http://edx.devstack.lms:18000/oauth2
       ENABLE_DJANGO_TOOLBAR: 1
       DJANGO_WATCHMAN_TIMEOUT: 30
@@ -334,7 +334,7 @@ services:
     depends_on:
       - elasticsearch710
       - memcached
-      - mysql57
+      - mysql80
       - opensearch12
       - redis
     # Allows attachment to the discovery service using 'docker attach <containerID>'.
@@ -343,7 +343,7 @@ services:
     environment:
       # This next DB_MIGRATION_HOST line can be removed once edx/configuration has been updated with this value for
       # a while and most people have had a chance to do a "make pull" to get the latest images.
-      DB_MIGRATION_HOST: edx.devstack.mysql57
+      DB_MIGRATION_HOST: edx.devstack.mysql80
       TEST_ELASTICSEARCH_URL: "edx.devstack.elasticsearch710"
       ENABLE_DJANGO_TOOLBAR: 1
       DJANGO_WATCHMAN_TIMEOUT: 30
@@ -366,7 +366,7 @@ services:
       - discovery
       - lms
       - memcached
-      - mysql57
+      - mysql80
     # Allows attachment to the ecommerce service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
@@ -393,7 +393,7 @@ services:
       - devpi
       - elasticsearch710
       - lms
-      - mysql57
+      - mysql80
     image: openedx/edx-notes-api-dev:${OPENEDX_RELEASE:-latest}
     networks:
       default:
@@ -403,7 +403,7 @@ services:
       - "18120:18120"
     environment:
       DB_ENGINE: "django.db.backends.mysql"
-      DB_HOST: "edx.devstack.mysql57"
+      DB_HOST: "edx.devstack.mysql80"
       DB_NAME: "notes"
       DB_PASSWORD: "password"
       DB_PORT: "3306"
@@ -443,11 +443,16 @@ services:
       - forum
       - memcached
       - mongo
-      - mysql57
+      - mysql80
     # Allows attachment to the LMS service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
+      DB_HOST: edx.devstack.mysql80
+      DB_NAME: edxapp
+      DB_PORT: 3306
+      DB_USER: edxapp001
+      DB_PASSWORD: password
       BOK_CHOY_HOSTNAME: edx.devstack.lms
       BOK_CHOY_LMS_PORT: 18003
       BOK_CHOY_CMS_PORT: 18031
@@ -479,14 +484,14 @@ services:
     hostname: insights.devstack.edx
     depends_on:
       - analyticsapi
-      - mysql57
+      - mysql80
       - lms
       - memcached
     # Allows attachment to the insights service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_NAME: dashboard
       DB_PORT: 3306
       DB_USER: analytics001
@@ -511,13 +516,13 @@ services:
     container_name: edx.devstack.analyticsapi
     hostname: analyticsapi
     depends_on:
-      - mysql57
+      - mysql80
       - elasticsearch710
     command: bash -c 'source /edx/app/analytics_api/analytics_api_env && while true;  do python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:19001 --settings analyticsdataserver.settings.devstack; sleep 2; done'
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_PORT: 3306
       DB_USER: analytics001
       DB_PASSWORD: password
@@ -536,15 +541,15 @@ services:
     depends_on:
       - discovery
       - lms
-      - mysql57
+      - mysql80
       - memcached
       - redis
       - registrar-worker
     # Allows attachment to the registrar service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_NAME: registrar
       DB_PORT: 3306
       DB_USER: registrar001
@@ -578,12 +583,12 @@ services:
     hostname: registrar-worker.devstack.edx
     depends_on:
       - lms
-      - mysql57
+      - mysql80
       - redis
     stdin_open: true
     tty: true
     environment:
-      DB_HOST: edx.devstack.mysql57
+      DB_HOST: edx.devstack.mysql80
       DB_NAME: registrar
       DB_PORT: 3306
       DB_USER: registrar001
@@ -616,11 +621,16 @@ services:
       - lms
       - memcached
       - mongo
-      - mysql57
+      - mysql80
     # Allows attachment to the Studio service using 'docker attach <containerID>'.
     stdin_open: true
     tty: true
     environment:
+      DB_HOST: edx.devstack.mysql80
+      DB_NAME: edxapp
+      DB_PORT: 3306
+      DB_USER: edxapp001
+      DB_PASSWORD: password
       BOK_CHOY_HOSTNAME: edx.devstack.studio
       BOK_CHOY_LMS_PORT: 18103
       BOK_CHOY_CMS_PORT: 18131
@@ -655,7 +665,7 @@ services:
       - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue
       - ${PWD}/configuration_files/xqueue.yml:/edx/etc/xqueue.yml
     depends_on:
-      - mysql57
+      - mysql80
     environment:
       XQUEUE_CFG: "/edx/etc/xqueue.yml"
     networks:
@@ -674,7 +684,7 @@ services:
       - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue
       - ${PWD}/configuration_files/xqueue.yml:/edx/etc/xqueue.yml
     depends_on:
-      - mysql57
+      - mysql80
     networks:
       default:
         aliases:
@@ -866,4 +876,4 @@ volumes:
   elasticsearch710_data:
   mongo_data:
   opensearch12_data:
-  mysql57_data:
+  mysql80_data:

docs/developing_on_named_release_branches.rst

@@ -44,7 +44,7 @@ Switch between your Devstack releases by doing the following:
 Examples of Docker Service Names After Setting the ``COMPOSE_PROJECT_NAME`` variable. Notice that the **devstack-juniper.master** name represents the ``COMPOSE_PROJECT_NAME``.
 
 -  edx.devstack-juniper.master.lms
--  edx.devstack-juniper.master.mysql57
+-  edx.devstack-juniper.master.mysql80
 
 Each instance has an isolated set of databases. This could, for example, be used to quickly switch between versions of Open edX without hitting as many issues with migrations, data integrity, etc.
 

docs/devstack_faq.rst

@@ -134,7 +134,7 @@ How do I connect to the databases from an outside editor?
 ---------------------------------------------------------
 
 To connect to the databases from an outside editor (such as MySQLWorkbench),
-first uncomment these lines from ``docker-compose.yml``'s ``mysql57`` section
+first uncomment these lines from ``docker-compose.yml``'s ``mysql80`` section
 
 .. code:: yaml
 
@@ -145,8 +145,8 @@ Then bring your mysql container down and back up by running:
 
 .. code:: sh
 
-  docker-compose stop mysql57
-  docker-compose up -d mysql57
+  docker-compose stop mysql80
+  docker-compose up -d mysql80
 
 Then connect using the values below. Note that the username and password will
 vary depending on the database. For all of the options, see ``provision.sql``.
@@ -158,7 +158,7 @@ vary depending on the database. For all of the options, see ``provision.sql``.
 
 If you have trouble connecting, ensure the port was mapped successfully by
 running ``make dev.ps`` and looking for a line like this:
-``edx.devstack.mysql57 docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``.
+``edx.devstack.mysql80 docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``.
 
 How do I build the service images myself?
 -----------------------------------------
@@ -209,7 +209,7 @@ To access the MySQL shell for a particular database, run:
 
 .. code:: sh
 
-   make dev.shell.mysql57
+   make dev.shell.mysql80
    mysql
    use <database>;
 

drop-sql-provision.sql

@@ -0,0 +1,21 @@
+drop user 'credentials001'@'%';
+drop user 'discov001'@'%';
+drop user 'ecomm001'@'%';
+drop user 'notes001'@'%';
+drop user 'registrar001'@'%';
+drop user 'xqueue001'@'%';
+drop user 'analytics001'@'%';
+drop user 'edxapp001'@'%';
+
+#drop database `analytics-api`;
+#drop database `credentials`;
+#drop database `dashboard`;
+#drop database `discovery`;
+#drop database `ecommerce`;
+#drop database `edxapp`;
+#drop database `edxapp_csmh`;
+#drop database `notes`;
+#drop database `reports`;
+#drop database `reports_v1`;
+#drop database `registrar`;
+#drop database `xqueue`;

load-db.sh

@@ -16,6 +16,6 @@ then
 fi
 
 echo "Loading the $1 database..."
-mysql_container=$(make --silent --no-print-directory dev.print-container.mysql57)
+mysql_container=$(make --silent --no-print-directory dev.print-container.mysql80)
 docker exec -i "$mysql_container" mysql -uroot $1 < $1.sql
 echo "Finished loading the $1 database!"

options.mk

@@ -90,4 +90,4 @@ credentials+discovery+ecommerce+insights+lms+registrar+studio
 # All third-party services.
 # Separated by plus signs. Listed in alphabetical order for clarity.
 THIRD_PARTY_SERVICES ?= \
-chrome+coursegraph+devpi+elasticsearch+elasticsearch7+elasticsearch710+firefox+memcached+mongo+mysql57+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica
+chrome+coursegraph+devpi+elasticsearch+elasticsearch7+elasticsearch710+firefox+memcached+mongo+mysql80+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica

provision-credentials.sh

@@ -12,6 +12,9 @@ port=18150
 
 docker-compose up -d $name
 
+echo -e "${GREEN}Change MySQL Host to mysql80 for ${name}...${NC}"
+docker-compose exec -T ${name}  bash -e -c 'find /edx/ -type f -exec sed -i 's/mysql57/mysql80/g' {} +' -- "$name"
+
 echo -e "${GREEN}Installing requirements for ${name}...${NC}"
 docker-compose exec -T ${name}  bash -e -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && make requirements' -- "$name"
 

provision-ecommerce.sh

@@ -8,6 +8,9 @@ set -x
 ./provision-ida.sh ecommerce ecommerce 18130
 
 # Configure ecommerce
+echo -e "Change MySQL Host to mysql80"
+docker-compose exec -T ecommerce bash -e -c 'find /edx/ -type f -exec sed -i 's/mysql57/mysql80/g' {} +'
+
 docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_or_update_site --site-id=1 --site-domain=localhost:18130 --partner-code=edX --partner-name="Open edX" --lms-url-root=http://edx.devstack.lms:18000 --lms-public-url-root=http://localhost:18000 --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id=ecommerce-sso-key --sso-client-secret=ecommerce-sso-secret --backend-service-client-id=ecommerce-backend-service-key --backend-service-client-secret=ecommerce-backend-service-secret --from-email [email protected] --discovery_api_url=http://edx.devstack.discovery:18381/api/v1/ --enable-microfrontend-for-basket-page=1 --payment-microfrontend-url=http://localhost:1998'
 docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py oscar_populate_countries --initial-only'
 docker-compose exec -T ecommerce bash -e -c 'source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py create_demo_data --partner=edX'

provision-forum.sh

@@ -3,4 +3,5 @@ set -eu -o pipefail
 set -x
 
 docker-compose up -d forum
+docker-compose exec -T forum bash -e -c 'find /edx/ -type f -exec sed -i 's/mysql57/mysql80/g' {} +'
 docker-compose exec -T forum bash -e -c 'source /edx/app/forum/ruby_env && cd /edx/app/forum/cs_comments_service && bundle install --deployment --path /edx/app/forum/.gem/'