Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #32239 from openedx/feanil/migration_check

Browse Source 
Migrate migration checks to `minimal.yml` config.
This commit is contained in:
Feanil Patel
2023-06-15 11:06:14 -04:00
committed by GitHub
parent 68aa7556c1 c32514a453
commit 82dde016f6
6 changed files with 76 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
.github/workflows/migrations-check-mysql8.yml vendored
View File

@@ -1,76 +0,0 @@
name: Migrations check on MySql 8.0
on:
workflow_dispatch:
pull_request:
push:
branches:
- master
jobs:
check_migrations:
name: check migrations mysql8
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-20.04 ]
python-version: [ 3.8 ]
steps:
- name: Checkout repo
uses: actions/checkout@v2
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install system Packages
run: |
sudo apt-get update
sudo apt-get install -y libxmlsec1-dev
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Cache pip dependencies
id: cache-dependencies
uses: actions/cache@v3
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Ubuntu and sql Versions
run: |
lsb_release -a
mysql -V
- name: Install Python dependencies
run: |
make dev-requirements
- name: Initiate Services
run: |
sudo systemctl start mongod
sudo /etc/init.d/mysql start
- name: Reset mysql password
run: |
cat <<EOF | mysql -h 127.0.0.1 -u root --password=root
UPDATE mysql.user SET authentication_string = null WHERE user = 'root';
FLUSH PRIVILEGES;
EOF
- name: Run Tests
env:
LMS_CFG: lms/envs/bok_choy.yml
run: |
echo "CREATE DATABASE IF NOT EXISTS edxtest;" | mysql -u root
echo "CREATE DATABASE IF NOT EXISTS student_module_history_test;" | mysql -u root
echo "Running the LMS migrations."
./manage.py lms --settings bok_choy migrate
echo "Running the CMS migrations."
./manage.py cms --settings bok_choy migrate

96
.github/workflows/migrations-check.yml vendored
View File

@@ -1,4 +1,4 @@
name: Migrations check on MySql 57
name: Check Django Migrations
on:
workflow_dispatch:
@@ -15,8 +15,54 @@ jobs:
matrix:
os: [ ubuntu-20.04 ]
python-version: [ 3.8 ]
mongo-version: ["4"]
mysql-version: ["5.7", "8"]
services:
mongo:
image: mongo:${{ matrix.mongo-version }}
ports:
- 27017:27017
# Note: Calling mongo here only works with mongo 4, in newer versions of mongo
# we'll have to use `mongosh`
options: >-
--health-cmd "mongo --quiet --eval 'db.runCommand(\"ping\")'"
--health-interval 10s
--health-timeout 5s
--health-retries 3
mysql:
image: mysql:${{ matrix.mysql-version }}
ports:
- 3306:3306
env:
MYSQL_DATABASE: "edxapp"
MYSQL_USER: "edxapp001"
MYSQL_PASSWORD: "password"
MYSQL_RANDOM_ROOT_PASSWORD: true
options: >-
--health-cmd "mysqladmin ping"
--health-interval 10s
--health-timeout 5s
--health-retries 3
steps:
- name: Setup mongodb user
run: |
mongosh edxapp --eval '
db.createUser(
{
user: "edxapp",
pwd: "password",
roles: [
{ role: "readWrite", db: "edxapp" },
]
}
);
'
- name: Verify mongo and mysql db credentials
run: |
mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp
mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp
- name: Checkout repo
uses: actions/checkout@v2
@@ -28,7 +74,7 @@ jobs:
- name: Install system Packages
run: |
sudo apt-get update
sudo apt-get install -y libxmlsec1-dev
make ubuntu-requirements
- name: Get pip cache dir
id: pip-cache-dir
@@ -43,51 +89,17 @@ jobs:
key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Install MySQL 5.7
run: |
mkdir ${RUNNER_WORKSPACE}/mysql_packages && cd ${RUNNER_WORKSPACE}/mysql_packages
sudo apt-get remove --purge *mysql* -y
sudo rm -rvf /etc/init.d/mysql* /etc/mysql* /var/lib/mysql*
sudo apt-get install libaio1 libnuma1 libtinfo5 -y
wget https://cdn.mysql.com/archives/mysql-5.7/mysql-server_5.7.35-1ubuntu18.04_amd64.deb-bundle.tar
tar -xf mysql-server_5.7*.tar
sudo DEBIAN_FRONTEND=noninteractive dpkg --install mysql-common_5.7*.deb libmysqlclient*.deb mysql-server_5.7*.deb mysql-client_5.7*.deb mysql-community-server_5.7*.deb mysql-community-client_5.7*.deb
- name: Ubuntu and sql Versions
run: |
lsb_release -a
mysql -V
- name: Install Python dependencies
run: |
make dev-requirements
- name: Initiate Services
run: |
sudo systemctl start mongod
sudo /etc/init.d/mysql start
- name: Reset mysql password
run: |
cat <<EOF | sudo mysql -u root
UPDATE mysql.user SET plugin = 'mysql_native_password' WHERE user = 'root';
UPDATE mysql.user SET authentication_string = null WHERE user = 'root';
FLUSH PRIVILEGES;
EOF
- name: Install mysqlclient-dev binary
run: |
sudo apt-get update
sudo apt-get install -y libmysqlclient-dev
- name: Run Tests
env:
LMS_CFG: lms/envs/bok_choy.yml
LMS_CFG: lms/envs/minimal.yml
# This is from the LMS dir on purpose since we don't need anything different for the CMS yet.
STUDIO_CFG: lms/envs/minimal.yml
run: |
echo "CREATE DATABASE IF NOT EXISTS edxtest;" | sudo mysql -u root
echo "CREATE DATABASE IF NOT EXISTS student_module_history_test;" | sudo mysql -u root
echo "Running the LMS migrations."
./manage.py lms --settings bok_choy migrate
./manage.py lms migrate
echo "Running the CMS migrations."
./manage.py cms --settings bok_choy migrate
./manage.py cms migrate

6
cms/envs/common.py
View File

@@ -1052,7 +1052,7 @@ DATABASES = {
'ATOMIC_REQUESTS': True,
'CONN_MAX_AGE': 0,
'ENGINE': 'django.db.backends.mysql',
'HOST': 'localhost',
'HOST': '127.0.0.1',
'NAME': 'edxapp',
'OPTIONS': {},
'PASSWORD': 'password',
@@ -1062,7 +1062,7 @@ DATABASES = {
'read_replica': {
'CONN_MAX_AGE': 0,
'ENGINE': 'django.db.backends.mysql',
'HOST': 'localhost',
'HOST': '127.0.0.1',
'NAME': 'edxapp',
'OPTIONS': {},
'PASSWORD': 'password',
@@ -1072,7 +1072,7 @@ DATABASES = {
'student_module_history': {
'CONN_MAX_AGE': 0,
'ENGINE': 'django.db.backends.mysql',
'HOST': 'localhost',
'HOST': '127.0.0.1',
'NAME': 'edxapp_csmh',
'OPTIONS': {},
'PASSWORD': 'password',

5
lms/envs/common.py
View File

@@ -1631,7 +1631,7 @@ MODULESTORE = {
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': {
'default_class': 'xmodule.hidden_block.HiddenBlock',
'fs_root': DATA_DIR,
'fs_root': lambda settings: settings.DATA_DIR,
'render_template': 'common.djangoapps.edxmako.shortcuts.render_to_string',
}
},
@@ -1641,7 +1641,7 @@ MODULESTORE = {
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': {
'default_class': 'xmodule.hidden_block.HiddenBlock',
'fs_root': DATA_DIR,
'fs_root': lambda settings: settings.DATA_DIR,
'render_template': 'common.djangoapps.edxmako.shortcuts.render_to_string',
}
}
@@ -1650,6 +1650,7 @@ MODULESTORE = {
}
}
DATABASES = {
# edxapp's edxapp-migrate scripts and the edxapp_migrate play
# will ensure that any DB not named read_replica will be migrated

4
lms/envs/minimal.yml
View File

@@ -26,6 +26,10 @@ REST_FRAMEWORK: {}
# off-machine storage to more easily scale the system.
MEDIA_ROOT: "/tmp/edx-platform/media/"
# DATA_DIR is overridden twice in ./lms/envs/common.py override it here temporarily until that's cleaned up and we
# can default to something dev friendly and have overrides in production.py for production friendly settings.
DATA_DIR: "/tmp/edx-platform/data_dir"
# For just the CMS
LMS_ROOT_URL: "http://localhost"
LMS_INTERNAL_ROOT_URL: "http://localhost"

12
lms/envs/production.py
View File

@@ -490,6 +490,18 @@ XQUEUE_INTERFACE = AUTH_TOKENS.get('XQUEUE_INTERFACE', XQUEUE_INTERFACE)
# Get the MODULESTORE from auth.json, but if it doesn't exist,
# use the one from common.py
MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS.get('MODULESTORE', MODULESTORE))
# After conversion above, the modulestore will have a "stores" list with all defined stores, for all stores, add the
# fs_root entry to derived collection so that if it's a callable it can be resolved. We need to do this because the
# `derived_collection_entry` takes an exact index value but the config file might have overidden the number of stores
# and so we can't be sure that the 2 we define in common.py will be there when we try to derive settings. This could
# lead to execptions being thrown when the `derive_settings` call later in this file tries to update settings. We call
# the derived_collection_entry function here to ensure that we update the fs_root for any callables that remain after
# we've updated the MODULESTORE setting from our config file.
for idx, store in enumerate(MODULESTORE['default']['OPTIONS']['stores']):
if 'OPTIONS' in store and 'fs_root' in store["OPTIONS"]:
derived_collection_entry('MODULESTORE', 'default', 'OPTIONS', 'stores', idx, 'OPTIONS', 'fs_root')
MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {})
EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', '') # django default is ''