From 74d1008102805a0ec032ccf4beb6357fa0c1f67e Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Tue, 9 May 2023 16:30:35 -0400 Subject: [PATCH 1/4] fix: Be able to connect to SQL in containers from CMS By default if you use `localhost` as the `HOST` value for mysql, it tries to connect to a file socket on disk rather than trying to connect to the loopback hostname. This prevents us from running MySQL in a container while running the LMS on your local machine. Setting the host to `127.0.0.1` forces the SQL connection to go over TCP instead. This allows you to map your container port to your localhost without any issues. We did this in lms/envs/common.py in an earlier change but did not update cms/envs/common.py at that time. --- cms/envs/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index 77e8fe161d..41ef3f60c2 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1052,7 +1052,7 @@ DATABASES = { 'ATOMIC_REQUESTS': True, 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '127.0.0.1', 'NAME': 'edxapp', 'OPTIONS': {}, 'PASSWORD': 'password', @@ -1062,7 +1062,7 @@ DATABASES = { 'read_replica': { 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '127.0.0.1', 'NAME': 'edxapp', 'OPTIONS': {}, 'PASSWORD': 'password', @@ -1072,7 +1072,7 @@ DATABASES = { 'student_module_history': { 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '127.0.0.1', 'NAME': 'edxapp_csmh', 'OPTIONS': {}, 'PASSWORD': 'password', From 4338f555484314632562e684bf4fb9a2eb67a153 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 18 May 2023 14:40:12 -0400 Subject: [PATCH 2/4] fix: Load the DATA_DIR via derived_settings tooling. If we don't make the DATA_DIR a derived_setting, the only way to override it is to override the entirety of the MODULESTORE config, or override DATA_DIR in common.py in your fork. --- lms/envs/common.py | 5 +++-- lms/envs/minimal.yml | 4 ++++ lms/envs/production.py | 12 ++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index 8d0ac13e4a..b80e0878ec 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1614,7 +1614,7 @@ MODULESTORE = { 'DOC_STORE_CONFIG': DOC_STORE_CONFIG, 'OPTIONS': { 'default_class': 'xmodule.hidden_block.HiddenBlock', - 'fs_root': DATA_DIR, + 'fs_root': lambda settings: settings.DATA_DIR, 'render_template': 'common.djangoapps.edxmako.shortcuts.render_to_string', } }, @@ -1624,7 +1624,7 @@ MODULESTORE = { 'DOC_STORE_CONFIG': DOC_STORE_CONFIG, 'OPTIONS': { 'default_class': 'xmodule.hidden_block.HiddenBlock', - 'fs_root': DATA_DIR, + 'fs_root': lambda settings: settings.DATA_DIR, 'render_template': 'common.djangoapps.edxmako.shortcuts.render_to_string', } } @@ -1633,6 +1633,7 @@ MODULESTORE = { } } + DATABASES = { # edxapp's edxapp-migrate scripts and the edxapp_migrate play # will ensure that any DB not named read_replica will be migrated diff --git a/lms/envs/minimal.yml b/lms/envs/minimal.yml index f66dffc700..0688b7c49c 100644 --- a/lms/envs/minimal.yml +++ b/lms/envs/minimal.yml @@ -26,6 +26,10 @@ REST_FRAMEWORK: {} # off-machine storage to more easily scale the system. MEDIA_ROOT: "/tmp/edx-platform/media/" +# DATA_DIR is overridden twice in ./lms/envs/common.py override it here temporarily until that's cleaned up and we +# can default to something dev friendly and have overrides in production.py for production friendly settings. +DATA_DIR: "/tmp/edx-platform/data_dir" + # For just the CMS LMS_ROOT_URL: "http://localhost" LMS_INTERNAL_ROOT_URL: "http://localhost" diff --git a/lms/envs/production.py b/lms/envs/production.py index 342877206b..c8587f8fa6 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -490,6 +490,18 @@ XQUEUE_INTERFACE = AUTH_TOKENS.get('XQUEUE_INTERFACE', XQUEUE_INTERFACE) # Get the MODULESTORE from auth.json, but if it doesn't exist, # use the one from common.py MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS.get('MODULESTORE', MODULESTORE)) + +# After conversion above, the modulestore will have a "stores" list with all defined stores, for all stores, add the +# fs_root entry to derived collection so that if it's a callable it can be resolved. We need to do this because the +# `derived_collection_entry` takes an exact index value but the config file might have overidden the number of stores +# and so we can't be sure that the 2 we define in common.py will be there when we try to derive settings. This could +# lead to execptions being thrown when the `derive_settings` call later in this file tries to update settings. We call +# the derived_collection_entry function here to ensure that we update the fs_root for any callables that remain after +# we've updated the MODULESTORE setting from our config file. +for idx, store in enumerate(MODULESTORE['default']['OPTIONS']['stores']): + if 'OPTIONS' in store and 'fs_root' in store["OPTIONS"]: + derived_collection_entry('MODULESTORE', 'default', 'OPTIONS', 'stores', idx, 'OPTIONS', 'fs_root') + MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {}) EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', '') # django default is '' From 640e35e731704946e3e908490f1a386df78778d3 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 15 May 2023 12:01:33 -0400 Subject: [PATCH 3/4] test: Change how we test migrations. Use the default settings file and the minimal.yml config to build the migrations. --- .github/workflows/migrations-check.yml | 96 +++++++++++++++----------- 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 8bff55c0f6..aa5ceb23d0 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -1,4 +1,4 @@ -name: Migrations check on MySql 57 +name: Check Django Migrations on: workflow_dispatch: @@ -15,8 +15,54 @@ jobs: matrix: os: [ ubuntu-20.04 ] python-version: [ 3.8 ] - + mongo-version: ["4"] + mysql-version: ["5.7"] + services: + mongo: + image: mongo:${{ matrix.mongo-version }} + ports: + - 27017:27017 + # Note: Calling mongo here only works with mongo 4, in newer versions of mongo + # we'll have to use `mongosh` + options: >- + --health-cmd "mongo --quiet --eval 'db.runCommand(\"ping\")'" + --health-interval 10s + --health-timeout 5s + --health-retries 3 + mysql: + image: mysql:${{ matrix.mysql-version }} + ports: + - 3306:3306 + env: + MYSQL_DATABASE: "edxapp" + MYSQL_USER: "edxapp001" + MYSQL_PASSWORD: "password" + MYSQL_RANDOM_ROOT_PASSWORD: true + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 3 steps: + - name: Setup mongodb user + run: | + mongosh edxapp --eval ' + db.createUser( + { + user: "edxapp", + pwd: "password", + roles: [ + { role: "readWrite", db: "edxapp" }, + ] + } + ); + ' + + - name: Verify mongo and mysql db credentials + run: | + mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp + mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp + - name: Checkout repo uses: actions/checkout@v2 @@ -28,7 +74,7 @@ jobs: - name: Install system Packages run: | sudo apt-get update - sudo apt-get install -y libxmlsec1-dev + make ubuntu-requirements - name: Get pip cache dir id: pip-cache-dir @@ -43,51 +89,17 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} restore-keys: ${{ runner.os }}-pip- - - name: Install MySQL 5.7 - run: | - mkdir ${RUNNER_WORKSPACE}/mysql_packages && cd ${RUNNER_WORKSPACE}/mysql_packages - sudo apt-get remove --purge *mysql* -y - sudo rm -rvf /etc/init.d/mysql* /etc/mysql* /var/lib/mysql* - sudo apt-get install libaio1 libnuma1 libtinfo5 -y - wget https://cdn.mysql.com/archives/mysql-5.7/mysql-server_5.7.35-1ubuntu18.04_amd64.deb-bundle.tar - tar -xf mysql-server_5.7*.tar - sudo DEBIAN_FRONTEND=noninteractive dpkg --install mysql-common_5.7*.deb libmysqlclient*.deb mysql-server_5.7*.deb mysql-client_5.7*.deb mysql-community-server_5.7*.deb mysql-community-client_5.7*.deb - - - name: Ubuntu and sql Versions - run: | - lsb_release -a - mysql -V - - name: Install Python dependencies run: | make dev-requirements - - name: Initiate Services - run: | - sudo systemctl start mongod - sudo /etc/init.d/mysql start - - - name: Reset mysql password - run: | - cat < Date: Fri, 19 May 2023 12:25:56 -0400 Subject: [PATCH 4/4] test: Test both mysql 5.7 and 8 in one workflow. Now that we're using standard service containers, we can use a matrix to test both the current supported versions in one job. This should also make it easier to test with future versions as we get ready to upgrade. --- .github/workflows/migrations-check-mysql8.yml | 76 ------------------- .github/workflows/migrations-check.yml | 2 +- 2 files changed, 1 insertion(+), 77 deletions(-) delete mode 100644 .github/workflows/migrations-check-mysql8.yml diff --git a/.github/workflows/migrations-check-mysql8.yml b/.github/workflows/migrations-check-mysql8.yml deleted file mode 100644 index dd681f292f..0000000000 --- a/.github/workflows/migrations-check-mysql8.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Migrations check on MySql 8.0 - -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - -jobs: - check_migrations: - name: check migrations mysql8 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] - - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system Packages - run: | - sudo apt-get update - sudo apt-get install -y libxmlsec1-dev - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Ubuntu and sql Versions - run: | - lsb_release -a - mysql -V - - - name: Install Python dependencies - run: | - make dev-requirements - - - name: Initiate Services - run: | - sudo systemctl start mongod - sudo /etc/init.d/mysql start - - - name: Reset mysql password - run: | - cat <