From 42ca0ddec46c19ccd5ecfaed939de6512e960916 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 27 Sep 2023 10:58:37 +0500 Subject: [PATCH 01/35] chore: remove bok choy settings --- .github/workflows/js-tests.yml | 14 +- .github/workflows/static-assets-check.yml | 5 - .gitignore | 1 - cms/envs/bok_choy.auth.json | 116 ----- cms/envs/bok_choy.env.json | 127 ----- cms/envs/bok_choy.py | 184 ------- cms/envs/bok_choy.yml | 150 ------ cms/envs/bok_choy_docker.auth.json | 116 ----- cms/envs/bok_choy_docker.env.json | 126 ----- cms/envs/bok_choy_docker.py | 26 - cms/envs/bok_choy_docker.yml | 147 ------ common/djangoapps/terrain/stubs/lti.py | 4 +- .../static/common/js/jasmine.common.conf.js | 12 - .../static/common/js/jasmine_stack_trace.js | 29 -- common/static/common/js/karma.common.conf.js | 467 ------------------ .../0004-managing-django-settings.rst | 2 - lms/envs/bok_choy.py | 154 ------ lms/envs/bok_choy.yml | 274 ---------- lms/envs/bok_choy_docker.py | 27 - lms/envs/bok_choy_docker.yml | 183 ------- .../management/commands/update_fixtures.py | 30 -- .../util/tests/test_update_fixtures.py | 42 -- .../content_type_gating/tests/test_access.py | 3 +- .../pytest_plugin.py | 25 +- pavelib/utils/envs.py | 93 +--- requirements/edx/development.txt | 8 - requirements/edx/testing.in | 2 - requirements/edx/testing.txt | 8 - scripts/reset-test-db.sh | 140 ------ tox.ini | 4 - xmodule/modulestore/tests/mongo_connection.py | 3 +- xmodule/tests/test_mongo_utils.py | 3 +- xmodule/tests/test_video.py | 3 +- 33 files changed, 28 insertions(+), 2500 deletions(-) delete mode 100644 cms/envs/bok_choy.auth.json delete mode 100644 cms/envs/bok_choy.env.json delete mode 100644 cms/envs/bok_choy.py delete mode 100644 cms/envs/bok_choy.yml delete mode 100644 cms/envs/bok_choy_docker.auth.json delete mode 100644 cms/envs/bok_choy_docker.env.json delete mode 100644 cms/envs/bok_choy_docker.py delete mode 100644 cms/envs/bok_choy_docker.yml delete mode 100644 common/static/common/js/jasmine.common.conf.js delete mode 100644 common/static/common/js/jasmine_stack_trace.js delete mode 100644 common/static/common/js/karma.common.conf.js delete mode 100644 lms/envs/bok_choy.py delete mode 100644 lms/envs/bok_choy.yml delete mode 100644 lms/envs/bok_choy_docker.py delete mode 100644 lms/envs/bok_choy_docker.yml delete mode 100644 openedx/core/djangoapps/util/management/commands/update_fixtures.py delete mode 100644 openedx/core/djangoapps/util/tests/test_update_fixtures.py delete mode 100755 scripts/reset-test-db.sh diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index c26e4e79fc..092e6e38e2 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -30,13 +30,13 @@ jobs: - name: Setup npm run: npm i -g npm@8.5.x - - name: Install Firefox 61.0 - run: | - sudo apt-get purge firefox - wget "https://ftp.mozilla.org/pub/firefox/releases/61.0/linux-x86_64/en-US/firefox-61.0.tar.bz2" - tar -xjf firefox-61.0.tar.bz2 - sudo mv firefox /opt/firefox - sudo ln -s /opt/firefox/firefox /usr/bin/firefox + # - name: Install Firefox 61.0 + # run: | + # sudo apt-get purge firefox + # wget "https://ftp.mozilla.org/pub/firefox/releases/61.0/linux-x86_64/en-US/firefox-61.0.tar.bz2" + # tar -xjf firefox-61.0.tar.bz2 + # sudo mv firefox /opt/firefox + # sudo ln -s /opt/firefox/firefox /usr/bin/firefox - name: Install Required System Packages run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index c4da37f2d1..7aa7dce783 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -62,11 +62,6 @@ jobs: - name: Add node_modules bin to $Path run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH - - name: Run Static Assets Check - env: - LMS_CFG: lms/envs/bok_choy.yml - CMS_CFG: cms/envs/bok_choy.yml - run: | paver update_assets lms paver update_assets cms diff --git a/.gitignore b/.gitignore index d6f1202bf4..e7f12787f5 100644 --- a/.gitignore +++ b/.gitignore @@ -86,7 +86,6 @@ test_root/paver_logs/ test_root/uploads/ django-pyfs .tox/ -common/test/db_cache/bok_choy_*.yaml common/test/data/badges/*.png ### Installation artifacts diff --git a/cms/envs/bok_choy.auth.json b/cms/envs/bok_choy.auth.json deleted file mode 100644 index db92b5ba92..0000000000 --- a/cms/envs/bok_choy.auth.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "AWS_ACCESS_KEY_ID": "", - "AWS_SECRET_ACCESS_KEY": "", - "CELERY_BROKER_PASSWORD": "celery", - "CELERY_BROKER_USER": "celery", - "CONTENTSTORE": { - "DOC_STORE_CONFIG": { - "collection": "modulestore", - "db": "test", - "host": [ - "localhost" - ], - "port": 27017 - }, - "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", - "OPTIONS": { - "db": "test", - "host": [ - "localhost" - ], - "port": 27017 - } - }, - "DATABASES": { - "default": { - "ENGINE": "django.db.backends.mysql", - "HOST": "localhost", - "NAME": "edxtest", - "PASSWORD": "", - "PORT": "3306", - "USER": "root" - }, - "student_module_history": { - "ENGINE": "django.db.backends.mysql", - "HOST": "localhost", - "NAME": "student_module_history_test", - "PASSWORD": "", - "PORT": "3306", - "USER": "root" - } - }, - "DOC_STORE_CONFIG": { - "collection": "modulestore", - "db": "test", - "host": [ - "localhost" - ], - "port": 27017 - }, - "JWT_AUTH": { - "JWT_SECRET_KEY": "super-secret-key", - "JWT_PUBLIC_SIGNING_JWK_SET": "{\"keys\": [{\"kid\": \"BTZ9HA6K\", \"e\": \"AQAB\", \"kty\": \"RSA\", \"n\": \"o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ\"}]}" - }, - "MODULESTORE": { - "default": { - "ENGINE": "xmodule.modulestore.mixed.MixedModuleStore", - "OPTIONS": { - "mappings": {}, - "stores": [ - { - "NAME": "draft", - "DOC_STORE_CONFIG": { - "collection": "modulestore", - "db": "test", - "host": [ - "localhost" - ], - "port": 27017 - }, - "ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore", - "OPTIONS": { - "collection": "modulestore", - "db": "test", - "default_class": "xmodule.hidden_block.HiddenBlock", - "fs_root": "** OVERRIDDEN **", - "host": [ - "localhost" - ], - "port": 27017, - "render_template": "common.djangoapps.edxmako.shortcuts.render_to_string" - } - }, - { - "NAME": "xml", - "ENGINE": "xmodule.modulestore.xml.XMLModuleStore", - "OPTIONS": { - "data_dir": "** OVERRIDDEN **", - "default_class": "xmodule.hidden_block.HiddenBlock" - } - } - ] - } - } - }, - "DJFS": { - "type": "s3fs", - "bucket": "test", - "prefix": "test", - "aws_access_key_id": "test", - "aws_secret_access_key": "test" - }, - "SECRET_KEY": "", - "XQUEUE_INTERFACE": { - "basic_auth": [ - "edx", - "edx" - ], - "django_auth": { - "password": "password", - "username": "lms" - }, - "url": "http://localhost:18040" - }, - "ZENDESK_API_KEY": "", - "ZENDESK_USER": "" -} diff --git a/cms/envs/bok_choy.env.json b/cms/envs/bok_choy.env.json deleted file mode 100644 index 6343dc048c..0000000000 --- a/cms/envs/bok_choy.env.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "BUGS_EMAIL": "bugs@example.com", - "BULK_EMAIL_DEFAULT_FROM_EMAIL": "no-reply@example.com", - "CACHES": { - "celery": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "OPTIONS": { - "no_delay": true, - "ignore_exc": true, - "use_pooling": true - }, - "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", - "KEY_PREFIX": "integration_celery", - "LOCATION": [ - "localhost:11211" - ] - }, - "default": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "OPTIONS": { - "no_delay": true, - "ignore_exc": true, - "use_pooling": true - }, - "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", - "KEY_PREFIX": "sandbox_default", - "LOCATION": [ - "localhost:11211" - ] - }, - "general": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "OPTIONS": { - "no_delay": true, - "ignore_exc": true, - "use_pooling": true - }, - "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", - "KEY_PREFIX": "sandbox_general", - "LOCATION": [ - "localhost:11211" - ] - }, - "mongo_metadata_inheritance": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "OPTIONS": { - "no_delay": true, - "ignore_exc": true, - "use_pooling": true - }, - "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", - "KEY_PREFIX": "integration_mongo_metadata_inheritance", - "LOCATION": [ - "localhost:11211" - ] - }, - "staticfiles": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "OPTIONS": { - "no_delay": true, - "ignore_exc": true, - "use_pooling": true - }, - "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", - "KEY_PREFIX": "integration_static_files", - "LOCATION": [ - "localhost:11211" - ] - } - }, - "CELERY_ALWAYS_EAGER": true, - "CELERY_BROKER_HOSTNAME": "localhost", - "CELERY_BROKER_TRANSPORT": "amqp", - "CERT_QUEUE": "certificates", - "CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION": false, - "CMS_BASE": "localhost:8031", - "CODE_JAIL": { - "limits": { - "REALTIME": 3, - "VMEM": 0 - } - }, - "COMMENTS_SERVICE_KEY": "password", - "COMMENTS_SERVICE_URL": "http://localhost:4567", - "CONTACT_EMAIL": "info@example.com", - "DEFAULT_FEEDBACK_EMAIL": "feedback@example.com", - "DEFAULT_FROM_EMAIL": "registration@example.com", - "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend", - "SOCIAL_SHARING_SETTINGS": { - "CUSTOM_COURSE_URLS": true - }, - "FEATURES": { - "CERTIFICATES_HTML_VIEW": true, - "ENABLE_DISCUSSION_SERVICE": true, - "ENABLE_GRADE_DOWNLOADS": true, - "ENTRANCE_EXAMS": true, - "MILESTONES_APP": true, - "PREVIEW_LMS_BASE": "preview.localhost:8003", - "ENABLE_CONTENT_LIBRARIES": true, - "ENABLE_SPECIAL_EXAMS": true, - "SHOW_HEADER_LANGUAGE_SELECTOR": true, - "ENABLE_EXTENDED_COURSE_DETAILS": true, - "CUSTOM_COURSES_EDX": true - }, - "FEEDBACK_SUBMISSION_EMAIL": "", - "GITHUB_REPO_ROOT": "** OVERRIDDEN **", - "GRADES_DOWNLOAD": { - "BUCKET": "edx-grades", - "ROOT_PATH": "/tmp/edx-s3/grades", - "STORAGE_TYPE": "localfs" - }, - "LMS_BASE": "localhost:8003", - "LMS_ROOT_URL": "http://localhost:8003", - "LOCAL_LOGLEVEL": "INFO", - "LOGGING_ENV": "sandbox", - "LOG_DIR": "** OVERRIDDEN **", - "MEDIA_URL": "/media/", - "MKTG_URL_LINK_MAP": {}, - "SERVER_EMAIL": "devops@example.com", - "SESSION_COOKIE_DOMAIN": null, - "SITE_NAME": "localhost", - "STATIC_URL_BASE": "/static/", - "SYSLOG_SERVER": "", - "TECH_SUPPORT_EMAIL": "technical@example.com", - "TIME_ZONE": "America/New_York", - "WIKI_ENABLED": true, -} diff --git a/cms/envs/bok_choy.py b/cms/envs/bok_choy.py deleted file mode 100644 index f96183840c..0000000000 --- a/cms/envs/bok_choy.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -Settings for Bok Choy tests that are used when running Studio. - -Bok Choy uses two different settings files: -1. test_static_optimized is used when invoking collectstatic -2. bok_choy is used when running the tests - -Note: it isn't possible to have a single settings file, because Django doesn't -support both generating static assets to a directory and also serving static -from the same directory. -""" - - -# Silence noisy logs -import logging -import os - -from django.utils.translation import gettext_lazy -from path import Path as path - -from openedx.core.release import RELEASE_LINE -from xmodule.modulestore.modulestore_settings import update_module_store_settings # lint-amnesty, pylint: disable=wrong-import-order - -########################## Prod-like settings ################################### -# These should be as close as possible to the settings we use in production. -# As in prod, we read in environment and auth variables from JSON files. -# Unlike in prod, we use the JSON files stored in this repo. -# This is a convenience for ensuring (a) that we can consistently find the files -# and (b) that the files are the same in Jenkins as in local dev. -os.environ['SERVICE_VARIANT'] = 'bok_choy_docker' if 'BOK_CHOY_HOSTNAME' in os.environ else 'bok_choy' -CONFIG_ROOT = path(__file__).abspath().dirname() -os.environ['STUDIO_CFG'] = str.format("{config_root}/{service_variant}.yml", - config_root=CONFIG_ROOT, - service_variant=os.environ['SERVICE_VARIANT']) -os.environ['REVISION_CFG'] = f"{CONFIG_ROOT}/revisions.yml" - -from .production import * # pylint: disable=wildcard-import, unused-wildcard-import, wrong-import-position - - -######################### Testing overrides #################################### - -# Redirect to the test_root folder within the repo -TEST_ROOT = REPO_ROOT / "test_root" -GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath() -LOG_DIR = (TEST_ROOT / "log").abspath() -DATA_DIR = TEST_ROOT / "data" - -# Configure modulestore to use the test folder within the repo -update_module_store_settings( - MODULESTORE, - module_store_options={ - 'fs_root': (TEST_ROOT / "data").abspath(), - }, - xml_store_options={ - 'data_dir': (TEST_ROOT / "data").abspath(), - }, - default_store=os.environ.get('DEFAULT_STORE', 'draft'), -) - -# Needed to enable licensing on video blocks -XBLOCK_SETTINGS.update({'VideoBlock': {'licensing_enabled': True}}) - -# Capture the console log via template includes, until webdriver supports log capture again -CAPTURE_CONSOLE_LOG = True - -PLATFORM_NAME = gettext_lazy("édX") -PLATFORM_DESCRIPTION = gettext_lazy("Open édX Platform") -STUDIO_NAME = gettext_lazy("Your Platform 𝓢𝓽𝓾𝓭𝓲𝓸") -STUDIO_SHORT_NAME = gettext_lazy("𝓢𝓽𝓾𝓭𝓲𝓸") - -############################ STATIC FILES ############################# - -# Enable debug so that static assets are served by Django -DEBUG = True - -# Serve static files at /static directly from the staticfiles directory under test root -# Note: optimized files for testing are generated with settings from test_static_optimized -STATIC_URL = "/static/" -STATICFILES_FINDERS = [ - 'django.contrib.staticfiles.finders.FileSystemFinder', -] -STATICFILES_DIRS = [ - (TEST_ROOT / "staticfiles" / "cms").abspath(), -] - -DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' -MEDIA_ROOT = TEST_ROOT / "uploads" - -WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = TEST_ROOT / "staticfiles" / "cms" / "webpack-stats.json" - -LOG_OVERRIDES = [ - ('common.djangoapps.track.middleware', logging.CRITICAL), - ('edx.discussion', logging.CRITICAL), -] -for log_name, log_level in LOG_OVERRIDES: - logging.getLogger(log_name).setLevel(log_level) - -# Use the auto_auth workflow for creating users and logging them in -FEATURES['AUTOMATIC_AUTH_FOR_TESTING'] = True -FEATURES['RESTRICT_AUTOMATIC_AUTH'] = False - -# Enable milestones app -FEATURES['MILESTONES_APP'] = True - -# Enable pre-requisite course -FEATURES['ENABLE_PREREQUISITE_COURSES'] = True - -# Enable student notes -FEATURES['ENABLE_EDXNOTES'] = True - -# Enable teams feature -FEATURES['ENABLE_TEAMS'] = True - -# Enable custom content licensing -FEATURES['LICENSING'] = True - -FEATURES['ENABLE_MOBILE_REST_API'] = True # Enable video bumper in Studio -FEATURES['ENABLE_VIDEO_BUMPER'] = True # Enable video bumper in Studio settings - -FEATURES['ENABLE_ENROLLMENT_TRACK_USER_PARTITION'] = True - -# Whether archived courses (courses with end dates in the past) should be -# shown in Studio in a separate list. -FEATURES['ENABLE_SEPARATE_ARCHIVED_COURSES'] = True - -# Enable support for OpenBadges accomplishments -FEATURES['ENABLE_OPENBADGES'] = True - -# Enable partner support link in Studio footer -PARTNER_SUPPORT_EMAIL = 'partner-support@example.com' - -########################### Entrance Exams ################################# -FEATURES['ENTRANCE_EXAMS'] = True - -FEATURES['ENABLE_SPECIAL_EXAMS'] = True - -# Point the URL used to test YouTube availability to our stub YouTube server -YOUTUBE_PORT = 9080 -YOUTUBE['TEST_TIMEOUT'] = 5000 -YOUTUBE_HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', '127.0.0.1') -YOUTUBE['API'] = f"http://{YOUTUBE_HOSTNAME}:{YOUTUBE_PORT}/get_youtube_api/" -YOUTUBE['METADATA_URL'] = f"http://{YOUTUBE_HOSTNAME}:{YOUTUBE_PORT}/test_youtube/" - -FEATURES['ENABLE_COURSEWARE_INDEX'] = True -FEATURES['ENABLE_LIBRARY_INDEX'] = True -FEATURES['ENABLE_CONTENT_LIBRARY_INDEX'] = False - -ORGANIZATIONS_AUTOCREATE = False - -SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine" -# Path at which to store the mock index -MOCK_SEARCH_BACKING_FILE = ( - TEST_ROOT / "index_file.dat" -).abspath() - -# this secret key should be the same as lms/envs/bok_choy.py's -SECRET_KEY = "very_secret_bok_choy_key" - -LMS_ROOT_URL = "http://localhost:8003" -if RELEASE_LINE == "master": - # On master, acceptance tests use edX books, not the default Open edX books. - HELP_TOKENS_BOOKS = { - 'learner': 'https://edx.readthedocs.io/projects/edx-guide-for-students', - 'course_author': 'https://edx.readthedocs.io/projects/edx-partner-course-staff', - } - -########################## VIDEO TRANSCRIPTS STORAGE ############################ -VIDEO_TRANSCRIPTS_SETTINGS = dict( - VIDEO_TRANSCRIPTS_MAX_BYTES=3 * 1024 * 1024, # 3 MB - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), - DIRECTORY_PREFIX='video-transcripts/', -) - -INSTALLED_APPS.append('openedx.testing.coverage_context_listener') - -##################################################################### -# Lastly, see if the developer has any local overrides. -try: - from .private import * # pylint: disable=wildcard-import -except ImportError: - pass diff --git a/cms/envs/bok_choy.yml b/cms/envs/bok_choy.yml deleted file mode 100644 index 03bd022414..0000000000 --- a/cms/envs/bok_choy.yml +++ /dev/null @@ -1,150 +0,0 @@ -# ingested bok_choy.env.json -# ingested bok_choy.auth.json -AWS_ACCESS_KEY_ID: '' -AWS_SECRET_ACCESS_KEY: '' -BUGS_EMAIL: bugs@example.com -BULK_EMAIL_DEFAULT_FROM_EMAIL: no-reply@example.com -CACHES: - celery: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - ignore_exc: true - no_delay: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_celery - LOCATION: ['localhost:11211'] - default: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - ignore_exc: true - no_delay: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: sandbox_default - LOCATION: ['localhost:11211'] - general: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - ignore_exc: true - no_delay: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: sandbox_general - LOCATION: ['localhost:11211'] - mongo_metadata_inheritance: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - ignore_exc: true - no_delay: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_mongo_metadata_inheritance - LOCATION: ['localhost:11211'] - staticfiles: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - ignore_exc: true - no_delay: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_static_files - LOCATION: ['localhost:11211'] -CELERY_ALWAYS_EAGER: true -CELERY_BROKER_HOSTNAME: localhost -CELERY_BROKER_PASSWORD: celery -CELERY_BROKER_TRANSPORT: amqp -CELERY_BROKER_USER: celery -CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION: false -CERT_QUEUE: certificates -CMS_BASE: localhost:8031 -CODE_JAIL: - limits: {REALTIME: 3, VMEM: 0} -COMMENTS_SERVICE_KEY: password -COMMENTS_SERVICE_URL: http://localhost:4567 -CONTACT_EMAIL: info@example.com -CONTENTSTORE: - DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [localhost] - port: 27017 - ENGINE: xmodule.contentstore.mongo.MongoContentStore - OPTIONS: - db: test - host: [localhost] - port: 27017 -DATABASES: - default: {ENGINE: django.db.backends.mysql, HOST: localhost, NAME: edxtest, PASSWORD: '', - PORT: '3306', USER: root} - student_module_history: {ENGINE: django.db.backends.mysql, HOST: localhost, NAME: student_module_history_test, - PASSWORD: '', PORT: '3306', USER: root} -DEFAULT_FEEDBACK_EMAIL: feedback@example.com -DEFAULT_FROM_EMAIL: registration@example.com -DJFS: {aws_access_key_id: test, aws_secret_access_key: test, bucket: test, prefix: test, - type: s3fs} -DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [localhost] - port: 27017 -EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend -FEATURES: {CERTIFICATES_HTML_VIEW: true, CUSTOM_COURSES_EDX: true, - ENABLE_CONTENT_LIBRARIES: true, ENABLE_DISCUSSION_SERVICE: true, ENABLE_EXTENDED_COURSE_DETAILS: true, - ENABLE_GRADE_DOWNLOADS: true, ENABLE_SPECIAL_EXAMS: true, ENTRANCE_EXAMS: true, - MILESTONES_APP: true, PREVIEW_LMS_BASE: 'preview.localhost:8003', SHOW_HEADER_LANGUAGE_SELECTOR: true} -GITHUB_REPO_ROOT: '** OVERRIDDEN **' -GRADES_DOWNLOAD: {BUCKET: edx-grades, ROOT_PATH: /tmp/edx-s3/grades, STORAGE_TYPE: localfs} -JWT_AUTH: {JWT_PUBLIC_SIGNING_JWK_SET: '{"keys": [{"kid": - "BTZ9HA6K", "e": "AQAB", "kty": "RSA", "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ"}]}', - JWT_SECRET_KEY: super-secret-key} -LMS_BASE: localhost:8003 -LMS_ROOT_URL: http://localhost:8003 -LOCAL_LOGLEVEL: INFO -LOGGING_ENV: sandbox -LOG_DIR: '** OVERRIDDEN **' -MEDIA_URL: /media/ -MKTG_URL_LINK_MAP: {} -MODULESTORE: - default: - ENGINE: xmodule.modulestore.mixed.MixedModuleStore - OPTIONS: - mappings: {} - stores: - - DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [localhost] - port: 27017 - ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore - NAME: draft - OPTIONS: - collection: modulestore - db: test - default_class: xmodule.hidden_block.HiddenBlock - fs_root: '** OVERRIDDEN **' - host: [localhost] - port: 27017 - render_template: common.djangoapps.edxmako.shortcuts.render_to_string - - ENGINE: xmodule.modulestore.xml.XMLModuleStore - NAME: xml - OPTIONS: {data_dir: '** OVERRIDDEN **', default_class: xmodule.hidden_block.HiddenBlock} -# We need to test different scenarios, following setting effectively disbale rate limiting -PASSWORD_RESET_IP_RATE: '1/s' -PASSWORD_RESET_EMAIL_RATE: '1/s' -SECRET_KEY: '' -SERVER_EMAIL: devops@example.com -SESSION_COOKIE_DOMAIN: null -SITE_NAME: localhost -SOCIAL_SHARING_SETTINGS: {CUSTOM_COURSE_URLS: true} -STATIC_URL_BASE: /static/ -SYSLOG_SERVER: '' -TECH_SUPPORT_EMAIL: technical@example.com -TIME_ZONE: America/New_York -WIKI_ENABLED: true -XQUEUE_INTERFACE: - basic_auth: [edx, edx] - django_auth: {password: password, username: lms} - url: http://localhost:18040 -ZENDESK_API_KEY: '' -ZENDESK_USER: '' diff --git a/cms/envs/bok_choy_docker.auth.json b/cms/envs/bok_choy_docker.auth.json deleted file mode 100644 index fce8a970fe..0000000000 --- a/cms/envs/bok_choy_docker.auth.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "AWS_ACCESS_KEY_ID": "", - "AWS_SECRET_ACCESS_KEY": "", - "CELERY_BROKER_PASSWORD": "celery", - "CELERY_BROKER_USER": "celery", - "CONTENTSTORE": { - "DOC_STORE_CONFIG": { - "collection": "modulestore", - "db": "test", - "host": [ - "edx.devstack.mongo" - ], - "port": 27017 - }, - "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", - "OPTIONS": { - "db": "test", - "host": [ - "edx.devstack.mongo" - ], - "port": 27017 - } - }, - "DATABASES": { - "default": { - "ENGINE": "django.db.backends.mysql", - "HOST": "edx.devstack.mysql57", - "NAME": "edxtest", - "PASSWORD": "", - "PORT": "3306", - "USER": "root" - }, - "student_module_history": { - "ENGINE": "django.db.backends.mysql", - "HOST": "edx.devstack.mysql57", - "NAME": "student_module_history_test", - "PASSWORD": "", - "PORT": "3306", - "USER": "root" - } - }, - "DOC_STORE_CONFIG": { - "collection": "modulestore", - "db": "test", - "host": [ - "edx.devstack.mongo" - ], - "port": 27017 - }, - "JWT_AUTH": { - "JWT_SECRET_KEY": "super-secret-key", - "JWT_PUBLIC_SIGNING_JWK_SET": "{\"keys\": [{\"kid\": \"BTZ9HA6K\", \"e\": \"AQAB\", \"kty\": \"RSA\", \"n\": \"o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ\"}]}" - }, - "MODULESTORE": { - "default": { - "ENGINE": "xmodule.modulestore.mixed.MixedModuleStore", - "OPTIONS": { - "mappings": {}, - "stores": [ - { - "NAME": "draft", - "DOC_STORE_CONFIG": { - "collection": "modulestore", - "db": "test", - "host": [ - "edx.devstack.mongo" - ], - "port": 27017 - }, - "ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore", - "OPTIONS": { - "collection": "modulestore", - "db": "test", - "default_class": "xmodule.hidden_block.HiddenBlock", - "fs_root": "** OVERRIDDEN **", - "host": [ - "edx.devstack.mongo" - ], - "port": 27017, - "render_template": "common.djangoapps.edxmako.shortcuts.render_to_string" - } - }, - { - "NAME": "xml", - "ENGINE": "xmodule.modulestore.xml.XMLModuleStore", - "OPTIONS": { - "data_dir": "** OVERRIDDEN **", - "default_class": "xmodule.hidden_block.HiddenBlock" - } - } - ] - } - } - }, - "DJFS": { - "type": "s3fs", - "bucket": "test", - "prefix": "test", - "aws_access_key_id": "test", - "aws_secret_access_key": "test" - }, - "SECRET_KEY": "", - "XQUEUE_INTERFACE": { - "basic_auth": [ - "edx", - "edx" - ], - "django_auth": { - "password": "password", - "username": "lms" - }, - "url": "http://localhost:18040" - }, - "ZENDESK_API_KEY": "", - "ZENDESK_USER": "" -} diff --git a/cms/envs/bok_choy_docker.env.json b/cms/envs/bok_choy_docker.env.json deleted file mode 100644 index 47fed49f1a..0000000000 --- a/cms/envs/bok_choy_docker.env.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "BUGS_EMAIL": "bugs@example.com", - "BULK_EMAIL_DEFAULT_FROM_EMAIL": "no-reply@example.com", - "CACHES": { - "celery": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "OPTIONS": { - "no_delay": true, - "ignore_exc": true, - "use_pooling": true - }, - "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", - "KEY_PREFIX": "integration_celery", - "LOCATION": [ - "edx.devstack.memcached:11211" - ] - }, - "default": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "OPTIONS": { - "no_delay": true, - "ignore_exc": true, - "use_pooling": true - }, - "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", - "KEY_PREFIX": "sandbox_default", - "LOCATION": [ - "edx.devstack.memcached:11211" - ] - }, - "general": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "OPTIONS": { - "no_delay": true, - "ignore_exc": true, - "use_pooling": true - }, - "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", - "KEY_PREFIX": "sandbox_general", - "LOCATION": [ - "edx.devstack.memcached:11211" - ] - }, - "mongo_metadata_inheritance": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "OPTIONS": { - "no_delay": true, - "ignore_exc": true, - "use_pooling": true - }, - "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", - "KEY_PREFIX": "integration_mongo_metadata_inheritance", - "LOCATION": [ - "edx.devstack.memcached:11211" - ] - }, - "staticfiles": { - "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", - "OPTIONS": { - "no_delay": true, - "ignore_exc": true, - "use_pooling": true - }, - "KEY_FUNCTION": "common.djangoapps.util.memcache.safe_key", - "KEY_PREFIX": "integration_static_files", - "LOCATION": [ - "edx.devstack.memcached:11211" - ] - } - }, - "CELERY_ALWAYS_EAGER": true, - "CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION": false, - "CELERY_BROKER_HOSTNAME": "localhost", - "CELERY_BROKER_TRANSPORT": "amqp", - "CERT_QUEUE": "certificates", - "CMS_BASE": "** OVERRIDDEN **", - "CODE_JAIL": { - "limits": { - "REALTIME": 3, - "VMEM": 0 - } - }, - "COMMENTS_SERVICE_KEY": "password", - "COMMENTS_SERVICE_URL": "http://edx.devstack.studio:4567", - "CONTACT_EMAIL": "info@example.com", - "DEFAULT_FEEDBACK_EMAIL": "feedback@example.com", - "DEFAULT_FROM_EMAIL": "registration@example.com", - "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend", - "SOCIAL_SHARING_SETTINGS": { - "CUSTOM_COURSE_URLS": true - }, - "FEATURES": { - "CERTIFICATES_HTML_VIEW": true, - "ENABLE_DISCUSSION_SERVICE": true, - "ENABLE_GRADE_DOWNLOADS": true, - "ENTRANCE_EXAMS": true, - "MILESTONES_APP": true, - "PREVIEW_LMS_BASE": "preview.localhost:8003", - "ENABLE_CONTENT_LIBRARIES": true, - "ENABLE_SPECIAL_EXAMS": true, - "SHOW_HEADER_LANGUAGE_SELECTOR": true, - "ENABLE_EXTENDED_COURSE_DETAILS": true, - "CUSTOM_COURSES_EDX": true - }, - "GITHUB_REPO_ROOT": "** OVERRIDDEN **", - "GRADES_DOWNLOAD": { - "BUCKET": "edx-grades", - "ROOT_PATH": "/tmp/edx-s3/grades", - "STORAGE_TYPE": "localfs" - }, - "LMS_BASE": "** OVERRIDDEN **", - "LMS_ROOT_URL": "** OVERRIDDEN **", - "LOCAL_LOGLEVEL": "INFO", - "LOGGING_ENV": "sandbox", - "LOG_DIR": "** OVERRIDDEN **", - "MEDIA_URL": "/media/", - "MKTG_URL_LINK_MAP": {}, - "SERVER_EMAIL": "devops@example.com", - "SESSION_COOKIE_DOMAIN": null, - "SITE_NAME": "localhost", - "STATIC_URL_BASE": "/static/", - "SYSLOG_SERVER": "", - "TECH_SUPPORT_EMAIL": "technical@example.com", - "TIME_ZONE": "America/New_York", - "WIKI_ENABLED": true, -} diff --git a/cms/envs/bok_choy_docker.py b/cms/envs/bok_choy_docker.py deleted file mode 100644 index 878537de00..0000000000 --- a/cms/envs/bok_choy_docker.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Settings for Bok Choy tests that are used when running Studio in Docker-based devstack. -""" - -# noinspection PyUnresolvedReferences -from .bok_choy import * # pylint: disable=wildcard-import - -CMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ.get('BOK_CHOY_CMS_PORT', 8031)) -LMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ.get('BOK_CHOY_LMS_PORT', 8003)) -LMS_ROOT_URL = f'http://{LMS_BASE}' -LOGIN_REDIRECT_WHITELIST = [CMS_BASE] - -COMMENTS_SERVICE_URL = 'http://{}:4567'.format(os.environ['BOK_CHOY_HOSTNAME']) -EDXNOTES_PUBLIC_API = 'http://{}:8042/api/v1'.format(os.environ['BOK_CHOY_HOSTNAME']) - -# Docker does not support the syslog socket at /dev/log. Rely on the console. -LOGGING['handlers']['local'] = LOGGING['handlers']['tracking'] = { - 'class': 'logging.NullHandler', -} - -LOGGING['loggers']['tracking']['handlers'] = ['console'] - -# Point the URL used to test YouTube availability to our stub YouTube server -BOK_CHOY_HOST = os.environ['BOK_CHOY_HOSTNAME'] -YOUTUBE['API'] = f"http://{BOK_CHOY_HOST}:{YOUTUBE_PORT}/get_youtube_api/" -YOUTUBE['METADATA_URL'] = f"http://{BOK_CHOY_HOST}:{YOUTUBE_PORT}/test_youtube/" diff --git a/cms/envs/bok_choy_docker.yml b/cms/envs/bok_choy_docker.yml deleted file mode 100644 index bc9727d7cd..0000000000 --- a/cms/envs/bok_choy_docker.yml +++ /dev/null @@ -1,147 +0,0 @@ -# ingested bok_choy_docker.env.json -# ingested bok_choy_docker.auth.json -AWS_ACCESS_KEY_ID: '' -AWS_SECRET_ACCESS_KEY: '' -BUGS_EMAIL: bugs@example.com -BULK_EMAIL_DEFAULT_FROM_EMAIL: no-reply@example.com -CACHES: - celery: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_celery - LOCATION: ['edx.devstack.memcached:11211'] - default: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: sandbox_default - LOCATION: ['edx.devstack.memcached:11211'] - general: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: sandbox_general - LOCATION: ['edx.devstack.memcached:11211'] - mongo_metadata_inheritance: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_mongo_metadata_inheritance - LOCATION: ['edx.devstack.memcached:11211'] - staticfiles: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_static_files - LOCATION: ['edx.devstack.memcached:11211'] -CELERY_ALWAYS_EAGER: true -CELERY_BROKER_HOSTNAME: localhost -CELERY_BROKER_PASSWORD: celery -CELERY_BROKER_TRANSPORT: amqp -CELERY_BROKER_USER: celery -CERT_QUEUE: certificates -CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION: false -CMS_BASE: '** OVERRIDDEN **' -CODE_JAIL: - limits: {REALTIME: 3, VMEM: 0} -COMMENTS_SERVICE_KEY: password -COMMENTS_SERVICE_URL: http://edx.devstack.studio:4567 -CONTACT_EMAIL: info@example.com -CONTENTSTORE: - DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [edx.devstack.mongo] - port: 27017 - ENGINE: xmodule.contentstore.mongo.MongoContentStore - OPTIONS: - db: test - host: [edx.devstack.mongo] - port: 27017 -DATABASES: - default: {ENGINE: django.db.backends.mysql, HOST: edx.devstack.mysql57, NAME: edxtest, - PASSWORD: '', PORT: '3306', USER: root} - student_module_history: {ENGINE: django.db.backends.mysql, HOST: edx.devstack.mysql57, - NAME: student_module_history_test, PASSWORD: '', PORT: '3306', USER: root} -DEFAULT_FEEDBACK_EMAIL: feedback@example.com -DEFAULT_FROM_EMAIL: registration@example.com -DJFS: {aws_access_key_id: test, aws_secret_access_key: test, bucket: test, prefix: test, - type: s3fs} -DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [edx.devstack.mongo] - port: 27017 -EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend -FEATURES: {CERTIFICATES_HTML_VIEW: true, CUSTOM_COURSES_EDX: true, - ENABLE_CONTENT_LIBRARIES: true, ENABLE_DISCUSSION_SERVICE: true, ENABLE_EXTENDED_COURSE_DETAILS: true, - ENABLE_GRADE_DOWNLOADS: true, ENABLE_SPECIAL_EXAMS: true, ENTRANCE_EXAMS: true, - MILESTONES_APP: true, PREVIEW_LMS_BASE: 'preview.localhost:8003', SHOW_HEADER_LANGUAGE_SELECTOR: true} -GITHUB_REPO_ROOT: '** OVERRIDDEN **' -GRADES_DOWNLOAD: {BUCKET: edx-grades, ROOT_PATH: /tmp/edx-s3/grades, STORAGE_TYPE: localfs} -JWT_AUTH: {JWT_PUBLIC_SIGNING_JWK_SET: '{"keys": [{"kid": - "BTZ9HA6K", "e": "AQAB", "kty": "RSA", "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ"}]}', - JWT_SECRET_KEY: super-secret-key} -LMS_BASE: '** OVERRIDDEN **' -LMS_ROOT_URL: '** OVERRIDDEN **' -LOCAL_LOGLEVEL: INFO -LOGGING_ENV: sandbox -LOG_DIR: '** OVERRIDDEN **' -MEDIA_URL: /media/ -MKTG_URL_LINK_MAP: {} -MODULESTORE: - default: - ENGINE: xmodule.modulestore.mixed.MixedModuleStore - OPTIONS: - mappings: {} - stores: - - DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [edx.devstack.mongo] - port: 27017 - ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore - NAME: draft - OPTIONS: - collection: modulestore - db: test - default_class: xmodule.hidden_block.HiddenBlock - fs_root: '** OVERRIDDEN **' - host: [edx.devstack.mongo] - port: 27017 - render_template: common.djangoapps.edxmako.shortcuts.render_to_string - - ENGINE: xmodule.modulestore.xml.XMLModuleStore - NAME: xml - OPTIONS: {data_dir: '** OVERRIDDEN **', default_class: xmodule.hidden_block.HiddenBlock} -SECRET_KEY: '' -SERVER_EMAIL: devops@example.com -SESSION_COOKIE_DOMAIN: null -SITE_NAME: localhost -SOCIAL_SHARING_SETTINGS: {CUSTOM_COURSE_URLS: true} -STATIC_URL_BASE: /static/ -SYSLOG_SERVER: '' -TECH_SUPPORT_EMAIL: technical@example.com -TIME_ZONE: America/New_York -WIKI_ENABLED: true -XQUEUE_INTERFACE: - basic_auth: [edx, edx] - django_auth: {password: password, username: lms} - url: http://localhost:18040 -ZENDESK_API_KEY: '' -ZENDESK_USER: '' diff --git a/common/djangoapps/terrain/stubs/lti.py b/common/djangoapps/terrain/stubs/lti.py index c406da5e6e..b3518f22cf 100644 --- a/common/djangoapps/terrain/stubs/lti.py +++ b/common/djangoapps/terrain/stubs/lti.py @@ -78,7 +78,7 @@ class StubLtiHandler(StubHttpRequestHandler): 'callback_url': self.post_dict.get('lis_outcome_service_url').replace('https', 'http'), 'sourcedId': self.post_dict.get('lis_result_sourcedid') } - host = os.environ.get('BOK_CHOY_HOSTNAME', self.server.server_address[0]) + host = os.environ.get(self.server.server_address[0]) submit_url = f'//{host}:{self.server.server_address[1]}' content = self._create_content(status_message, submit_url) self.send_response(200, content) @@ -296,7 +296,7 @@ class StubLtiHandler(StubHttpRequestHandler): """ client_secret = str(self.server.config.get('client_secret', self.DEFAULT_CLIENT_SECRET)) - host = os.environ.get('BOK_CHOY_HOSTNAME', '127.0.0.1') + host = os.environ.get('127.0.0.1') port = self.server.server_address[1] lti_base = self.DEFAULT_LTI_ADDRESS.format(host=host, port=port) lti_endpoint = self.server.config.get('lti_endpoint', self.DEFAULT_LTI_ENDPOINT) diff --git a/common/static/common/js/jasmine.common.conf.js b/common/static/common/js/jasmine.common.conf.js deleted file mode 100644 index d6d7f79773..0000000000 --- a/common/static/common/js/jasmine.common.conf.js +++ /dev/null @@ -1,12 +0,0 @@ -/* eslint-env node */ - -'use strict'; - -// By default, fixtures are loaded from spec/javascripts/fixtures but in karma everything gets served from /base -jasmine.getFixtures().fixturesPath = '/base/'; - -// https://github.com/edx/js-test-tool/blob/master/js_test_tool/templates/jasmine_test_runner.html#L10 -// Stub out modal dialog alerts, which will prevent -// us from accessing the test results in the DOM -window.confirm = function() { return true; }; -window.alert = function() { }; diff --git a/common/static/common/js/jasmine_stack_trace.js b/common/static/common/js/jasmine_stack_trace.js deleted file mode 100644 index f61d591f0a..0000000000 --- a/common/static/common/js/jasmine_stack_trace.js +++ /dev/null @@ -1,29 +0,0 @@ -/* This file overrides ExceptionFormatter of jasmine before it's initialization in karma-jasmine's - boot.js. It's important because ExceptionFormatter returns a constructor function. Once the method has been - initialized we can't override the ExceptionFormatter as Jasmine then uses the stored reference to the function */ -(function() { - /* globals jasmineRequire */ - - 'use strict'; - - var OldExceptionFormatter = jasmineRequire.ExceptionFormatter(), - oldExceptionFormatter = new OldExceptionFormatter(), - MAX_STACK_TRACE_LINES = 10; - - jasmineRequire.ExceptionFormatter = function() { - function ExceptionFormatter() { - this.message = oldExceptionFormatter.message; - this.stack = function(error) { - var errorMsg = null; - - if (error) { - errorMsg = error.stack.split('\n').slice(0, MAX_STACK_TRACE_LINES).join('\n'); - } - - return errorMsg; - }; - } - - return ExceptionFormatter; - }; -}()); diff --git a/common/static/common/js/karma.common.conf.js b/common/static/common/js/karma.common.conf.js deleted file mode 100644 index 7b92442aaf..0000000000 --- a/common/static/common/js/karma.common.conf.js +++ /dev/null @@ -1,467 +0,0 @@ -// Common settings and helpers for setting up Karma config. -// -// To run all the tests in a suite and print results to the console: -// -// karma start -// E.g. karma start lms/static/karma_lms.conf.js -// -// -// To run the tests for debugging: Debugging can be done in any browser -// but Chrome's developer console debugging experience is best. -// -// karma start --browsers=BROWSER --single-run=false -// -// -// To run the tests with coverage and junit reports: -// -// karma start --browsers=BROWSER -// --coverage --junitreportpath= --coveragereportpath= -// -// where `BROWSER` could be Chrome or Firefox. -// -// -// Troubleshooting tips: -// -// If you get an error like: "TypeError: __cov_KBCc7ZI4xZm8W2BC5NQLDg.s is undefined", -// that means the patterns in sourceFiles and specFiles are matching the same file. -// This causes Istanbul, which is used for tracking coverage to instrument the file -// multiple times. -// -// -// If you see the error: "EMFILE, too many open files" that means the files pattern -// that has been added is matching too many files. The glob library used by Karma -// does not use graceful-fs and tries to read files simultaneously. -// - -/* eslint-env node */ -/* globals process */ - -'use strict'; - -var path = require('path'); -var _ = require('underscore'); - -var appRoot = path.join(__dirname, '../../../../'); -// eslint-disable-next-line import/no-extraneous-dependencies -var webdriver = require('selenium-webdriver'); -// eslint-disable-next-line import/no-extraneous-dependencies -var firefox = require('selenium-webdriver/firefox'); - -var webpackConfig = require(path.join(appRoot, 'webpack.dev.config.js')); - -// The following crazy bit is to work around the webpack.optimize.CommonsChunkPlugin -// plugin. The problem is that it it factors out the code that defines webpackJsonp -// and puts in in the commons JS, which Karma doesn't know to load first. This is a -// workaround recommended in the karma-webpack bug report that basically just removes -// the plugin for the purposes of Karma testing (the plugin is meant to be an -// optimization only). -// https://github.com/webpack-contrib/karma-webpack/issues/24#issuecomment-257613167 -// -// This should be fixed in v3 of karma-webpack -var commonsChunkPluginIndex = webpackConfig[0].plugins.findIndex(function(plugin) { return plugin.chunkNames; }); - -// Files which are needed by all lms/cms suites. -var commonFiles = { - libraryFiles: [ - {pattern: 'common/js/vendor/**/*.js'}, - {pattern: 'edx-ui-toolkit/js/**/*.js'}, - {pattern: 'xmodule_js/common_static/common/js/**/!(*spec).js'}, - {pattern: 'xmodule_js/common_static/js/**/!(*spec).js'}, - {pattern: 'xmodule_js/src/**/*.js'} - ], - - sourceFiles: [ - {pattern: 'common/js/!(spec_helpers)/**/!(*spec).js'} - ], - - specFiles: [ - {pattern: 'common/js/spec_helpers/**/*.js'} - ], - - fixtureFiles: [ - {pattern: 'common/templates/**/*.underscore'} - ] -}; - -webpackConfig[0].plugins.splice(commonsChunkPluginIndex, 1); - -delete webpackConfig[0].entry; - -/** - * Customize the name attribute in xml testcase element - * @param {Object} browser - * @param {Object} result - * @return {String} - */ -function junitNameFormatter(browser, result) { - return result.suite[0] + ': ' + result.description; -} - -/** - * Customize the classname attribute in xml testcase element - * @param {Object} browser - * @return {String} - */ -function junitClassNameFormatter(browser) { - return 'Javascript.' + browser.name.split(' ')[0]; -} - -/** - * Return array containing default and user supplied reporters - * @param {Object} config - * @return {Array} - */ -function reporters(config) { - var defaultReporters = ['spec', 'junit', 'kjhtml']; - if (config.coverage) { - defaultReporters.push('coverage'); - } - return defaultReporters; -} - -/** - * Split a filepath into basepath and filename - * @param {String} filepath - * @return {Object} - */ -function getBasepathAndFilename(filepath) { - var file, dir; - - if (!filepath) { - // these will configure the reporters to create report files relative to this karma config file - return { - dir: undefined, - file: undefined - }; - } - file = filepath.replace(/^.*[\\/]/, ''); - dir = filepath.replace(file, ''); - - return { - dir: dir, - file: file - }; -} - -/** - * Return coverage reporter settings - * @param {String} config - * @return {Object} - */ -function coverageSettings(config) { - var pth = getBasepathAndFilename(config.coveragereportpath); - return { - dir: pth.dir, - subdir: '.', - includeAllSources: true, - reporters: [ - {type: 'cobertura', file: pth.file}, - {type: 'text-summary'} - ] - }; -} - -/** - * Return junit reporter settings - * @param {String} config - * @return {Object} - */ -function junitSettings(config) { - var pth = getBasepathAndFilename(config.junitreportpath); - return { - outputDir: pth.dir, - outputFile: pth.file, - suite: 'javascript', - useBrowserName: false, - nameFormatter: junitNameFormatter, - classNameFormatter: junitClassNameFormatter - }; -} - -/** - * Return absolute path for files in common and xmodule_js symlink dirs. - * @param {String} appRoot - * @param {String} pattern - * @return {String} - */ -// I'd like to fix the no-shadow violation on the next line, but it would break this shared conf's API. -function defaultNormalizeFunc(appRoot, pattern) { // eslint-disable-line no-shadow - var pat = pattern; - if (pat.match(/^common\/js/)) { - pat = path.join(appRoot, '/common/static/' + pat); - } else if (pat.match(/^xmodule_js\/common_static/)) { - pat = path.join(appRoot, '/common/static/' - + pat.replace(/^xmodule_js\/common_static\//, '')); - } - return pat; -} - -function normalizePathsForCoverage(files, normalizeFunc, preprocessors) { - var normalizeFn = normalizeFunc || defaultNormalizeFunc, - normalizedFile, - filesForCoverage = {}; - - files.forEach(function(file) { - if (!file.ignoreCoverage) { - normalizedFile = normalizeFn(appRoot, file.pattern); - if (preprocessors && preprocessors.hasOwnProperty(normalizedFile)) { - filesForCoverage[normalizedFile] = ['coverage'].concat(preprocessors[normalizedFile]); - } else { - filesForCoverage[normalizedFile] = ['coverage']; - } - } - }); - - return filesForCoverage; -} - -/** - * Sets defaults for each file pattern. - * RequireJS files are excluded by default. - * Webpack files are included by default. - * @param {Object} files - * @return {Object} - */ -function setDefaults(files) { - return files.map(function(f) { - var file = _.isObject(f) ? f : {pattern: f}; - if (!file.included && !file.webpack) { - file.included = false; - } - return file; - }); -} - -function getBaseConfig(config, useRequireJs) { - var getFrameworkFiles = function() { - var files = [ - 'common/static/common/js/vendor/jquery.js', - 'node_modules/jasmine-core/lib/jasmine-core/jasmine.js', - 'common/static/common/js/jasmine_stack_trace.js', - 'node_modules/karma-jasmine/lib/boot.js', - 'node_modules/karma-jasmine/lib/adapter.js', - 'node_modules/jasmine-jquery/lib/jasmine-jquery.js', - 'node_modules/popper.js/dist/umd/popper.js', - 'node_modules/bootstrap/dist/js/bootstrap.js', - 'node_modules/underscore/underscore.js', - 'node_modules/backbone/backbone.js', - 'common/static/js/test/i18n.js' - ]; - - if (useRequireJs) { - files = files.concat([ - 'node_modules/requirejs/require.js', - 'node_modules/karma-requirejs/lib/adapter.js' - ]); - } - - return files; - }; - - // Manually prepends the framework files to the karma files array - // bypassing the karma's framework config. This is necessary if you want - // to add a library or framework that isn't a karma plugin. e.g. we add jasmine-jquery - // which isn't a karma plugin. Though a karma framework for jasmine-jquery is available - // but it's not actively maintained. In future we also wanna add jQuery at the top when - // we upgrade to jQuery 2 - var initFrameworks = function(files) { - getFrameworkFiles().reverse().forEach(function(f) { - files.unshift({ - pattern: path.join(appRoot, f), - included: true, - served: true, - watch: false - }); - }); - }; - - var hostname = 'localhost'; - var port = 9876; - var customPlugin = { - 'framework:custom': ['factory', initFrameworks] - }; - - if (process.env.hasOwnProperty('BOK_CHOY_HOSTNAME')) { - hostname = process.env.BOK_CHOY_HOSTNAME; - if (hostname === 'edx.devstack.lms') { - port = 19876; - } else { - port = 19877; - } - } - - initFrameworks.$inject = ['config.files']; - - return { - // base path that will be used to resolve all patterns (eg. files, exclude) - basePath: '', - - // frameworks to use - // available frameworks: https://npmjs.org/browse/keyword/karma-adapter - frameworks: ['custom'], - - plugins: [ - 'karma-jasmine', - 'karma-jasmine-html-reporter', - 'karma-requirejs', - 'karma-junit-reporter', - 'karma-coverage', - 'karma-chrome-launcher', - 'karma-firefox-launcher', - 'karma-spec-reporter', - 'karma-selenium-webdriver-launcher', - 'karma-webpack', - 'karma-sourcemap-loader', - customPlugin - ], - - // list of files to exclude - exclude: [], - - // karma-reporter - reporters: reporters(config), - - // Spec Reporter configuration - specReporter: { - maxLogLines: 5, - showSpecTiming: true - }, - - coverageReporter: coverageSettings(config), - - junitReporter: junitSettings(config), - - // web server hostname and port - hostname: hostname, - port: port, - - // enable / disable colors in the output (reporters and logs) - colors: true, - - // level of logging - /* possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN - || config.LOG_INFO || config.LOG_DEBUG */ - logLevel: config.LOG_INFO, - - // enable / disable watching file and executing tests whenever any file changes - autoWatch: false, - - // start these browsers - // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher - browsers: ['FirefoxNoUpdates'], - - customLaunchers: { - // Firefox configuration that doesn't perform auto-updates - FirefoxNoUpdates: { - base: 'Firefox', - prefs: { - 'app.update.auto': false, - 'app.update.enabled': false - } - }, - ChromeDocker: { - base: 'SeleniumWebdriver', - browserName: 'chrome', - getDriver: function() { - return new webdriver.Builder() - .forBrowser('chrome') - .usingServer('http://edx.devstack.chrome:4444/wd/hub') - .build(); - } - }, - FirefoxDocker: { - base: 'SeleniumWebdriver', - browserName: 'firefox', - getDriver: function() { - var options = new firefox.Options(), - profile = new firefox.Profile(); - profile.setPreference('focusmanager.testmode', true); - options.setProfile(profile); - return new webdriver.Builder() - .forBrowser('firefox') - .usingServer('http://edx.devstack.firefox:4444/wd/hub') - .setFirefoxOptions(options) - .build(); - } - } - }, - - // Continuous Integration mode - // if true, Karma captures browsers, runs the tests and exits - singleRun: config.singleRun, - - // Concurrency level - // how many browser should be started simultaneous - concurrency: Infinity, - - browserNoActivityTimeout: 50000, - - client: { - captureConsole: false - }, - - webpack: webpackConfig[0], - - webpackMiddleware: { - watchOptions: { - poll: true - } - } - }; -} - -function configure(config, options) { - var useRequireJs = options.useRequireJs === undefined ? true : options.useRequireJs, - baseConfig = getBaseConfig(config, useRequireJs), - files, filesForCoverage, preprocessors; - - if (options.includeCommonFiles) { - _.forEach(['libraryFiles', 'sourceFiles', 'specFiles', 'fixtureFiles'], function(collectionName) { - options[collectionName] = _.flatten([commonFiles[collectionName], options[collectionName]]); - }); - } - - files = _.flatten( - _.map( - ['libraryFilesToInclude', 'libraryFiles', 'sourceFiles', 'specFiles', 'fixtureFiles', 'runFiles'], - function(collectionName) { return options[collectionName] || []; } - ) - ); - - files.unshift( - {pattern: path.join(appRoot, 'common/static/common/js/jasmine.common.conf.js'), included: true} - ); - - if (useRequireJs) { - files.unshift({pattern: 'common/js/utils/require-serial.js', included: true}); - } - - // Karma sets included=true by default. - // We set it to false by default because RequireJS should be used instead. - files = setDefaults(files); - - filesForCoverage = _.flatten( - _.map( - ['sourceFiles', 'specFiles'], - function(collectionName) { return options[collectionName]; } - ) - ); - - // If we give symlink paths to Istanbul, coverage for each path gets tracked - // separately. So we pass absolute paths to the karma-coverage preprocessor. - preprocessors = _.extend( - {}, - options.preprocessors, - normalizePathsForCoverage(filesForCoverage, options.normalizePathsForCoverageFunc, options.preprocessors) - ); - - config.set(_.extend(baseConfig, { - files: files, - preprocessors: preprocessors - })); -} - -module.exports = { - configure: configure, - appRoot: appRoot -}; diff --git a/docs/decisions/0004-managing-django-settings.rst b/docs/decisions/0004-managing-django-settings.rst index 97c15b6cf4..bf5698af39 100644 --- a/docs/decisions/0004-managing-django-settings.rst +++ b/docs/decisions/0004-managing-django-settings.rst @@ -58,8 +58,6 @@ and for all environments to use __init__.py to load their settings. The following files should be obviated by this change: -* bok_choy_docker.py -* bok_choy.py * devstack_docker.py * devstack_optimized.py * devstack.py diff --git a/lms/envs/bok_choy.py b/lms/envs/bok_choy.py deleted file mode 100644 index 9a39d4d8bb..0000000000 --- a/lms/envs/bok_choy.py +++ /dev/null @@ -1,154 +0,0 @@ -""" -Settings for Bok Choy tests that are used when running LMS. - -Bok Choy uses two different settings files: -1. test_static_optimized is used when invoking collectstatic -2. bok_choy is used when running the tests - -Note: it isn't possible to have a single settings file, because Django doesn't -support both generating static assets to a directory and also serving static -from the same directory. -""" - - -# Silence noisy logs -import logging -import os -from tempfile import mkdtemp - -from django.utils.translation import gettext_lazy -from path import Path as path - -from openedx.core.release import RELEASE_LINE -from xmodule.modulestore.modulestore_settings import update_module_store_settings # lint-amnesty, pylint: disable=wrong-import-order - -CONFIG_ROOT = path(__file__).abspath().dirname() -TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root" - -########################## Prod-like settings ################################### -# These should be as close as possible to the settings we use in production. -# As in prod, we read in environment and auth variables from JSON files. -# Unlike in prod, we use the JSON files stored in this repo. -# This is a convenience for ensuring (a) that we can consistently find the files -# and (b) that the files are the same in Jenkins as in local dev. -os.environ['SERVICE_VARIANT'] = 'bok_choy_docker' if 'BOK_CHOY_HOSTNAME' in os.environ else 'bok_choy' -os.environ['LMS_CFG'] = str.format("{config_root}/{service_variant}.yml", - config_root=CONFIG_ROOT, service_variant=os.environ['SERVICE_VARIANT']) -os.environ['REVISION_CFG'] = f"{CONFIG_ROOT}/revisions.yml" - -from .production import * # pylint: disable=wildcard-import, unused-wildcard-import, wrong-import-position - - -######################### Testing overrides #################################### - -# Redirect to the test_root folder within the repo -GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath() -LOG_DIR = (TEST_ROOT / "log").abspath() - -# Configure modulestore to use the test folder within the repo -update_module_store_settings( - MODULESTORE, - module_store_options={ - 'fs_root': (TEST_ROOT / "data").abspath(), - }, - xml_store_options={ - 'data_dir': (TEST_ROOT / "data").abspath(), - }, - default_store=os.environ.get('DEFAULT_STORE', 'draft'), -) - -PLATFORM_NAME = gettext_lazy("édX") -PLATFORM_DESCRIPTION = gettext_lazy("Open édX Platform") - -############################ STATIC FILES ############################# - -# Serve static files at /static directly from the staticfiles directory under test root -# Note: optimized files for testing are generated with settings from test_static_optimized -STATIC_URL = "/static/" -STATICFILES_FINDERS = ['django.contrib.staticfiles.finders.FileSystemFinder'] -STATICFILES_DIRS = [ - (TEST_ROOT / "staticfiles" / "lms").abspath(), -] - -DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' -MEDIA_ROOT = TEST_ROOT / "uploads" - -# Webpack loader must use webpack output setting -WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = TEST_ROOT / "staticfiles" / "lms" / "webpack-stats.json" - -# Don't use compression during tests -PIPELINE['JS_COMPRESSOR'] = None - -###################### Grades ###################### -GRADES_DOWNLOAD = { - 'STORAGE_TYPE': 'localfs', - 'BUCKET': 'edx-grades', - 'ROOT_PATH': os.path.join(mkdtemp(), 'edx-s3', 'grades'), -} - - -LOG_OVERRIDES = [ - ('track.middleware', logging.CRITICAL), - ('common.djangoapps.edxmako.shortcuts', logging.ERROR), - ('edx.discussion', logging.CRITICAL), -] -for log_name, log_level in LOG_OVERRIDES: - logging.getLogger(log_name).setLevel(log_level) - - -YOUTUBE_HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', '127.0.0.1') -# Point the URL used to test YouTube availability to our stub YouTube server -YOUTUBE_PORT = 9080 -YOUTUBE['TEST_TIMEOUT'] = 5000 -YOUTUBE['API'] = f"http://{YOUTUBE_HOSTNAME}:{YOUTUBE_PORT}/get_youtube_api/" -YOUTUBE['METADATA_URL'] = f"http://{YOUTUBE_HOSTNAME}:{YOUTUBE_PORT}/test_youtube/" - -############################# SECURITY SETTINGS ################################ -# Default to advanced security in common.py, so tests can reset here to use -# a simpler security model - -# Path at which to store the mock index -MOCK_SEARCH_BACKING_FILE = ( - TEST_ROOT / "index_file.dat" -).abspath() - -# Verify student settings -VERIFY_STUDENT["SOFTWARE_SECURE"] = { - "API_ACCESS_KEY": "BBBBBBBBBBBBBBBBBBBB", - "API_SECRET_KEY": "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC", -} - -# Set dummy values for profile image settings. -PROFILE_IMAGE_BACKEND = { - 'class': 'openedx.core.storage.OverwriteStorage', - 'options': { - 'location': os.path.join(MEDIA_ROOT, 'profile-images/'), - 'base_url': os.path.join(MEDIA_URL, 'profile-images/'), - }, -} - -LMS_ROOT_URL = "http://localhost:{}".format(os.environ.get('BOK_CHOY_LMS_PORT', 8003)) -CMS_BASE = "localhost:{}".format(os.environ.get('BOK_CHOY_CMS_PORT', 8031)) -LOGIN_REDIRECT_WHITELIST = [CMS_BASE] - -INSTALLED_APPS.append('openedx.testing.coverage_context_listener') - -if RELEASE_LINE == "master": - # On master, acceptance tests use edX books, not the default Open edX books. - HELP_TOKENS_BOOKS = { - 'learner': 'https://edx.readthedocs.io/projects/edx-guide-for-students', - 'course_author': 'https://edx.readthedocs.io/projects/edx-partner-course-staff', - } - -# API access management -API_ACCESS_MANAGER_EMAIL = 'api-access@example.com' -API_ACCESS_FROM_EMAIL = 'api-requests@example.com' -API_DOCUMENTATION_URL = 'https://course-catalog-api-guide.readthedocs.io/en/latest/' -AUTH_DOCUMENTATION_URL = 'https://course-catalog-api-guide.readthedocs.io/en/latest/authentication/index.html' - -##################################################################### -# Lastly, see if the developer has any local overrides. -try: - from .private import * # pylint: disable=wildcard-import -except ImportError: - pass diff --git a/lms/envs/bok_choy.yml b/lms/envs/bok_choy.yml deleted file mode 100644 index 0468867eb7..0000000000 --- a/lms/envs/bok_choy.yml +++ /dev/null @@ -1,274 +0,0 @@ -# ingested edx-platform/lms/envs/bok_choy.auth.json -# ingested edx-platform/lms/envs/bok_choy.env.json -ACTIVATION_EMAIL_SUPPORT_LINK: https://support.example.com/activation-email-help.html -ANALYTICS_DASHBOARD_URL: '' -AWS_ACCESS_KEY_ID: '' -AWS_SECRET_ACCESS_KEY: '' -BUGS_EMAIL: bugs@example.com -BULK_EMAIL_DEFAULT_FROM_EMAIL: no-reply@example.com -BADGING_BACKEND: 'lms.djangoapps.badges.backends.tests.dummy_backend.DummyBackend' -BLOCK_STRUCTURES_SETTINGS: - # We have CELERY_ALWAYS_EAGER set to True, so there's no asynchronous - # code running and the celery routing is unimportant. - # It does not make sense to retry. - TASK_MAX_RETRIES: 0 - # course publish task delay is irrelevant is because the task is run synchronously - COURSE_PUBLISH_TASK_DELAY: 0 - # retry delay is irrelevent because we never retry - TASK_DEFAULT_RETRY_DELAY: 0 - -CACHES: - celery: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_celery - LOCATION: ['localhost:11211'] - default: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: sandbox_default - LOCATION: ['localhost:11211'] - general: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: sandbox_general - LOCATION: ['localhost:11211'] - mongo_metadata_inheritance: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_mongo_metadata_inheritance - LOCATION: ['localhost:11211'] - staticfiles: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_static_files - LOCATION: ['localhost:11211'] - -# Capture the console log via template includes, until webdriver supports log capture again -CAPTURE_CONSOLE_LOG: True - -CELERY_BROKER_HOSTNAME: localhost -CELERY_BROKER_PASSWORD: celery -CELERY_BROKER_TRANSPORT: amqp -CELERY_BROKER_USER: celery -CELERY_ALWAYS_EAGER: True -CELERY_RESULT_BACKEND: 'django-cache' -CLEAR_REQUEST_CACHE_ON_TASK_COMPLETION: False - -CERT_QUEUE: certificates -CMS_BASE: localhost:8031 -CODE_JAIL: - limits: {REALTIME: 3, VMEM: 0} -COMMENTS_SERVICE_KEY: password -COMMENTS_SERVICE_URL: http://localhost:4567 -COMPLETION_BY_VIEWING_DELAY_MS: 1000 -CONTACT_EMAIL: info@example.com -CONTENTSTORE: - DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [localhost] - port: 27017 - ENGINE: xmodule.contentstore.mongo.MongoContentStore - OPTIONS: - db: test - host: [localhost] - port: 27017 -DATABASES: - default: {ENGINE: django.db.backends.mysql, HOST: localhost, NAME: edxtest, PASSWORD: '', - PORT: '3306', USER: root} - student_module_history: {ENGINE: django.db.backends.mysql, HOST: localhost, NAME: student_module_history_test, - PASSWORD: '', PORT: '3306', USER: root} -DEFAULT_FEEDBACK_EMAIL: feedback@example.com -DEFAULT_FROM_EMAIL: registration@example.com - -# Enable debug so that static assets are served by Django -DEBUG: True -DJFS: {aws_access_key_id: test, aws_secret_access_key: test, bucket: test, prefix: test, - type: s3fs} -DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [localhost] - port: 27017 -# Configure the LMS to use our stub eCommerce implementation -ECOMMERCE_API_URL: 'http://localhost:8043/api/v2/' -# Configure the LMS to use our stub EdxNotes implementation -EDXNOTES_PUBLIC_API: 'http://localhost:8042/api/v1' -EDXNOTES_INTERNAL_API: 'http://localhost:8042/api/v1' - -EDXNOTES_CONNECT_TIMEOUT: 10 # time in seconds -EDXNOTES_READ_TIMEOUT: 10 # time in seconds - -NOTES_DISABLED_TABS: [] - -EMAIL_BACKEND: django.core.mail.backends.dummy.EmailBackend -EVENT_TRACKING_BACKENDS: - mongo: - ENGINE: eventtracking.backends.mongodb.MongoBackend - OPTIONS: {collection: events, database: test} -FEATURES: - ALLOW_AUTOMATED_SIGNUPS: true - AUTOMATIC_AUTH_FOR_TESTING: true - AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING: true - CERTIFICATES_HTML_VIEW: true - CERTIFICATES_INSTRUCTOR_GENERATION: true - CUSTOM_COURSES_EDX: true, - ENABLE_COURSE_DISCOVERY: true - ENABLE_DISCUSSION_SERVICE: true - ENABLE_GRADE_DOWNLOADS: true - ENABLE_SPECIAL_EXAMS: true - ENABLE_THIRD_PARTY_AUTH: true - ENABLE_VERIFIED_CERTIFICATES: true - EXPOSE_CACHE_PROGRAMS_ENDPOINT: true - MODE_CREATION_FOR_TESTING: true - PREVIEW_LMS_BASE: 'preview.localhost:8003' - RESTRICT_AUTOMATIC_AUTH: false - SHOW_HEADER_LANGUAGE_SELECTOR: true - ENABLE_MAX_FAILED_LOGIN_ATTEMPTS: False - SQUELCH_PII_IN_LOGS: False - PREVENT_CONCURRENT_LOGINS: False - ENABLE_MOBILE_REST_API: True # Show video bumper in LMS - ENABLE_VIDEO_BUMPER: True # Show video bumper in LMS - SHOW_BUMPER_PERIODICITY: 1 - # Enable courseware search for tests - ENABLE_COURSEWARE_SEARCH: True - # Enable dashboard search for tests - ENABLE_DASHBOARD_SEARCH: True - # discussion home panel, which includes a subscription on/off setting for discussion digest emails. - ENABLE_DISCUSSION_HOME_PANEL: True - # Enable support for OpenBadges accomplishments - ENABLE_OPENBADGES: True - ENABLE_LTI_PROVIDER: True - # Enable milestones app - MILESTONES_APP: True - # Enable oauth authentication, which we test. - ENABLE_OAUTH2_PROVIDER: True - OAUTH_ENFORCE_SECURE: False - ENABLE_PREREQUISITE_COURSES: True - ENABLE_COURSE_DISCOVERY: True - ENABLE_EDXNOTES: True - ENABLE_TEAMS: True - LICENSING: True - # Use the auto_auth workflow for creating users and logging them in - AUTOMATIC_AUTH_FOR_TESTING: True - RESTRICT_AUTOMATIC_AUTH: False - # Open up endpoint for faking Software Secure responses - ENABLE_SOFTWARE_SECURE_FAKE: True - # Disable instructor dash buttons for downloading course data when enrollment exceeds this number - MAX_ENROLLMENT_INSTR_BUTTONS: 4 - ENABLE_ENROLLMENT_TRACK_USER_PARTITION: True - ENTRANCE_EXAMS: True - ENABLE_SPECIAL_EXAMS: True - - -GITHUB_REPO_ROOT: '** OVERRIDDEN **' -JWT_AUTH: {JWT_PRIVATE_SIGNING_JWK: '{"e": "AQAB", "d": "HIiV7KNjcdhVbpn3KT-I9n3JPf5YbGXsCIedmPqDH1d4QhBofuAqZ9zebQuxkRUpmqtYMv0Zi6ECSUqH387GYQF_XvFUFcjQRPycISd8TH0DAKaDpGr-AYNshnKiEtQpINhcP44I1AYNPCwyoxXA1fGTtmkKChsuWea7o8kytwU5xSejvh5-jiqu2SF4GEl0BEXIAPZsgbzoPIWNxgO4_RzNnWs6nJZeszcaDD0CyezVSuH9QcI6g5QFzAC_YuykSsaaFJhZ05DocBsLczShJ9Omf6PnK9xlm26I84xrEh_7x4fVmNBg3xWTLh8qOnHqGko93A1diLRCrKHOvnpvgQ", - "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ", - "q": "3T3DEtBUka7hLGdIsDlC96Uadx_q_E4Vb1cxx_4Ss_wGp1Loz3N3ZngGyInsKlmbBgLo1Ykd6T9TRvRNEWEtFSOcm2INIBoVoXk7W5RuPa8Cgq2tjQj9ziGQ08JMejrPlj3Q1wmALJr5VTfvSYBu0WkljhKNCy1KB6fCby0C9WE", - "p": "vUqzWPZnDG4IXyo-k5F0bHV0BNL_pVhQoLW7eyFHnw74IOEfSbdsMspNcPSFIrtgPsn7981qv3lN_staZ6JflKfHayjB_lvltHyZxfl0dvruShZOx1N6ykEo7YrAskC_qxUyrIvqmJ64zPW3jkuOYrFs7Ykj3zFx3Zq1H5568G0", - "kid": "BTZ9HA6K", "kty": "RSA"}', JWT_PUBLIC_SIGNING_JWK_SET: '{"keys": [{"kid": - "BTZ9HA6K", "e": "AQAB", "kty": "RSA", "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ"}]}', - JWT_SECRET_KEY: super-secret-key} -LMS_BASE: localhost:8003 -LMS_ROOT_URL: http://localhost:8003 -LOCAL_LOGLEVEL: INFO -LOGGING_ENV: sandbox -LOG_DIR: '** OVERRIDDEN **' -MEDIA_URL: /media/ -MKTG_URL_LINK_MAP: {ABOUT: about, BLOG: blog, CAREERS: careers, CONTACT: contact, - COURSES: courses, DONATE: donate, HELP_CENTER: help-center, HONOR: honor, NEWS: news, - PRESS: press, PRIVACY: privacy, ROOT: root, SITEMAP.XML: sitemap_xml, TOS: tos, - WHAT_IS_VERIFIED_CERT: verified-certificate} -MODULESTORE: - default: - ENGINE: xmodule.modulestore.mixed.MixedModuleStore - OPTIONS: - mappings: {} - stores: - - DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [localhost] - port: 27017 - ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore - NAME: draft - OPTIONS: - collection: modulestore - db: test - default_class: xmodule.hidden_block.HiddenBlock - fs_root: '** OVERRIDDEN **' - host: [localhost] - port: 27017 - render_template: common.djangoapps.edxmako.shortcuts.render_to_string - - ENGINE: xmodule.modulestore.xml.XMLModuleStore - NAME: xml - OPTIONS: {data_dir: '** OVERRIDDEN **', default_class: xmodule.hidden_block.HiddenBlock} -# We need to test different scenarios, following setting effectively disbale rate limiting -PASSWORD_RESET_IP_RATE: '1/s' -PASSWORD_RESET_EMAIL_RATE: '1/s' -PASSWORD_RESET_SUPPORT_LINK: https://support.example.com/password-reset-help.html -REGISTRATION_EXTENSION_FORM: openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm -REGISTRATION_EXTRA_FIELDS: {city: hidden, country: required, gender: optional, goals: optional, - honor_code: required, level_of_education: optional, mailing_address: optional, terms_of_service: hidden, - year_of_birth: optional} -# Use MockSearchEngine as the search engine for test scenario -SEARCH_ENGINE: "search.tests.mock_search_engine.MockSearchEngine" -# this secret key should be the same as cms/envs/bok_choy.py's -SECRET_KEY: "very_secret_bok_choy_key" - -SERVER_EMAIL: devops@example.com -SESSION_COOKIE_DOMAIN: null -SITE_NAME: localhost:8003 -SOCIAL_SHARING_SETTINGS: {CERTIFICATE_FACEBOOK: true, CERTIFICATE_FACEBOOK_TEXT: 'Testing - facebook feature:', CUSTOM_COURSE_URLS: true, DASHBOARD_FACEBOOK: true, DASHBOARD_TWITTER: true, - DASHBOARD_TWITTER_TEXT: 'Testing feature:'} -STATIC_URL_BASE: /static/ -SUPPORT_SITE_LINK: https://support.example.com -SYSLOG_SERVER: '' -TECH_SUPPORT_EMAIL: technical@example.com -THIRD_PARTY_AUTH_BACKENDS: [social_core.backends.google.GoogleOAuth2, social_core.backends.linkedin.LinkedinOAuth2, - social_core.backends.facebook.FacebookOAuth2, common.djangoapps.third_party_auth.dummy.DummyBackend, - common.djangoapps.third_party_auth.saml.SAMLAuthBackend] -THIRD_PARTY_AUTH: - Google: - SOCIAL_AUTH_GOOGLE_OAUTH2_KEY": "test" - SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET": "test" - Facebook: - SOCIAL_AUTH_FACEBOOK_KEY": "test" - SOCIAL_AUTH_FACEBOOK_SECRET": "test" -TIME_ZONE: America/New_York -TRACKING_BACKENDS: - mongo: - ENGINE: common.djangoapps.track.backends.mongodb.MongoBackend - OPTIONS: {collection: events, database: test} -WIKI_ENABLED: true -WAFFLE_OVERRIDE: True -XQUEUE_INTERFACE: - basic_auth: [edx, edx] - django_auth: {password: password, username: lms} - # Configure the LMS to use our stub XQueue implementation - url: 'http://localhost:8040' - -ZENDESK_API_KEY: '' -ZENDESK_USER: '' diff --git a/lms/envs/bok_choy_docker.py b/lms/envs/bok_choy_docker.py deleted file mode 100644 index 991b22e373..0000000000 --- a/lms/envs/bok_choy_docker.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Settings for Bok Choy tests that are used when running Studio in Docker-based devstack. -""" - -# noinspection PyUnresolvedReferences -from .bok_choy import * # pylint: disable=wildcard-import - -CMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ.get('BOK_CHOY_CMS_PORT', 8031)) -LMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ.get('BOK_CHOY_LMS_PORT', 8003)) -LMS_ROOT_URL = f'http://{LMS_BASE}' -LOGIN_REDIRECT_WHITELIST = [CMS_BASE] -SITE_NAME = LMS_BASE - -COMMENTS_SERVICE_URL = 'http://{}:4567'.format(os.environ['BOK_CHOY_HOSTNAME']) -EDXNOTES_PUBLIC_API = 'http://{}:8042/api/v1'.format(os.environ['BOK_CHOY_HOSTNAME']) - -# Docker does not support the syslog socket at /dev/log. Rely on the console. -LOGGING['handlers']['local'] = LOGGING['handlers']['tracking'] = { - 'class': 'logging.NullHandler', -} - -LOGGING['loggers']['tracking']['handlers'] = ['console'] - -# Point the URL used to test YouTube availability to our stub YouTube server -BOK_CHOY_HOST = os.environ['BOK_CHOY_HOSTNAME'] -YOUTUBE['API'] = f"http://{BOK_CHOY_HOST}:{YOUTUBE_PORT}/get_youtube_api/" -YOUTUBE['METADATA_URL'] = f"http://{BOK_CHOY_HOST}:{YOUTUBE_PORT}/test_youtube/" diff --git a/lms/envs/bok_choy_docker.yml b/lms/envs/bok_choy_docker.yml deleted file mode 100644 index afef10bb5d..0000000000 --- a/lms/envs/bok_choy_docker.yml +++ /dev/null @@ -1,183 +0,0 @@ -# ingested edx-platform/lms/envs/bok_choy_docker.auth.json -# ingested edx-platform/lms/envs/bok_choy_docker.env.json -ACTIVATION_EMAIL_SUPPORT_LINK: https://support.example.com/activation-email-help.html -ANALYTICS_DASHBOARD_URL: '' -AWS_ACCESS_KEY_ID: '' -AWS_SECRET_ACCESS_KEY: '' -BUGS_EMAIL: bugs@example.com -BULK_EMAIL_DEFAULT_FROM_EMAIL: no-reply@example.com -CACHES: - celery: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_celery - LOCATION: ['edx.devstack.memcached:11211'] - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - default: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: sandbox_default - LOCATION: ['edx.devstack.memcached:11211'] - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - general: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: sandbox_general - LOCATION: ['edx.devstack.memcached:11211'] - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - mongo_metadata_inheritance: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_mongo_metadata_inheritance - LOCATION: ['edx.devstack.memcached:11211'] - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - staticfiles: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_static_files - LOCATION: ['edx.devstack.memcached:11211'] - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true -CELERY_BROKER_HOSTNAME: localhost -CELERY_BROKER_PASSWORD: celery -CELERY_BROKER_TRANSPORT: amqp -CELERY_BROKER_USER: celery -CERT_QUEUE: certificates -CMS_BASE: '** OVERRIDDEN **' -CODE_JAIL: - limits: {REALTIME: 3, VMEM: 0} -COMMENTS_SERVICE_KEY: password -COMMENTS_SERVICE_URL: http://edx.devstack.lms:4567 -CONTACT_EMAIL: info@example.com -CONTENTSTORE: - DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [edx.devstack.mongo] - port: 27017 - ENGINE: xmodule.contentstore.mongo.MongoContentStore - OPTIONS: - db: test - host: [edx.devstack.mongo] - port: 27017 -DATABASES: - default: {ENGINE: django.db.backends.mysql, HOST: edx.devstack.mysql57, NAME: edxtest, - PASSWORD: '', PORT: '3306', USER: root} - student_module_history: {ENGINE: django.db.backends.mysql, HOST: edx.devstack.mysql57, - NAME: student_module_history_test, PASSWORD: '', PORT: '3306', USER: root} -DEFAULT_FEEDBACK_EMAIL: feedback@example.com -DEFAULT_FROM_EMAIL: registration@example.com -DJFS: {aws_access_key_id: test, aws_secret_access_key: test, bucket: test, prefix: test, - type: s3fs} -DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [edx.devstack.mongo] - port: 27017 -EMAIL_BACKEND: django.core.mail.backends.dummy.EmailBackend -EVENT_TRACKING_BACKENDS: - mongo: - ENGINE: eventtracking.backends.mongodb.MongoBackend - OPTIONS: - collection: events - database: test - host: [edx.devstack.mongo] - port: 27017 -FEATURES: {ALLOW_AUTOMATED_SIGNUPS: true, AUTOMATIC_AUTH_FOR_TESTING: true, - AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING: true, CERTIFICATES_HTML_VIEW: true, - CERTIFICATES_INSTRUCTOR_GENERATION: true, CUSTOM_COURSES_EDX: true, - ENABLE_COURSE_DISCOVERY: true, ENABLE_DISCUSSION_SERVICE: true, ENABLE_GRADE_DOWNLOADS: true, - ENABLE_SPECIAL_EXAMS: true, ENABLE_THIRD_PARTY_AUTH: true, - ENABLE_VERIFIED_CERTIFICATES: true, EXPOSE_CACHE_PROGRAMS_ENDPOINT: true, MODE_CREATION_FOR_TESTING: true, - PREVIEW_LMS_BASE: 'preview.localhost:8003', RESTRICT_AUTOMATIC_AUTH: false, SHOW_HEADER_LANGUAGE_SELECTOR: true} -GITHUB_REPO_ROOT: '** OVERRIDDEN **' -JWT_AUTH: {JWT_PRIVATE_SIGNING_JWK: '{"e": "AQAB", "d": "HIiV7KNjcdhVbpn3KT-I9n3JPf5YbGXsCIedmPqDH1d4QhBofuAqZ9zebQuxkRUpmqtYMv0Zi6ECSUqH387GYQF_XvFUFcjQRPycISd8TH0DAKaDpGr-AYNshnKiEtQpINhcP44I1AYNPCwyoxXA1fGTtmkKChsuWea7o8kytwU5xSejvh5-jiqu2SF4GEl0BEXIAPZsgbzoPIWNxgO4_RzNnWs6nJZeszcaDD0CyezVSuH9QcI6g5QFzAC_YuykSsaaFJhZ05DocBsLczShJ9Omf6PnK9xlm26I84xrEh_7x4fVmNBg3xWTLh8qOnHqGko93A1diLRCrKHOvnpvgQ", - "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ", - "q": "3T3DEtBUka7hLGdIsDlC96Uadx_q_E4Vb1cxx_4Ss_wGp1Loz3N3ZngGyInsKlmbBgLo1Ykd6T9TRvRNEWEtFSOcm2INIBoVoXk7W5RuPa8Cgq2tjQj9ziGQ08JMejrPlj3Q1wmALJr5VTfvSYBu0WkljhKNCy1KB6fCby0C9WE", - "p": "vUqzWPZnDG4IXyo-k5F0bHV0BNL_pVhQoLW7eyFHnw74IOEfSbdsMspNcPSFIrtgPsn7981qv3lN_staZ6JflKfHayjB_lvltHyZxfl0dvruShZOx1N6ykEo7YrAskC_qxUyrIvqmJ64zPW3jkuOYrFs7Ykj3zFx3Zq1H5568G0", - "kid": "BTZ9HA6K", "kty": "RSA"}', JWT_PUBLIC_SIGNING_JWK_SET: '{"keys": [{"kid": - "BTZ9HA6K", "e": "AQAB", "kty": "RSA", "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ"}]}', - JWT_SECRET_KEY: super-secret-key} -LMS_BASE: http://edx.devstack.lms:18003 -LMS_ROOT_URL: http://edx.devstack.lms:18003 -LOCAL_LOGLEVEL: INFO -LOGGING_ENV: sandbox -LOG_DIR: '** OVERRIDDEN **' -MEDIA_URL: /media/ -MKTG_URL_LINK_MAP: {ABOUT: about, BLOG: blog, CAREERS: careers, CONTACT: contact, - COURSES: courses, DONATE: donate, HELP_CENTER: help-center, HONOR: honor, NEWS: news, - PRESS: press, PRIVACY: privacy, ROOT: root, SITEMAP.XML: sitemap_xml, TOS: tos, - WHAT_IS_VERIFIED_CERT: verified-certificate} -MODULESTORE: - default: - ENGINE: xmodule.modulestore.mixed.MixedModuleStore - OPTIONS: - mappings: {} - stores: - - DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [edx.devstack.mongo] - port: 27017 - ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore - NAME: draft - OPTIONS: - collection: modulestore - db: test - default_class: xmodule.hidden_block.HiddenBlock - fs_root: '** OVERRIDDEN **' - host: [edx.devstack.mongo] - port: 27017 - render_template: common.djangoapps.edxmako.shortcuts.render_to_string - - ENGINE: xmodule.modulestore.xml.XMLModuleStore - NAME: xml - OPTIONS: {data_dir: '** OVERRIDDEN **', default_class: xmodule.hidden_block.HiddenBlock} -PASSWORD_RESET_SUPPORT_LINK: https://support.example.com/password-reset-help.html -REGISTRATION_EXTENSION_FORM: openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm -REGISTRATION_EXTRA_FIELDS: {city: hidden, country: required, gender: optional, goals: optional, - honor_code: required, level_of_education: optional, mailing_address: optional, terms_of_service: hidden, - year_of_birth: optional} -SECRET_KEY: 'bokchoy_docker_secret_key' -SERVER_EMAIL: devops@example.com -SESSION_COOKIE_DOMAIN: null -SITE_NAME: localhost:8003 -SOCIAL_SHARING_SETTINGS: {CERTIFICATE_FACEBOOK: true, CERTIFICATE_FACEBOOK_TEXT: 'Testing - facebook feature:', CUSTOM_COURSE_URLS: true, DASHBOARD_FACEBOOK: true, DASHBOARD_TWITTER: true, - DASHBOARD_TWITTER_TEXT: 'Testing feature:'} -STATIC_URL_BASE: /static/ -SUPPORT_SITE_LINK: https://support.example.com -SYSLOG_SERVER: '' -TECH_SUPPORT_EMAIL: technical@example.com -THIRD_PARTY_AUTH_BACKENDS: [social_core.backends.google.GoogleOAuth2, social_core.backends.linkedin.LinkedinOAuth2, - social_core.backends.facebook.FacebookOAuth2, common.djangoapps.third_party_auth.dummy.DummyBackend, - common.djangoapps.third_party_auth.saml.SAMLAuthBackend] -TIME_ZONE: America/New_York -TRACKING_BACKENDS: - mongo: - ENGINE: common.djangoapps.track.backends.mongodb.MongoBackend - OPTIONS: - collection: events - database: test - host: [edx.devstack.mongo] - port: 27017 -WIKI_ENABLED: true -XQUEUE_INTERFACE: - basic_auth: [edx, edx] - django_auth: {password: password, username: lms} - url: '** OVERRIDDEN **' -ZENDESK_API_KEY: '' -ZENDESK_USER: '' diff --git a/openedx/core/djangoapps/util/management/commands/update_fixtures.py b/openedx/core/djangoapps/util/management/commands/update_fixtures.py deleted file mode 100644 index 53e7db56ea..0000000000 --- a/openedx/core/djangoapps/util/management/commands/update_fixtures.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Django management command to update the loaded test fixtures as necessary for -the current test environment. Currently just sets an appropriate domain for -each Site fixture. -""" - - -import os - -from django.contrib.sites.models import Site -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - """ - update_fixtures management command - """ - - help = "Update fixtures to match the current test environment." - - def handle(self, *args, **options): - if 'BOK_CHOY_HOSTNAME' in os.environ: - # Fix the Site fixture domains so third party auth tests work correctly - host = os.environ['BOK_CHOY_HOSTNAME'] - cms_port = os.environ['BOK_CHOY_CMS_PORT'] - lms_port = os.environ['BOK_CHOY_LMS_PORT'] - cms_domain = f'{host}:{cms_port}' - Site.objects.filter(name='cms').update(domain=cms_domain) - lms_domain = f'{host}:{lms_port}' - Site.objects.filter(name='lms').update(domain=lms_domain) diff --git a/openedx/core/djangoapps/util/tests/test_update_fixtures.py b/openedx/core/djangoapps/util/tests/test_update_fixtures.py deleted file mode 100644 index fb43fa34a8..0000000000 --- a/openedx/core/djangoapps/util/tests/test_update_fixtures.py +++ /dev/null @@ -1,42 +0,0 @@ -""" # lint-amnesty, pylint: disable=django-not-configured -Tests of the update_fixtures management command for bok-choy test database -initialization. -""" - - -import os - -import pytest -from django.contrib.sites.models import Site -from django.core.management import call_command - - -@pytest.fixture(scope='function') -def sites(db): # lint-amnesty, pylint: disable=unused-argument - Site.objects.create(name='cms', domain='localhost:8031') - Site.objects.create(name='lms', domain='localhost:8003') - - -def test_localhost(db, monkeypatch, sites): # lint-amnesty, pylint: disable=redefined-outer-name, unused-argument - monkeypatch.delitem(os.environ, 'BOK_CHOY_HOSTNAME', raising=False) - call_command('update_fixtures') - assert Site.objects.get(name='cms').domain == 'localhost:8031' - assert Site.objects.get(name='lms').domain == 'localhost:8003' - - -def test_devstack_cms(db, monkeypatch, sites): # lint-amnesty, pylint: disable=redefined-outer-name, unused-argument - monkeypatch.setitem(os.environ, 'BOK_CHOY_HOSTNAME', 'edx.devstack.cms') - monkeypatch.setitem(os.environ, 'BOK_CHOY_CMS_PORT', '18031') - monkeypatch.setitem(os.environ, 'BOK_CHOY_LMS_PORT', '18003') - call_command('update_fixtures') - assert Site.objects.get(name='cms').domain == 'edx.devstack.cms:18031' - assert Site.objects.get(name='lms').domain == 'edx.devstack.cms:18003' - - -def test_devstack_lms(db, monkeypatch, sites): # lint-amnesty, pylint: disable=redefined-outer-name, unused-argument - monkeypatch.setitem(os.environ, 'BOK_CHOY_HOSTNAME', 'edx.devstack.lms') - monkeypatch.setitem(os.environ, 'BOK_CHOY_CMS_PORT', '18031') - monkeypatch.setitem(os.environ, 'BOK_CHOY_LMS_PORT', '18003') - call_command('update_fixtures') - assert Site.objects.get(name='cms').domain == 'edx.devstack.lms:18031' - assert Site.objects.get(name='lms').domain == 'edx.devstack.lms:18003' diff --git a/openedx/features/content_type_gating/tests/test_access.py b/openedx/features/content_type_gating/tests/test_access.py index c66359fc92..53582df34a 100644 --- a/openedx/features/content_type_gating/tests/test_access.py +++ b/openedx/features/content_type_gating/tests/test_access.py @@ -228,8 +228,7 @@ class TestProblemTypeAccess(SharedModuleStoreTestCase, MasqueradeMixin): # pyli graded=False, ) cls.graded_score_weight_blocks[(graded, has_score, weight)] = block - - host = os.environ.get('BOK_CHOY_HOSTNAME', '127.0.0.1') + host = os.environ.get('127.0.0.1') metadata_lti_xblock = { 'lti_id': 'correct_lti_id', 'launch_url': 'http://{}:{}/{}'.format(host, '8765', 'correct_lti_endpoint'), diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py index cdafa7df81..e4530e76f5 100644 --- a/openedx/testing/coverage_context_listener/pytest_plugin.py +++ b/openedx/testing/coverage_context_listener/pytest_plugin.py @@ -3,9 +3,9 @@ A pytest plugin that reports test contexts to coverage running in another proces """ import pytest -import requests +# import requests -from pavelib.utils.envs import Env +# from pavelib.utils.envs import Env class RemoteContextPlugin: @@ -25,16 +25,17 @@ class RemoteContextPlugin: def pytest_runtest_call(self, item): self.doit(item, "call") - def doit(self, item, when): # lint-amnesty, pylint: disable=missing-function-docstring - if self.active: - for cfg in Env.BOK_CHOY_SERVERS.values(): - result = requests.post( - 'http://{host}:{port}/coverage_context/update_context'.format(**cfg), - { - 'context': f"{item.nodeid}|{when}", - } - ) - assert result.status_code == 204 + # commented for testing + # def doit(self, item, when): # lint-amnesty, pylint: disable=missing-function-docstring + # if self.active: + # for cfg in Env.BOK_CHOY_SERVERS.values(): + # result = requests.post( + # 'http://{host}:{port}/coverage_context/update_context'.format(**cfg), + # { + # 'context': f"{item.nodeid}|{when}", + # } + # ) + # assert result.status_code == 204 @pytest.hookimpl(tryfirst=True) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 9e2e2fe5b8..fe7cd0964d 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -61,18 +61,6 @@ class Env: # Python unittest dirs PYTHON_COVERAGERC = REPO_ROOT / ".coveragerc" - # Bok_choy dirs - BOK_CHOY_DIR = REPO_ROOT / "common" / "test" / "acceptance" - BOK_CHOY_LOG_DIR = GEN_LOG_DIR - BOK_CHOY_REPORT_DIR = REPORT_DIR / "bok_choy" - BOK_CHOY_A11Y_REPORT_DIR = REPORT_DIR / "a11y" - BOK_CHOY_COVERAGERC = BOK_CHOY_DIR / ".coveragerc" - BOK_CHOY_A11Y_COVERAGERC = BOK_CHOY_DIR / ".a11ycoveragerc" - BOK_CHOY_A11Y_CUSTOM_RULES_FILE = ( - REPO_ROOT / "node_modules" / "edx-custom-a11y-rules" / - "lib" / "custom_a11y_rules.js" - ) - # Which Python version should be used in xdist workers? PYTHON_VERSION = os.environ.get("PYTHON_VERSION", "2.7") @@ -82,95 +70,18 @@ class Env: # build steps. For local development/testing, this shouldn't be needed. if os.environ.get("SHARD", None): shard_str = "shard_{}".format(os.environ.get("SHARD")) - BOK_CHOY_REPORT_DIR = BOK_CHOY_REPORT_DIR / shard_str - BOK_CHOY_LOG_DIR = BOK_CHOY_LOG_DIR / shard_str - - # The stubs package is currently located in the Django app called "terrain" - # from when they were used by both the bok-choy and lettuce (deprecated) acceptance tests - BOK_CHOY_STUB_DIR = REPO_ROOT / "common" / "djangoapps" / "terrain" # Directory that videos are served from VIDEO_SOURCE_DIR = REPO_ROOT / "test_root" / "data" / "video" - PRINT_SETTINGS_LOG_FILE = BOK_CHOY_LOG_DIR / "print_settings.log" + PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" # Detect if in a Docker container, and if so which one - SERVER_HOST = os.environ.get('BOK_CHOY_HOSTNAME', '0.0.0.0') + SERVER_HOST = os.environ.get('0.0.0.0') USING_DOCKER = SERVER_HOST != '0.0.0.0' - SETTINGS = 'bok_choy_docker' if USING_DOCKER else 'bok_choy' DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' TEST_SETTINGS = 'test' - BOK_CHOY_SERVERS = { - 'lms': { - 'host': SERVER_HOST, - 'port': os.environ.get('BOK_CHOY_LMS_PORT', '8003'), - 'log': BOK_CHOY_LOG_DIR / "bok_choy_lms.log" - }, - 'cms': { - 'host': SERVER_HOST, - 'port': os.environ.get('BOK_CHOY_CMS_PORT', '8031'), - 'log': BOK_CHOY_LOG_DIR / "bok_choy_studio.log" - } - } - - BOK_CHOY_STUBS = { - - 'xqueue': { - 'port': 8040, - 'log': BOK_CHOY_LOG_DIR / "bok_choy_xqueue.log", - 'config': 'register_submission_url=http://0.0.0.0:8041/test/register_submission', - }, - - 'ora': { - 'port': 8041, - 'log': BOK_CHOY_LOG_DIR / "bok_choy_ora.log", - 'config': '', - }, - - 'comments': { - 'port': 4567, - 'log': BOK_CHOY_LOG_DIR / "bok_choy_comments.log", - }, - - 'video': { - 'port': 8777, - 'log': BOK_CHOY_LOG_DIR / "bok_choy_video_sources.log", - 'config': f"root_dir={VIDEO_SOURCE_DIR}", - }, - - 'youtube': { - 'port': 9080, - 'log': BOK_CHOY_LOG_DIR / "bok_choy_youtube.log", - }, - - 'edxnotes': { - 'port': 8042, - 'log': BOK_CHOY_LOG_DIR / "bok_choy_edxnotes.log", - }, - - 'ecommerce': { - 'port': 8043, - 'log': BOK_CHOY_LOG_DIR / "bok_choy_ecommerce.log", - }, - - 'catalog': { - 'port': 8091, - 'log': BOK_CHOY_LOG_DIR / "bok_choy_catalog.log", - }, - - 'lti': { - 'port': 8765, - 'log': BOK_CHOY_LOG_DIR / "bok_choy_lti.log", - }, - } - - # Mongo databases that will be dropped before/after the tests run - MONGO_HOST = 'edx.devstack.mongo' if USING_DOCKER else 'localhost' - BOK_CHOY_MONGO_DATABASE = "test" - BOK_CHOY_CACHE_HOST = 'edx.devstack.memcached' if USING_DOCKER else '0.0.0.0' - BOK_CHOY_CACHE = memcache.Client([f'{BOK_CHOY_CACHE_HOST}:11211'], debug=0) - # Test Ids Directory TEST_DIR = REPO_ROOT / ".testids" diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 7c5173ccaa..3b1c6ce672 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -138,8 +138,6 @@ bleach[css]==6.0.0 # ora2 # xblock-drag-and-drop-v2 # xblock-poll -bok-choy==2.0.2 - # via -r requirements/edx/testing.txt boto==2.39.0 # via # -c requirements/edx/../constraints.txt @@ -1122,7 +1120,6 @@ lazy==1.6 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # acid-xblock - # bok-choy # lti-consumer-xblock # ora2 # xblock @@ -1809,10 +1806,6 @@ scipy==1.7.3 # -r requirements/edx/testing.txt # chem # openedx-calc -selenium==3.141.0 - # via - # -r requirements/edx/testing.txt - # bok-choy semantic-version==2.10.0 # via # -r requirements/edx/doc.txt @@ -2121,7 +2114,6 @@ urllib3==1.26.16 # pact-python # py2neo # requests - # selenium # snowflake-connector-python user-util==1.0.0 # via diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index 93310adc88..b7806b027e 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -18,7 +18,6 @@ -r coverage.txt # Utilities for calculating test coverage beautifulsoup4 # Library for extracting data from HTML and XML files -bok-choy # Framework for browser automation tests, based on selenium code-annotations # Perform code annotation checking, such as for PII annotations cssselect # Used to extract HTML fragments via CSS selectors in 2 test cases and pyquery ddt # Run a test case multiple times with different input; used in many, many of our tests @@ -41,7 +40,6 @@ pytest-json-report # Output json formatted warnings after running pytest pytest-metadata==1.8.0 # To prevent 'make upgrade' failure, dependency of pytest-json-report pytest-randomly # pytest plugin to randomly order tests pytest-xdist[psutil] # Parallel execution of tests on multiple CPU cores or hosts -selenium # Browser automation library, used for acceptance tests singledispatch # Backport of functools.singledispatch from Python 3.4+, used in tests of XBlock rendering testfixtures # Provides a LogCapture utility used by several tests tox # virtualenv management for tests diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 2a07c19de0..dd51467f15 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -104,8 +104,6 @@ bleach[css]==6.0.0 # ora2 # xblock-drag-and-drop-v2 # xblock-poll -bok-choy==2.0.2 - # via -r requirements/edx/testing.in boto==2.39.0 # via # -c requirements/edx/../constraints.txt @@ -856,7 +854,6 @@ lazy==1.6 # via # -r requirements/edx/base.txt # acid-xblock - # bok-choy # lti-consumer-xblock # ora2 # xblock @@ -1373,10 +1370,6 @@ scipy==1.7.3 # -r requirements/edx/base.txt # chem # openedx-calc -selenium==3.141.0 - # via - # -r requirements/edx/testing.in - # bok-choy semantic-version==2.10.0 # via # -r requirements/edx/base.txt @@ -1568,7 +1561,6 @@ urllib3==1.26.16 # pact-python # py2neo # requests - # selenium # snowflake-connector-python user-util==1.0.0 # via -r requirements/edx/base.txt diff --git a/scripts/reset-test-db.sh b/scripts/reset-test-db.sh deleted file mode 100755 index b0573a8e13..0000000000 --- a/scripts/reset-test-db.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash - -############################################################################ -# -# reset-test-db.sh -# -# Resets the MySQL test database for the bok-choy acceptance tests. -# -# If it finds a cached schema and migration history, it will start -# from the cached version to speed up migrations. -# -# If no cached database exists, it will create one. This can be -# checked into the repo to speed up future tests. -# -# Note that we do NOT want to re-use the cache between test runs! -# A newer commit could introduce migrations that do not exist -# in other commits, which could cause migrations to fail in the other -# commits. -# -# For this reason, we always use a cache that was committed to master -# at the time the branch was created. -# -############################################################################ - -# Fail fast -set -e - -DB_CACHE_DIR="common/test/db_cache" - -if [[ -z "$BOK_CHOY_HOSTNAME" ]]; then - MYSQL_HOST="" - SETTINGS="bok_choy" -else - MYSQL_HOST="--host=edx.devstack.mysql57" - SETTINGS="bok_choy_docker" -fi - -for i in "$@"; do - case $i in - -r|--rebuild_cache) - REBUILD_CACHE=true - ;; - -m|--migrations) - APPLY_MIGRATIONS=true - ;; - -c|--calculate_migrations) - CALCULATE_MIGRATIONS=true - ;; - -u|--use-existing-db) - USE_EXISTING_DB=true - ;; - esac -done - -declare -A databases -declare -a database_order -databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test") -database_order=("default" "student_module_history") - -calculate_migrations() { - echo "Calculating migrations for fingerprinting." - output_file="common/test/db_cache/bok_choy_${db}_migrations.yaml" - # Redirect stdout to /dev/null because the script will print - # out all migrations to both stdout and the output file. - ./manage.py lms --settings "$SETTINGS" show_unapplied_migrations --database "$db" --output_file "$output_file" 1>/dev/null -} - -run_migrations() { - echo "Running the lms migrations on the $db bok_choy DB." - ./manage.py lms --settings "$SETTINGS" migrate --database "$db" --traceback --noinput - echo "Running the cms migrations on the $db bok_choy DB." - ./manage.py cms --settings "$SETTINGS" migrate --database "$db" --traceback --noinput -} - -load_cache_into_db() { - echo "Loading the schema from the filesystem into the $db MySQL DB." - mysql "$MYSQL_HOST" -u root "${databases["$db"]}" < "$DB_CACHE_DIR/bok_choy_schema_$db.sql" - echo "Loading the fixture data from the filesystem into the $db MySQL DB." - ./manage.py lms --settings "$SETTINGS" loaddata --database "$db" "$DB_CACHE_DIR/bok_choy_data_$db.json" - echo "Loading the migration data from the filesystem into the $db MySQL DB." - mysql "$MYSQL_HOST" -u root "${databases["$db"]}" < "$DB_CACHE_DIR/bok_choy_migrations_data_$db.sql" -} - -rebuild_cache_for_db() { - # Make sure the DB has all migrations applied - run_migrations - - # Dump the schema and data to the cache - echo "Using the dumpdata command to save the $db fixture data to the filesystem." - ./manage.py lms --settings "$SETTINGS" dumpdata --database "$db" > "$DB_CACHE_DIR/bok_choy_data_$db.json" --exclude=api_admin.Catalog - echo "Saving the schema of the $db bok_choy DB to the filesystem." - mysqldump "$MYSQL_HOST" -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > "$DB_CACHE_DIR/bok_choy_schema_$db.sql" - - # dump_data does not dump the django_migrations table so we do it separately. - echo "Saving the django_migrations table of the $db bok_choy DB to the filesystem." - mysqldump $MYSQL_HOST -u root --no-create-info --skip-comments --skip-dump-date "${databases["$db"]}" django_migrations > "$DB_CACHE_DIR/bok_choy_migrations_data_$db.sql" -} - -for db in "${database_order[@]}"; do - if ! [[ $USE_EXISTING_DB ]]; then - echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql $MYSQL_HOST -u root - - # Clear out the test database - # - # We are using the reset_db command which uses "DROP DATABASE" and - # "CREATE DATABASE" in case the tests are being run in an environment (e.g. devstack - # or a jenkins worker environment) that already ran tests on another commit that had - # different migrations that created, dropped, or altered tables. - echo "Issuing a reset_db command to the $db bok_choy MySQL database." - ./manage.py lms --settings "$SETTINGS" reset_db --traceback --router "$db" - fi - - if ! [[ $CALCULATE_MIGRATIONS ]]; then - # If there are cached database schemas/data, then load them. - # If they are missing, then we will want to build new cache files even if - # not explicitly directed to do so via arguments passed to this script. - if [[ ! -f $DB_CACHE_DIR/bok_choy_schema_$db.sql || ! -f $DB_CACHE_DIR/bok_choy_data_$db.json || ! -f $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql ]]; then - REBUILD_CACHE=true - else - load_cache_into_db - fi - fi -done - -if [[ $REBUILD_CACHE ]]; then - echo "Cleaning the DB cache directory and building new files." - mkdir -p $DB_CACHE_DIR && rm -f $DB_CACHE_DIR/bok_choy* - - for db in "${database_order[@]}"; do - rebuild_cache_for_db - done -elif [[ $APPLY_MIGRATIONS ]]; then - for db in "${database_order[@]}"; do - run_migrations - done -elif [[ $CALCULATE_MIGRATIONS ]]; then - for db in "${database_order[@]}"; do - calculate_migrations - done -fi diff --git a/tox.ini b/tox.ini index 6894076506..ce00efa870 100644 --- a/tox.ini +++ b/tox.ini @@ -26,10 +26,6 @@ setenv = PYTHONHASHSEED=0 TOXENV={envname} passenv = - BOK_CHOY_CMS_PORT - BOKCHOY_HEADLESS - BOK_CHOY_HOSTNAME - BOK_CHOY_LMS_PORT DISABLE_COURSEENROLLMENT_HISTORY DISPLAY DJANGO_SETTINGS_MODULE diff --git a/xmodule/modulestore/tests/mongo_connection.py b/xmodule/modulestore/tests/mongo_connection.py index 8da1fbd423..a0ffe7e7c6 100644 --- a/xmodule/modulestore/tests/mongo_connection.py +++ b/xmodule/modulestore/tests/mongo_connection.py @@ -9,5 +9,4 @@ overrides don't exist import os MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) -MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', - 'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost') +MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost') diff --git a/xmodule/tests/test_mongo_utils.py b/xmodule/tests/test_mongo_utils.py index 94182b24a7..cfa08ba157 100644 --- a/xmodule/tests/test_mongo_utils.py +++ b/xmodule/tests/test_mongo_utils.py @@ -3,7 +3,6 @@ Tests for methods defined in mongo_utils.py """ -import os from unittest import TestCase from uuid import uuid4 @@ -29,7 +28,7 @@ class MongoUtilsTests(TestCase): """ Test that read_preference parameter gets converted to a valid pymongo read preference. """ - host = 'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost' + host = 'localhost' db = 'test_read_preference_%s' % uuid4().hex # Support for read_preference given in constant name form (ie. PRIMARY, SECONDARY_PREFERRED) connection = connect_to_mongodb(db, host, read_preference=enum_name) diff --git a/xmodule/tests/test_video.py b/xmodule/tests/test_video.py index 7daa6bdfe0..e5180843d7 100644 --- a/xmodule/tests/test_video.py +++ b/xmodule/tests/test_video.py @@ -15,7 +15,6 @@ the course, section, subsection, unit, etc. import datetime import json -import os import shutil import unittest from tempfile import mkdtemp @@ -937,7 +936,7 @@ class VideoBlockStudentViewDataTestCase(unittest.TestCase): @patch.object(settings, 'CONTENTSTORE', create=True, new={ 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', 'DOC_STORE_CONFIG': { - 'host': 'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost', + 'host': 'localhost', 'db': 'test_xcontent_%s' % uuid4().hex, }, # allow for additional options that can be keyed on a name, e.g. 'trashcan' From 73c4e110fee4b780588bf68352d5776cacff86e5 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 27 Sep 2023 11:27:41 +0500 Subject: [PATCH 02/35] chore: remove karma --- pavelib/utils/envs.py | 30 +++++++++++++++--------------- tox.ini | 6 +++--- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index fe7cd0964d..ca7ec0f4c6 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -86,25 +86,25 @@ class Env: TEST_DIR = REPO_ROOT / ".testids" # Configured browser to use for the js test suites - SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') - if USING_DOCKER: - KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' - else: - KARMA_BROWSER = 'FirefoxNoUpdates' + # SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') + # if USING_DOCKER: + # KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' + # else: + # KARMA_BROWSER = 'FirefoxNoUpdates' # Files used to run each of the js test suites # TODO: Store this as a dict. Order seems to matter for some # reason. See issue TE-415. - KARMA_CONFIG_FILES = [ - REPO_ROOT / 'cms/static/karma_cms.conf.js', - REPO_ROOT / 'cms/static/karma_cms_squire.conf.js', - REPO_ROOT / 'cms/static/karma_cms_webpack.conf.js', - REPO_ROOT / 'lms/static/karma_lms.conf.js', - REPO_ROOT / 'xmodule/js/karma_xmodule.conf.js', - REPO_ROOT / 'xmodule/js/karma_xmodule_webpack.conf.js', - REPO_ROOT / 'common/static/karma_common.conf.js', - REPO_ROOT / 'common/static/karma_common_requirejs.conf.js', - ] + # KARMA_CONFIG_FILES = [ + # REPO_ROOT / 'cms/static/karma_cms.conf.js', + # REPO_ROOT / 'cms/static/karma_cms_squire.conf.js', + # REPO_ROOT / 'cms/static/karma_cms_webpack.conf.js', + # REPO_ROOT / 'lms/static/karma_lms.conf.js', + # REPO_ROOT / 'xmodule/js/karma_xmodule.conf.js', + # REPO_ROOT / 'xmodule/js/karma_xmodule_webpack.conf.js', + # REPO_ROOT / 'common/static/karma_common.conf.js', + # REPO_ROOT / 'common/static/karma_common_requirejs.conf.js', + # ] JS_TEST_ID_KEYS = [ 'cms', diff --git a/tox.ini b/tox.ini index ce00efa870..cacd77ea0b 100644 --- a/tox.ini +++ b/tox.ini @@ -40,9 +40,9 @@ passenv = NPM_CONFIG_PREFIX PYTHON_VERSION SCRAPY_SETTINGS_MODULE - SELENIUM_BROWSER - SELENIUM_HOST - SELENIUM_PORT + #SELENIUM_BROWSER + #SELENIUM_HOST + #SELENIUM_PORT SHARD SKIP_NPM_INSTALL SSH_AUTH_SOCK From a06ac2f6a2869db7dc859b2c825029e09fccb370 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 27 Sep 2023 11:42:57 +0500 Subject: [PATCH 03/35] chore: remove karma --- pavelib/paver_tests/test_js_test.py | 120 +++++++++++++------------- pavelib/utils/test/suites/js_suite.py | 86 +++++++++--------- 2 files changed, 103 insertions(+), 103 deletions(-) diff --git a/pavelib/paver_tests/test_js_test.py b/pavelib/paver_tests/test_js_test.py index 9d89a94444..a066a3ea19 100644 --- a/pavelib/paver_tests/test_js_test.py +++ b/pavelib/paver_tests/test_js_test.py @@ -55,15 +55,15 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): ["--suite=lms"], ["--suite=lms --coverage"], ) - @ddt.unpack - def test_test_js_run(self, options_string): - """ - Test the "test_js_run" task. - """ - options = self.parse_options_string(options_string) - self.reset_task_messages() - call_task("pavelib.js_test.test_js_run", options=options) - self.verify_messages(options=options, dev_mode=False) + # @ddt.unpack + # def test_test_js_run(self, options_string): + # """ + # Test the "test_js_run" task. + # """ + # options = self.parse_options_string(options_string) + # self.reset_task_messages() + # call_task("pavelib.js_test.test_js_run", options=options) + # self.verify_messages(options=options, dev_mode=False) @ddt.data( [""], @@ -71,15 +71,15 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): ["--suite=lms"], ["--suite=lms --port=9999"], ) - @ddt.unpack - def test_test_js_dev(self, options_string): - """ - Test the "test_js_run" task. - """ - options = self.parse_options_string(options_string) - self.reset_task_messages() - call_task("pavelib.js_test.test_js_dev", options=options) - self.verify_messages(options=options, dev_mode=True) + # @ddt.unpack + # def test_test_js_dev(self, options_string): + # """ + # Test the "test_js_run" task. + # """ + # options = self.parse_options_string(options_string) + # self.reset_task_messages() + # call_task("pavelib.js_test.test_js_dev", options=options) + # self.verify_messages(options=options, dev_mode=True) def parse_options_string(self, options_string): """ @@ -101,50 +101,50 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): "port": port, } - def verify_messages(self, options, dev_mode): - """ - Verify that the messages generated when running tests are as expected - for the specified options and dev_mode. - """ - is_coverage = options['coverage'] - port = options['port'] - expected_messages = [] - suites = Env.JS_TEST_ID_KEYS if options['suite'] == 'all' else [options['suite']] + # def verify_messages(self, options, dev_mode): + # """ + # Verify that the messages generated when running tests are as expected + # for the specified options and dev_mode. + # """ + # is_coverage = options['coverage'] + # port = options['port'] + # expected_messages = [] + # suites = Env.JS_TEST_ID_KEYS if options['suite'] == 'all' else [options['suite']] - expected_messages.extend(self.EXPECTED_COMMANDS) - if not dev_mode and not is_coverage: - expected_messages.append(self.EXPECTED_DELETE_JAVASCRIPT_REPORT_COMMAND.format( - platform_root=self.platform_root - )) - expected_messages.append(self.EXPECTED_INSTALL_NPM_ASSETS_COMMAND) + # expected_messages.extend(self.EXPECTED_COMMANDS) + # if not dev_mode and not is_coverage: + # expected_messages.append(self.EXPECTED_DELETE_JAVASCRIPT_REPORT_COMMAND.format( + # platform_root=self.platform_root + # )) + # expected_messages.append(self.EXPECTED_INSTALL_NPM_ASSETS_COMMAND) - command_template = ( - 'node --max_old_space_size=4096 node_modules/.bin/karma start {options}' - ) + # command_template = ( + # 'node --max_old_space_size=4096 node_modules/.bin/karma start {options}' + # ) - for suite in suites: - # Karma test command - if suite != 'jest-snapshot': - karma_config_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(suite)] - expected_test_tool_command = command_template.format( - options=self.EXPECTED_KARMA_OPTIONS.format( - config_file=karma_config_file, - single_run='false' if dev_mode else 'true', - suite=suite, - platform_root=self.platform_root, - browser=Env.KARMA_BROWSER, - ), - ) - if is_coverage: - expected_test_tool_command += self.EXPECTED_COVERAGE_OPTIONS.format( - platform_root=self.platform_root, - suite=suite - ) - if port: - expected_test_tool_command += f" --port={port}" - else: - expected_test_tool_command = 'jest' + # for suite in suites: + # # Karma test command + # if suite != 'jest-snapshot': + # karma_config_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(suite)] + # expected_test_tool_command = command_template.format( + # options=self.EXPECTED_KARMA_OPTIONS.format( + # config_file=karma_config_file, + # single_run='false' if dev_mode else 'true', + # suite=suite, + # platform_root=self.platform_root, + # browser=Env.KARMA_BROWSER, + # ), + # ) + # if is_coverage: + # expected_test_tool_command += self.EXPECTED_COVERAGE_OPTIONS.format( + # platform_root=self.platform_root, + # suite=suite + # ) + # if port: + # expected_test_tool_command += f" --port={port}" + # else: + # expected_test_tool_command = 'jest' - expected_messages.append(expected_test_tool_command) + # expected_messages.append(expected_test_tool_command) - assert self.task_messages == expected_messages + # assert self.task_messages == expected_messages diff --git a/pavelib/utils/test/suites/js_suite.py b/pavelib/utils/test/suites/js_suite.py index 65c5feaf84..b7099c591f 100644 --- a/pavelib/utils/test/suites/js_suite.py +++ b/pavelib/utils/test/suites/js_suite.py @@ -47,55 +47,55 @@ class JsTestSuite(TestSuite): return [JsTestSubSuite(test_id, **self.opts) for test_id in Env.JS_TEST_ID_KEYS if test_id != 'jest-snapshot'] -class JsTestSubSuite(TestSuite): - """ - Class for JS suites like cms, cms-squire, lms, common, - common-requirejs and xmodule - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.test_id = args[0] - self.run_under_coverage = kwargs.get('with_coverage', True) - self.mode = kwargs.get('mode', 'run') - self.port = kwargs.get('port') - self.root = self.root + ' javascript' - self.report_dir = Env.JS_REPORT_DIR +# class JsTestSubSuite(TestSuite): +# """ +# Class for JS suites like cms, cms-squire, lms, common, +# common-requirejs and xmodule +# """ +# def __init__(self, *args, **kwargs): +# super().__init__(*args, **kwargs) +# self.test_id = args[0] +# self.run_under_coverage = kwargs.get('with_coverage', True) +# self.mode = kwargs.get('mode', 'run') +# self.port = kwargs.get('port') +# self.root = self.root + ' javascript' +# self.report_dir = Env.JS_REPORT_DIR - try: - self.test_conf_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(self.test_id)] - except ValueError: - self.test_conf_file = Env.KARMA_CONFIG_FILES[0] +# try: +# self.test_conf_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(self.test_id)] +# except ValueError: +# self.test_conf_file = Env.KARMA_CONFIG_FILES[0] - self.coverage_report = self.report_dir / f'coverage-{self.test_id}.xml' - self.xunit_report = self.report_dir / f'javascript_xunit-{self.test_id}.xml' +# self.coverage_report = self.report_dir / f'coverage-{self.test_id}.xml' +# self.xunit_report = self.report_dir / f'javascript_xunit-{self.test_id}.xml' - @property - def cmd(self): - """ - Run the tests using karma runner. - """ - cmd = [ - "node", - "--max_old_space_size=4096", - "node_modules/.bin/karma", - "start", - self.test_conf_file, - "--single-run={}".format('false' if self.mode == 'dev' else 'true'), - "--capture-timeout=60000", - f"--junitreportpath={self.xunit_report}", - f"--browsers={Env.KARMA_BROWSER}", - ] +# @property +# def cmd(self): +# """ +# Run the tests using karma runner. +# """ +# cmd = [ +# "node", +# "--max_old_space_size=4096", +# "node_modules/.bin/karma", +# "start", +# self.test_conf_file, +# "--single-run={}".format('false' if self.mode == 'dev' else 'true'), +# "--capture-timeout=60000", +# f"--junitreportpath={self.xunit_report}", +# f"--browsers={Env.KARMA_BROWSER}", +# ] - if self.port: - cmd.append(f"--port={self.port}") +# if self.port: +# cmd.append(f"--port={self.port}") - if self.run_under_coverage: - cmd.extend([ - "--coverage", - f"--coveragereportpath={self.coverage_report}", - ]) +# if self.run_under_coverage: +# cmd.extend([ +# "--coverage", +# f"--coveragereportpath={self.coverage_report}", +# ]) - return cmd +# return cmd class JestSnapshotTestSuite(TestSuite): From caeb8d9694998ce5d0bd9dc058bf8ec060a56fef Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 27 Sep 2023 16:11:11 +0500 Subject: [PATCH 04/35] chore: remove bok choy --- pavelib/utils/test/suites/js_suite.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pavelib/utils/test/suites/js_suite.py b/pavelib/utils/test/suites/js_suite.py index b7099c591f..40f491f5d6 100644 --- a/pavelib/utils/test/suites/js_suite.py +++ b/pavelib/utils/test/suites/js_suite.py @@ -25,7 +25,7 @@ class JsTestSuite(TestSuite): self.opts = kwargs suite = args[0] - self.subsuites = self._default_subsuites if suite == 'all' else [JsTestSubSuite(*args, **kwargs)] + # self.subsuites = self._default_subsuites if suite == 'all' else [JsTestSubSuite(*args, **kwargs)] def __enter__(self): super().__enter__() @@ -39,12 +39,12 @@ class JsTestSuite(TestSuite): if self.mode == 'run' and not self.run_under_coverage: test_utils.clean_dir(self.report_dir) - @property - def _default_subsuites(self): - """ - Returns all JS test suites - """ - return [JsTestSubSuite(test_id, **self.opts) for test_id in Env.JS_TEST_ID_KEYS if test_id != 'jest-snapshot'] + # @property + # def _default_subsuites(self): + # """ + # Returns all JS test suites + # """ + # return [JsTestSubSuite(test_id, **self.opts) for test_id in Env.JS_TEST_ID_KEYS if test_id != 'jest-snapshot'] # class JsTestSubSuite(TestSuite): From 3abf5971526f564d56cc90ec9feeafdbb374b74b Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 27 Sep 2023 16:57:53 +0500 Subject: [PATCH 05/35] chore: fix test cases --- .github/workflows/js-tests.yml | 8 --- .../pytest_plugin.py | 17 +---- pavelib/paver_tests/test_js_test.py | 70 +------------------ 3 files changed, 3 insertions(+), 92 deletions(-) diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 092e6e38e2..e4903af83b 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -30,14 +30,6 @@ jobs: - name: Setup npm run: npm i -g npm@8.5.x - # - name: Install Firefox 61.0 - # run: | - # sudo apt-get purge firefox - # wget "https://ftp.mozilla.org/pub/firefox/releases/61.0/linux-x86_64/en-US/firefox-61.0.tar.bz2" - # tar -xjf firefox-61.0.tar.bz2 - # sudo mv firefox /opt/firefox - # sudo ln -s /opt/firefox/firefox /usr/bin/firefox - - name: Install Required System Packages run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py index e4530e76f5..37051385c0 100644 --- a/openedx/testing/coverage_context_listener/pytest_plugin.py +++ b/openedx/testing/coverage_context_listener/pytest_plugin.py @@ -3,9 +3,6 @@ A pytest plugin that reports test contexts to coverage running in another proces """ import pytest -# import requests - -# from pavelib.utils.envs import Env class RemoteContextPlugin: @@ -25,19 +22,7 @@ class RemoteContextPlugin: def pytest_runtest_call(self, item): self.doit(item, "call") - # commented for testing - # def doit(self, item, when): # lint-amnesty, pylint: disable=missing-function-docstring - # if self.active: - # for cfg in Env.BOK_CHOY_SERVERS.values(): - # result = requests.post( - # 'http://{host}:{port}/coverage_context/update_context'.format(**cfg), - # { - # 'context': f"{item.nodeid}|{when}", - # } - # ) - # assert result.status_code == 204 - - + @pytest.hookimpl(tryfirst=True) def pytest_configure(config): config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") diff --git a/pavelib/paver_tests/test_js_test.py b/pavelib/paver_tests/test_js_test.py index a066a3ea19..ee4df7ea4e 100644 --- a/pavelib/paver_tests/test_js_test.py +++ b/pavelib/paver_tests/test_js_test.py @@ -55,15 +55,6 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): ["--suite=lms"], ["--suite=lms --coverage"], ) - # @ddt.unpack - # def test_test_js_run(self, options_string): - # """ - # Test the "test_js_run" task. - # """ - # options = self.parse_options_string(options_string) - # self.reset_task_messages() - # call_task("pavelib.js_test.test_js_run", options=options) - # self.verify_messages(options=options, dev_mode=False) @ddt.data( [""], @@ -71,16 +62,7 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): ["--suite=lms"], ["--suite=lms --port=9999"], ) - # @ddt.unpack - # def test_test_js_dev(self, options_string): - # """ - # Test the "test_js_run" task. - # """ - # options = self.parse_options_string(options_string) - # self.reset_task_messages() - # call_task("pavelib.js_test.test_js_dev", options=options) - # self.verify_messages(options=options, dev_mode=True) - + def parse_options_string(self, options_string): """ Parse a string containing the options for a test run @@ -99,52 +81,4 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): "suite": suite, "coverage": coverage, "port": port, - } - - # def verify_messages(self, options, dev_mode): - # """ - # Verify that the messages generated when running tests are as expected - # for the specified options and dev_mode. - # """ - # is_coverage = options['coverage'] - # port = options['port'] - # expected_messages = [] - # suites = Env.JS_TEST_ID_KEYS if options['suite'] == 'all' else [options['suite']] - - # expected_messages.extend(self.EXPECTED_COMMANDS) - # if not dev_mode and not is_coverage: - # expected_messages.append(self.EXPECTED_DELETE_JAVASCRIPT_REPORT_COMMAND.format( - # platform_root=self.platform_root - # )) - # expected_messages.append(self.EXPECTED_INSTALL_NPM_ASSETS_COMMAND) - - # command_template = ( - # 'node --max_old_space_size=4096 node_modules/.bin/karma start {options}' - # ) - - # for suite in suites: - # # Karma test command - # if suite != 'jest-snapshot': - # karma_config_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(suite)] - # expected_test_tool_command = command_template.format( - # options=self.EXPECTED_KARMA_OPTIONS.format( - # config_file=karma_config_file, - # single_run='false' if dev_mode else 'true', - # suite=suite, - # platform_root=self.platform_root, - # browser=Env.KARMA_BROWSER, - # ), - # ) - # if is_coverage: - # expected_test_tool_command += self.EXPECTED_COVERAGE_OPTIONS.format( - # platform_root=self.platform_root, - # suite=suite - # ) - # if port: - # expected_test_tool_command += f" --port={port}" - # else: - # expected_test_tool_command = 'jest' - - # expected_messages.append(expected_test_tool_command) - - # assert self.task_messages == expected_messages + } \ No newline at end of file From 3d893a510a8a222bbdcb1eea587238d4ddb37f0f Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 27 Sep 2023 18:56:25 +0500 Subject: [PATCH 06/35] chore: remove bok-choy --- openedx/testing/coverage_context_listener/pytest_plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py index 37051385c0..bb8dfcd37a 100644 --- a/openedx/testing/coverage_context_listener/pytest_plugin.py +++ b/openedx/testing/coverage_context_listener/pytest_plugin.py @@ -4,7 +4,6 @@ A pytest plugin that reports test contexts to coverage running in another proces import pytest - class RemoteContextPlugin: """ Pytest plugin for reporting pytests contexts to coverage running in another process @@ -22,7 +21,6 @@ class RemoteContextPlugin: def pytest_runtest_call(self, item): self.doit(item, "call") - @pytest.hookimpl(tryfirst=True) def pytest_configure(config): config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") From 0370f44b50b1063a9d018206d3226978c465dc55 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 27 Sep 2023 19:16:01 +0500 Subject: [PATCH 07/35] chore: fix test cases --- openedx/testing/coverage_context_listener/pytest_plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py index bb8dfcd37a..355c5abc0e 100644 --- a/openedx/testing/coverage_context_listener/pytest_plugin.py +++ b/openedx/testing/coverage_context_listener/pytest_plugin.py @@ -4,6 +4,7 @@ A pytest plugin that reports test contexts to coverage running in another proces import pytest + class RemoteContextPlugin: """ Pytest plugin for reporting pytests contexts to coverage running in another process @@ -21,6 +22,7 @@ class RemoteContextPlugin: def pytest_runtest_call(self, item): self.doit(item, "call") + @pytest.hookimpl(tryfirst=True) def pytest_configure(config): config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") From 875de8eb715a9bd4174806004923550817909b04 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Thu, 28 Sep 2023 12:27:00 +0500 Subject: [PATCH 08/35] chore: fix test cases --- pavelib/paver_tests/test_js_test.py | 70 ++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/pavelib/paver_tests/test_js_test.py b/pavelib/paver_tests/test_js_test.py index ee4df7ea4e..2180d28d45 100644 --- a/pavelib/paver_tests/test_js_test.py +++ b/pavelib/paver_tests/test_js_test.py @@ -55,6 +55,15 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): ["--suite=lms"], ["--suite=lms --coverage"], ) + @ddt.unpack + def test_test_js_run(self, options_string): + """ + Test the "test_js_run" task. + """ + options = self.parse_options_string(options_string) + self.reset_task_messages() + call_task("pavelib.js_test.test_js_run", options=options) + self.verify_messages(options=options, dev_mode=False) @ddt.data( [""], @@ -62,7 +71,16 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): ["--suite=lms"], ["--suite=lms --port=9999"], ) - + @ddt.unpack + def test_test_js_dev(self, options_string): + """ + Test the "test_js_run" task. + """ + options = self.parse_options_string(options_string) + self.reset_task_messages() + call_task("pavelib.js_test.test_js_dev", options=options) + self.verify_messages(options=options, dev_mode=True) + def parse_options_string(self, options_string): """ Parse a string containing the options for a test run @@ -81,4 +99,52 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): "suite": suite, "coverage": coverage, "port": port, - } \ No newline at end of file + } + + def verify_messages(self, options, dev_mode): + """ + Verify that the messages generated when running tests are as expected + for the specified options and dev_mode. + """ + is_coverage = options['coverage'] + port = options['port'] + expected_messages = [] + suites = Env.JS_TEST_ID_KEYS if options['suite'] == 'all' else [options['suite']] + + expected_messages.extend(self.EXPECTED_COMMANDS) + if not dev_mode and not is_coverage: + expected_messages.append(self.EXPECTED_DELETE_JAVASCRIPT_REPORT_COMMAND.format( + platform_root=self.platform_root + )) + expected_messages.append(self.EXPECTED_INSTALL_NPM_ASSETS_COMMAND) + + command_template = ( + 'node --max_old_space_size=4096 node_modules/.bin/karma start {options}' + ) + + # for suite in suites: + # # Karma test command + # if suite != 'jest-snapshot': + # karma_config_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(suite)] + # expected_test_tool_command = command_template.format( + # options=self.EXPECTED_KARMA_OPTIONS.format( + # config_file=karma_config_file, + # single_run='false' if dev_mode else 'true', + # suite=suite, + # platform_root=self.platform_root, + # browser=Env.KARMA_BROWSER, + # ), + # ) + # if is_coverage: + # expected_test_tool_command += self.EXPECTED_COVERAGE_OPTIONS.format( + # platform_root=self.platform_root, + # suite=suite + # ) + # if port: + # expected_test_tool_command += f" --port={port}" + # else: + # expected_test_tool_command = 'jest' + + # expected_messages.append(expected_test_tool_command) + + assert self.task_messages == expected_messages From a763287152bdff07267acd5b0d50decb100547b8 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Thu, 28 Sep 2023 16:05:15 +0500 Subject: [PATCH 09/35] chore: remove all reference for bok-choy --- .dockerignore | 1 - cms/envs/test_static_optimized.py | 4 ---- lms/envs/test_static_optimized.py | 4 ---- openedx/tests/xblock_integration/xblock_testcase.py | 2 +- scripts/generic-ci-tests.sh | 3 +-- 5 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.dockerignore b/.dockerignore index e8f6e66311..1665783ba4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -89,7 +89,6 @@ test_root/paver_logs/ test_root/uploads/ **/django-pyfs **/.tox/ -common/test/db_cache/bok_choy_*.yaml common/test/data/badges/*.png ### Installation artifacts diff --git a/cms/envs/test_static_optimized.py b/cms/envs/test_static_optimized.py index 61738c9254..c92d9a7262 100644 --- a/cms/envs/test_static_optimized.py +++ b/cms/envs/test_static_optimized.py @@ -1,10 +1,6 @@ """ Settings used when generating static assets for use in tests. -For example, Bok Choy uses two different settings files: -1. test_static_optimized is used when invoking collectstatic -2. bok_choy is used when running CMS and LMS - Note: it isn't possible to have a single settings file, because Django doesn't support both generating static assets to a directory and also serving static from the same directory. diff --git a/lms/envs/test_static_optimized.py b/lms/envs/test_static_optimized.py index 5c9cf06208..b57276b040 100644 --- a/lms/envs/test_static_optimized.py +++ b/lms/envs/test_static_optimized.py @@ -1,10 +1,6 @@ """ Settings used when generating static assets for use in tests. -For example, Bok Choy uses two different settings files: -1. test_static_optimized is used when invoking collectstatic -2. bok_choy is used when running CMS and LMS - Note: it isn't possible to have a single settings file, because Django doesn't support both generating static assets to a directory and also serving static from the same directory. diff --git a/openedx/tests/xblock_integration/xblock_testcase.py b/openedx/tests/xblock_integration/xblock_testcase.py index e75ca9a63d..a6a4ac67a4 100644 --- a/openedx/tests/xblock_integration/xblock_testcase.py +++ b/openedx/tests/xblock_integration/xblock_testcase.py @@ -71,7 +71,7 @@ class XBlockEventTestMixin: 2. assert_event_published verifies that an event of a given search specification was published. - The Mongo/bok_choy event tests in cohorts have nice examplars for + The Mongo event tests in cohorts have nice examplars for how such functionality might look. In the future, we would like to expand both search diff --git a/scripts/generic-ci-tests.sh b/scripts/generic-ci-tests.sh index 5aabf8fde4..f854a3e36c 100755 --- a/scripts/generic-ci-tests.sh +++ b/scripts/generic-ci-tests.sh @@ -20,11 +20,10 @@ set -e # - "pavelib-unit": Run Python unit tests from the pavelib/lib directory # - "pavelib-js-unit": Run the JavaScript tests and the Python unit # tests from the pavelib/lib directory -# - "bok-choy": Run acceptance tests that use the bok-choy framework # # `SHARD` is a number indicating which subset of the tests to build. # -# For "bok-choy" and "lms-unit", the tests are put into shard groups +# For "lms-unit", the tests are put into shard groups # using the 'attr' decorator (e.g. "@attr(shard=1)"). Anything with # the 'shard=n' attribute will run in the nth shard. If there isn't a # shard explicitly assigned, the test will run in the last shard. From 3fc454c7e31fcccbd65726dec74e08bd25df5f71 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Thu, 28 Sep 2023 16:24:40 +0500 Subject: [PATCH 10/35] chore: remove bok-choy --- .github/workflows/static-assets-check.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 7aa7dce783..3e136e6569 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -62,6 +62,11 @@ jobs: - name: Add node_modules bin to $Path run: echo $GITHUB_WORKSPACE/node_modules/.bin >> $GITHUB_PATH + - name: Run Static Assets Check + env: + LMS_CFG: lms/envs/minimal.yml + CMS_CFG: cms/envs/minimal.yml + run: | paver update_assets lms paver update_assets cms From 3ecf2b9a554bfccf7665d038e85d8fcda823ad98 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Thu, 28 Sep 2023 16:25:45 +0500 Subject: [PATCH 11/35] chore: remove bok-choy --- .github/workflows/static-assets-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 3e136e6569..db8b9ded21 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -66,7 +66,7 @@ jobs: env: LMS_CFG: lms/envs/minimal.yml CMS_CFG: cms/envs/minimal.yml - + run: | paver update_assets lms paver update_assets cms From d668af2fce52562a3ff178d143306045d560038a Mon Sep 17 00:00:00 2001 From: salman2013 Date: Mon, 2 Oct 2023 16:04:33 +0500 Subject: [PATCH 12/35] chore: fix comments, remove and add some code for karma --- common/djangoapps/terrain/stubs/lti.py | 4 +- .../static/common/js/jasmine.common.conf.js | 12 + .../static/common/js/jasmine_stack_trace.js | 29 ++ common/static/common/js/karma.common.conf.js | 415 ++++++++++++++++++ .../0004-managing-django-settings.rst | 1 + .../content_type_gating/tests/test_access.py | 2 +- .../pytest_plugin.py | 32 +- pavelib/paver_tests/test_js_test.py | 46 +- pavelib/utils/envs.py | 29 +- tox.ini | 6 +- 10 files changed, 511 insertions(+), 65 deletions(-) create mode 100644 common/static/common/js/jasmine.common.conf.js create mode 100644 common/static/common/js/jasmine_stack_trace.js create mode 100644 common/static/common/js/karma.common.conf.js diff --git a/common/djangoapps/terrain/stubs/lti.py b/common/djangoapps/terrain/stubs/lti.py index b3518f22cf..158bf860da 100644 --- a/common/djangoapps/terrain/stubs/lti.py +++ b/common/djangoapps/terrain/stubs/lti.py @@ -78,7 +78,7 @@ class StubLtiHandler(StubHttpRequestHandler): 'callback_url': self.post_dict.get('lis_outcome_service_url').replace('https', 'http'), 'sourcedId': self.post_dict.get('lis_result_sourcedid') } - host = os.environ.get(self.server.server_address[0]) + host = self.server.server_address[0] submit_url = f'//{host}:{self.server.server_address[1]}' content = self._create_content(status_message, submit_url) self.send_response(200, content) @@ -296,7 +296,7 @@ class StubLtiHandler(StubHttpRequestHandler): """ client_secret = str(self.server.config.get('client_secret', self.DEFAULT_CLIENT_SECRET)) - host = os.environ.get('127.0.0.1') + host = '127.0.0.1' port = self.server.server_address[1] lti_base = self.DEFAULT_LTI_ADDRESS.format(host=host, port=port) lti_endpoint = self.server.config.get('lti_endpoint', self.DEFAULT_LTI_ENDPOINT) diff --git a/common/static/common/js/jasmine.common.conf.js b/common/static/common/js/jasmine.common.conf.js new file mode 100644 index 0000000000..d6d7f79773 --- /dev/null +++ b/common/static/common/js/jasmine.common.conf.js @@ -0,0 +1,12 @@ +/* eslint-env node */ + +'use strict'; + +// By default, fixtures are loaded from spec/javascripts/fixtures but in karma everything gets served from /base +jasmine.getFixtures().fixturesPath = '/base/'; + +// https://github.com/edx/js-test-tool/blob/master/js_test_tool/templates/jasmine_test_runner.html#L10 +// Stub out modal dialog alerts, which will prevent +// us from accessing the test results in the DOM +window.confirm = function() { return true; }; +window.alert = function() { }; diff --git a/common/static/common/js/jasmine_stack_trace.js b/common/static/common/js/jasmine_stack_trace.js new file mode 100644 index 0000000000..f61d591f0a --- /dev/null +++ b/common/static/common/js/jasmine_stack_trace.js @@ -0,0 +1,29 @@ +/* This file overrides ExceptionFormatter of jasmine before it's initialization in karma-jasmine's + boot.js. It's important because ExceptionFormatter returns a constructor function. Once the method has been + initialized we can't override the ExceptionFormatter as Jasmine then uses the stored reference to the function */ +(function() { + /* globals jasmineRequire */ + + 'use strict'; + + var OldExceptionFormatter = jasmineRequire.ExceptionFormatter(), + oldExceptionFormatter = new OldExceptionFormatter(), + MAX_STACK_TRACE_LINES = 10; + + jasmineRequire.ExceptionFormatter = function() { + function ExceptionFormatter() { + this.message = oldExceptionFormatter.message; + this.stack = function(error) { + var errorMsg = null; + + if (error) { + errorMsg = error.stack.split('\n').slice(0, MAX_STACK_TRACE_LINES).join('\n'); + } + + return errorMsg; + }; + } + + return ExceptionFormatter; + }; +}()); diff --git a/common/static/common/js/karma.common.conf.js b/common/static/common/js/karma.common.conf.js new file mode 100644 index 0000000000..274d4efa7e --- /dev/null +++ b/common/static/common/js/karma.common.conf.js @@ -0,0 +1,415 @@ +// Common settings and helpers for setting up Karma config. +// +// To run all the tests in a suite and print results to the console: +// +// karma start +// E.g. karma start lms/static/karma_lms.conf.js +// +// +// To run the tests for debugging: Debugging can be done in any browser +// but Chrome's developer console debugging experience is best. +// +// karma start --browsers=BROWSER --single-run=false +// +// +// To run the tests with coverage and junit reports: +// +// karma start --browsers=BROWSER +// --coverage --junitreportpath= --coveragereportpath= +// +// where `BROWSER` could be Chrome or Firefox. +// +// +// Troubleshooting tips: +// +// If you get an error like: "TypeError: __cov_KBCc7ZI4xZm8W2BC5NQLDg.s is undefined", +// that means the patterns in sourceFiles and specFiles are matching the same file. +// This causes Istanbul, which is used for tracking coverage to instrument the file +// multiple times. +// +// +// If you see the error: "EMFILE, too many open files" that means the files pattern +// that has been added is matching too many files. The glob library used by Karma +// does not use graceful-fs and tries to read files simultaneously. +// + +/* eslint-env node */ +/* globals process */ + +'use strict'; + +var path = require('path'); +var _ = require('underscore'); + +var appRoot = path.join(__dirname, '../../../../'); +// eslint-disable-next-line import/no-extraneous-dependencies +var webdriver = require('selenium-webdriver'); +// eslint-disable-next-line import/no-extraneous-dependencies + +var webpackConfig = require(path.join(appRoot, 'webpack.dev.config.js')); + +// The following crazy bit is to work around the webpack.optimize.CommonsChunkPlugin +// plugin. The problem is that it it factors out the code that defines webpackJsonp +// and puts in in the commons JS, which Karma doesn't know to load first. This is a +// workaround recommended in the karma-webpack bug report that basically just removes +// the plugin for the purposes of Karma testing (the plugin is meant to be an +// optimization only). +// https://github.com/webpack-contrib/karma-webpack/issues/24#issuecomment-257613167 +// +// This should be fixed in v3 of karma-webpack +var commonsChunkPluginIndex = webpackConfig[0].plugins.findIndex(function(plugin) { return plugin.chunkNames; }); + +// Files which are needed by all lms/cms suites. +var commonFiles = { + libraryFiles: [ + {pattern: 'common/js/vendor/**/*.js'}, + {pattern: 'edx-ui-toolkit/js/**/*.js'}, + {pattern: 'xmodule_js/common_static/common/js/**/!(*spec).js'}, + {pattern: 'xmodule_js/common_static/js/**/!(*spec).js'}, + {pattern: 'xmodule_js/src/**/*.js'} + ], + + sourceFiles: [ + {pattern: 'common/js/!(spec_helpers)/**/!(*spec).js'} + ], + + specFiles: [ + {pattern: 'common/js/spec_helpers/**/*.js'} + ], + + fixtureFiles: [ + {pattern: 'common/templates/**/*.underscore'} + ] +}; + +webpackConfig[0].plugins.splice(commonsChunkPluginIndex, 1); + +delete webpackConfig[0].entry; + +/** + * Customize the name attribute in xml testcase element + * @param {Object} browser + * @param {Object} result + * @return {String} + */ +function junitNameFormatter(browser, result) { + return result.suite[0] + ': ' + result.description; +} + +/** + * Customize the classname attribute in xml testcase element + * @param {Object} browser + * @return {String} + */ +function junitClassNameFormatter(browser) { + return 'Javascript.' + browser.name.split(' ')[0]; +} + +/** + * Return array containing default and user supplied reporters + * @param {Object} config + * @return {Array} + */ +function reporters(config) { + var defaultReporters = ['spec', 'junit', 'kjhtml']; + if (config.coverage) { + defaultReporters.push('coverage'); + } + return defaultReporters; +} + +/** + * Split a filepath into basepath and filename + * @param {String} filepath + * @return {Object} + */ +function getBasepathAndFilename(filepath) { + var file, dir; + + if (!filepath) { + // these will configure the reporters to create report files relative to this karma config file + return { + dir: undefined, + file: undefined + }; + } + file = filepath.replace(/^.*[\\/]/, ''); + dir = filepath.replace(file, ''); + + return { + dir: dir, + file: file + }; +} + +/** + * Return coverage reporter settings + * @param {String} config + * @return {Object} + */ +function coverageSettings(config) { + var pth = getBasepathAndFilename(config.coveragereportpath); + return { + dir: pth.dir, + subdir: '.', + includeAllSources: true, + reporters: [ + {type: 'cobertura', file: pth.file}, + {type: 'text-summary'} + ] + }; +} + +/** + * Return junit reporter settings + * @param {String} config + * @return {Object} + */ +function junitSettings(config) { + var pth = getBasepathAndFilename(config.junitreportpath); + return { + outputDir: pth.dir, + outputFile: pth.file, + suite: 'javascript', + useBrowserName: false, + nameFormatter: junitNameFormatter, + classNameFormatter: junitClassNameFormatter + }; +} + +/** + * Return absolute path for files in common and xmodule_js symlink dirs. + * @param {String} appRoot + * @param {String} pattern + * @return {String} + */ +// I'd like to fix the no-shadow violation on the next line, but it would break this shared conf's API. +function defaultNormalizeFunc(appRoot, pattern) { // eslint-disable-line no-shadow + var pat = pattern; + if (pat.match(/^common\/js/)) { + pat = path.join(appRoot, '/common/static/' + pat); + } else if (pat.match(/^xmodule_js\/common_static/)) { + pat = path.join(appRoot, '/common/static/' + + pat.replace(/^xmodule_js\/common_static\//, '')); + } + return pat; +} + +function normalizePathsForCoverage(files, normalizeFunc, preprocessors) { + var normalizeFn = normalizeFunc || defaultNormalizeFunc, + normalizedFile, + filesForCoverage = {}; + + files.forEach(function(file) { + if (!file.ignoreCoverage) { + normalizedFile = normalizeFn(appRoot, file.pattern); + if (preprocessors && preprocessors.hasOwnProperty(normalizedFile)) { + filesForCoverage[normalizedFile] = ['coverage'].concat(preprocessors[normalizedFile]); + } else { + filesForCoverage[normalizedFile] = ['coverage']; + } + } + }); + + return filesForCoverage; +} + +/** + * Sets defaults for each file pattern. + * RequireJS files are excluded by default. + * Webpack files are included by default. + * @param {Object} files + * @return {Object} + */ +function setDefaults(files) { + return files.map(function(f) { + var file = _.isObject(f) ? f : {pattern: f}; + if (!file.included && !file.webpack) { + file.included = false; + } + return file; + }); +} + +function getBaseConfig(config, useRequireJs) { + var getFrameworkFiles = function() { + var files = [ + 'common/static/common/js/vendor/jquery.js', + 'node_modules/jasmine-core/lib/jasmine-core/jasmine.js', + 'common/static/common/js/jasmine_stack_trace.js', + 'node_modules/karma-jasmine/lib/boot.js', + 'node_modules/karma-jasmine/lib/adapter.js', + 'node_modules/jasmine-jquery/lib/jasmine-jquery.js', + 'node_modules/popper.js/dist/umd/popper.js', + 'node_modules/bootstrap/dist/js/bootstrap.js', + 'node_modules/underscore/underscore.js', + 'node_modules/backbone/backbone.js', + 'common/static/js/test/i18n.js' + ]; + + if (useRequireJs) { + files = files.concat([ + 'node_modules/requirejs/require.js', + 'node_modules/karma-requirejs/lib/adapter.js' + ]); + } + + return files; + }; + + // Manually prepends the framework files to the karma files array + // bypassing the karma's framework config. This is necessary if you want + // to add a library or framework that isn't a karma plugin. e.g. we add jasmine-jquery + // which isn't a karma plugin. Though a karma framework for jasmine-jquery is available + // but it's not actively maintained. In future we also wanna add jQuery at the top when + // we upgrade to jQuery 2 + var initFrameworks = function(files) { + getFrameworkFiles().reverse().forEach(function(f) { + files.unshift({ + pattern: path.join(appRoot, f), + included: true, + served: true, + watch: false + }); + }); + }; + + var hostname = 'localhost'; + var port = 9876; + var customPlugin = { + 'framework:custom': ['factory', initFrameworks] + }; + + initFrameworks.$inject = ['config.files']; + + return { + // base path that will be used to resolve all patterns (eg. files, exclude) + basePath: '', + + // frameworks to use + // available frameworks: https://npmjs.org/browse/keyword/karma-adapter + frameworks: ['custom'], + + plugins: [ + 'karma-jasmine', + 'karma-jasmine-html-reporter', + 'karma-requirejs', + 'karma-junit-reporter', + 'karma-coverage', + 'karma-spec-reporter', + 'karma-selenium-webdriver-launcher', + 'karma-webpack', + 'karma-sourcemap-loader', + customPlugin + ], + + // list of files to exclude + exclude: [], + + // karma-reporter + reporters: reporters(config), + + // Spec Reporter configuration + specReporter: { + maxLogLines: 5, + showSpecTiming: true + }, + + coverageReporter: coverageSettings(config), + + junitReporter: junitSettings(config), + + // web server hostname and port + hostname: hostname, + port: port, + + // enable / disable colors in the output (reporters and logs) + colors: true, + + // level of logging + /* possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN + || config.LOG_INFO || config.LOG_DEBUG */ + logLevel: config.LOG_INFO, + + // enable / disable watching file and executing tests whenever any file changes + autoWatch: false, + + // Continuous Integration mode + // if true, Karma captures browsers, runs the tests and exits + singleRun: config.singleRun, + + // Concurrency level + // how many browser should be started simultaneous + concurrency: Infinity, + + browserNoActivityTimeout: 50000, + + client: { + captureConsole: false + }, + + webpack: webpackConfig[0], + + webpackMiddleware: { + watchOptions: { + poll: true + } + } + }; +} + +function configure(config, options) { + var useRequireJs = options.useRequireJs === undefined ? true : options.useRequireJs, + baseConfig = getBaseConfig(config, useRequireJs), + files, filesForCoverage, preprocessors; + + if (options.includeCommonFiles) { + _.forEach(['libraryFiles', 'sourceFiles', 'specFiles', 'fixtureFiles'], function(collectionName) { + options[collectionName] = _.flatten([commonFiles[collectionName], options[collectionName]]); + }); + } + + files = _.flatten( + _.map( + ['libraryFilesToInclude', 'libraryFiles', 'sourceFiles', 'specFiles', 'fixtureFiles', 'runFiles'], + function(collectionName) { return options[collectionName] || []; } + ) + ); + + files.unshift( + {pattern: path.join(appRoot, 'common/static/common/js/jasmine.common.conf.js'), included: true} + ); + + if (useRequireJs) { + files.unshift({pattern: 'common/js/utils/require-serial.js', included: true}); + } + + // Karma sets included=true by default. + // We set it to false by default because RequireJS should be used instead. + files = setDefaults(files); + + filesForCoverage = _.flatten( + _.map( + ['sourceFiles', 'specFiles'], + function(collectionName) { return options[collectionName]; } + ) + ); + + // If we give symlink paths to Istanbul, coverage for each path gets tracked + // separately. So we pass absolute paths to the karma-coverage preprocessor. + preprocessors = _.extend( + {}, + options.preprocessors, + normalizePathsForCoverage(filesForCoverage, options.normalizePathsForCoverageFunc, options.preprocessors) + ); + + config.set(_.extend(baseConfig, { + files: files, + preprocessors: preprocessors + })); +} + +module.exports = { + configure: configure, + appRoot: appRoot +}; diff --git a/docs/decisions/0004-managing-django-settings.rst b/docs/decisions/0004-managing-django-settings.rst index bf5698af39..6da93a7616 100644 --- a/docs/decisions/0004-managing-django-settings.rst +++ b/docs/decisions/0004-managing-django-settings.rst @@ -58,6 +58,7 @@ and for all environments to use __init__.py to load their settings. The following files should be obviated by this change: +* bok_choy_docker.py * devstack_docker.py * devstack_optimized.py * devstack.py diff --git a/openedx/features/content_type_gating/tests/test_access.py b/openedx/features/content_type_gating/tests/test_access.py index 53582df34a..3ad9ab2757 100644 --- a/openedx/features/content_type_gating/tests/test_access.py +++ b/openedx/features/content_type_gating/tests/test_access.py @@ -228,7 +228,7 @@ class TestProblemTypeAccess(SharedModuleStoreTestCase, MasqueradeMixin): # pyli graded=False, ) cls.graded_score_weight_blocks[(graded, has_score, weight)] = block - host = os.environ.get('127.0.0.1') + host = '127.0.0.1' metadata_lti_xblock = { 'lti_id': 'correct_lti_id', 'launch_url': 'http://{}:{}/{}'.format(host, '8765', 'correct_lti_endpoint'), diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py index 355c5abc0e..1332516308 100644 --- a/openedx/testing/coverage_context_listener/pytest_plugin.py +++ b/openedx/testing/coverage_context_listener/pytest_plugin.py @@ -13,26 +13,16 @@ class RemoteContextPlugin: self.config = config self.active = config.getoption("pytest-contexts") - def pytest_runtest_setup(self, item): - self.doit(item, "setup") - - def pytest_runtest_teardown(self, item): - self.doit(item, "teardown") - - def pytest_runtest_call(self, item): - self.doit(item, "call") + @pytest.hookimpl(tryfirst=True) + def pytest_configure(config): + config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") -@pytest.hookimpl(tryfirst=True) -def pytest_configure(config): - config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") - - -def pytest_addoption(parser): - group = parser.getgroup("coverage") - group.addoption( - "--pytest-remote-contexts", - action="store_true", - dest="pytest-contexts", - help="Capture the pytest contexts that coverage is being captured in in another process", - ) + def pytest_addoption(parser): + group = parser.getgroup("coverage") + group.addoption( + "--pytest-remote-contexts", + action="store_true", + dest="pytest-contexts", + help="Capture the pytest contexts that coverage is being captured in in another process", + ) diff --git a/pavelib/paver_tests/test_js_test.py b/pavelib/paver_tests/test_js_test.py index 2180d28d45..c6860e127a 100644 --- a/pavelib/paver_tests/test_js_test.py +++ b/pavelib/paver_tests/test_js_test.py @@ -122,29 +122,29 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): 'node --max_old_space_size=4096 node_modules/.bin/karma start {options}' ) - # for suite in suites: - # # Karma test command - # if suite != 'jest-snapshot': - # karma_config_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(suite)] - # expected_test_tool_command = command_template.format( - # options=self.EXPECTED_KARMA_OPTIONS.format( - # config_file=karma_config_file, - # single_run='false' if dev_mode else 'true', - # suite=suite, - # platform_root=self.platform_root, - # browser=Env.KARMA_BROWSER, - # ), - # ) - # if is_coverage: - # expected_test_tool_command += self.EXPECTED_COVERAGE_OPTIONS.format( - # platform_root=self.platform_root, - # suite=suite - # ) - # if port: - # expected_test_tool_command += f" --port={port}" - # else: - # expected_test_tool_command = 'jest' + for suite in suites: + # Karma test command + if suite != 'jest-snapshot': + karma_config_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(suite)] + expected_test_tool_command = command_template.format( + options=self.EXPECTED_KARMA_OPTIONS.format( + config_file=karma_config_file, + single_run='false' if dev_mode else 'true', + suite=suite, + platform_root=self.platform_root, + browser=Env.KARMA_BROWSER, + ), + ) + if is_coverage: + expected_test_tool_command += self.EXPECTED_COVERAGE_OPTIONS.format( + platform_root=self.platform_root, + suite=suite + ) + if port: + expected_test_tool_command += f" --port={port}" + else: + expected_test_tool_command = 'jest' - # expected_messages.append(expected_test_tool_command) + expected_messages.append(expected_test_tool_command) assert self.task_messages == expected_messages diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index ca7ec0f4c6..40b616571e 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -77,34 +77,33 @@ class Env: PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" # Detect if in a Docker container, and if so which one - SERVER_HOST = os.environ.get('0.0.0.0') - USING_DOCKER = SERVER_HOST != '0.0.0.0' - DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' + # USING_DOCKER = SERVER_HOST != '0.0.0.0' + DEVSTACK_SETTINGS = 'devstack' TEST_SETTINGS = 'test' # Test Ids Directory TEST_DIR = REPO_ROOT / ".testids" # Configured browser to use for the js test suites - # SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') + SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') # if USING_DOCKER: # KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' # else: - # KARMA_BROWSER = 'FirefoxNoUpdates' + KARMA_BROWSER = 'FirefoxNoUpdates' # Files used to run each of the js test suites # TODO: Store this as a dict. Order seems to matter for some # reason. See issue TE-415. - # KARMA_CONFIG_FILES = [ - # REPO_ROOT / 'cms/static/karma_cms.conf.js', - # REPO_ROOT / 'cms/static/karma_cms_squire.conf.js', - # REPO_ROOT / 'cms/static/karma_cms_webpack.conf.js', - # REPO_ROOT / 'lms/static/karma_lms.conf.js', - # REPO_ROOT / 'xmodule/js/karma_xmodule.conf.js', - # REPO_ROOT / 'xmodule/js/karma_xmodule_webpack.conf.js', - # REPO_ROOT / 'common/static/karma_common.conf.js', - # REPO_ROOT / 'common/static/karma_common_requirejs.conf.js', - # ] + KARMA_CONFIG_FILES = [ + REPO_ROOT / 'cms/static/karma_cms.conf.js', + REPO_ROOT / 'cms/static/karma_cms_squire.conf.js', + REPO_ROOT / 'cms/static/karma_cms_webpack.conf.js', + REPO_ROOT / 'lms/static/karma_lms.conf.js', + REPO_ROOT / 'xmodule/js/karma_xmodule.conf.js', + REPO_ROOT / 'xmodule/js/karma_xmodule_webpack.conf.js', + REPO_ROOT / 'common/static/karma_common.conf.js', + REPO_ROOT / 'common/static/karma_common_requirejs.conf.js', + ] JS_TEST_ID_KEYS = [ 'cms', diff --git a/tox.ini b/tox.ini index cacd77ea0b..ce00efa870 100644 --- a/tox.ini +++ b/tox.ini @@ -40,9 +40,9 @@ passenv = NPM_CONFIG_PREFIX PYTHON_VERSION SCRAPY_SETTINGS_MODULE - #SELENIUM_BROWSER - #SELENIUM_HOST - #SELENIUM_PORT + SELENIUM_BROWSER + SELENIUM_HOST + SELENIUM_PORT SHARD SKIP_NPM_INSTALL SSH_AUTH_SOCK From 74a4c88c3d0ff82c073f9d5034985fa0f6506c0a Mon Sep 17 00:00:00 2001 From: salman2013 Date: Mon, 2 Oct 2023 16:51:12 +0500 Subject: [PATCH 13/35] chore: fix tests --- .github/workflows/static-assets-check.yml | 2 +- openedx/testing/coverage_context_listener/pytest_plugin.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index db8b9ded21..bec7349fe3 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -65,7 +65,7 @@ jobs: - name: Run Static Assets Check env: LMS_CFG: lms/envs/minimal.yml - CMS_CFG: cms/envs/minimal.yml + CMS_CFG: lms/envs/minimal.yml run: | paver update_assets lms diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py index 1332516308..28fd90fc6a 100644 --- a/openedx/testing/coverage_context_listener/pytest_plugin.py +++ b/openedx/testing/coverage_context_listener/pytest_plugin.py @@ -17,7 +17,6 @@ class RemoteContextPlugin: def pytest_configure(config): config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") - def pytest_addoption(parser): group = parser.getgroup("coverage") group.addoption( From 65d3b1084ba22de992e09f2000ee332472152a97 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Tue, 3 Oct 2023 00:11:23 +0500 Subject: [PATCH 14/35] chore: fix test cases --- common/djangoapps/terrain/stubs/lti.py | 1 - .../pytest_plugin.py | 27 +++++++++++-------- pavelib/paver_tests/test_js_test.py | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/common/djangoapps/terrain/stubs/lti.py b/common/djangoapps/terrain/stubs/lti.py index 158bf860da..46535abb9f 100644 --- a/common/djangoapps/terrain/stubs/lti.py +++ b/common/djangoapps/terrain/stubs/lti.py @@ -13,7 +13,6 @@ not possible to have this LTI multiple times on a single page in LMS. import base64 import hashlib import logging -import os import textwrap from unittest import mock from uuid import uuid4 diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py index 28fd90fc6a..6ed194b91b 100644 --- a/openedx/testing/coverage_context_listener/pytest_plugin.py +++ b/openedx/testing/coverage_context_listener/pytest_plugin.py @@ -3,6 +3,9 @@ A pytest plugin that reports test contexts to coverage running in another proces """ import pytest +# import requests + +# from pavelib.utils.envs import Env class RemoteContextPlugin: @@ -13,15 +16,17 @@ class RemoteContextPlugin: self.config = config self.active = config.getoption("pytest-contexts") - @pytest.hookimpl(tryfirst=True) - def pytest_configure(config): - config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") - def pytest_addoption(parser): - group = parser.getgroup("coverage") - group.addoption( - "--pytest-remote-contexts", - action="store_true", - dest="pytest-contexts", - help="Capture the pytest contexts that coverage is being captured in in another process", - ) +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") + + +def pytest_addoption(parser): + group = parser.getgroup("coverage") + group.addoption( + "--pytest-remote-contexts", + action="store_true", + dest="pytest-contexts", + help="Capture the pytest contexts that coverage is being captured in in another process", + ) diff --git a/pavelib/paver_tests/test_js_test.py b/pavelib/paver_tests/test_js_test.py index c6860e127a..9d89a94444 100644 --- a/pavelib/paver_tests/test_js_test.py +++ b/pavelib/paver_tests/test_js_test.py @@ -127,7 +127,7 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): if suite != 'jest-snapshot': karma_config_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(suite)] expected_test_tool_command = command_template.format( - options=self.EXPECTED_KARMA_OPTIONS.format( + options=self.EXPECTED_KARMA_OPTIONS.format( config_file=karma_config_file, single_run='false' if dev_mode else 'true', suite=suite, From 4dbefe33f4b2aba025f689ef95c56c3dace1461e Mon Sep 17 00:00:00 2001 From: salman2013 Date: Tue, 3 Oct 2023 08:59:21 +0500 Subject: [PATCH 15/35] chore: fix tests --- openedx/features/content_type_gating/tests/test_access.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openedx/features/content_type_gating/tests/test_access.py b/openedx/features/content_type_gating/tests/test_access.py index 3ad9ab2757..c42a530e3c 100644 --- a/openedx/features/content_type_gating/tests/test_access.py +++ b/openedx/features/content_type_gating/tests/test_access.py @@ -1,7 +1,6 @@ """ Test audit user's access to various content based on content-gating features. """ -import os from datetime import datetime, timedelta from unittest.mock import patch, Mock From 70eafef79842b0916298f5a089b5b432432a4e16 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 4 Oct 2023 15:28:49 +0500 Subject: [PATCH 16/35] chore: fixed comments as per review, code improvements --- .../pytest_plugin.py | 3 - pavelib/utils/envs.py | 4 - pavelib/utils/test/suites/js_suite.py | 98 +++++++++---------- 3 files changed, 49 insertions(+), 56 deletions(-) diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py index 6ed194b91b..a3679027e8 100644 --- a/openedx/testing/coverage_context_listener/pytest_plugin.py +++ b/openedx/testing/coverage_context_listener/pytest_plugin.py @@ -3,9 +3,6 @@ A pytest plugin that reports test contexts to coverage running in another proces """ import pytest -# import requests - -# from pavelib.utils.envs import Env class RemoteContextPlugin: diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 40b616571e..4cfbec7790 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -85,10 +85,6 @@ class Env: TEST_DIR = REPO_ROOT / ".testids" # Configured browser to use for the js test suites - SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') - # if USING_DOCKER: - # KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' - # else: KARMA_BROWSER = 'FirefoxNoUpdates' # Files used to run each of the js test suites diff --git a/pavelib/utils/test/suites/js_suite.py b/pavelib/utils/test/suites/js_suite.py index 40f491f5d6..d96d95f39c 100644 --- a/pavelib/utils/test/suites/js_suite.py +++ b/pavelib/utils/test/suites/js_suite.py @@ -39,63 +39,63 @@ class JsTestSuite(TestSuite): if self.mode == 'run' and not self.run_under_coverage: test_utils.clean_dir(self.report_dir) - # @property - # def _default_subsuites(self): - # """ - # Returns all JS test suites - # """ - # return [JsTestSubSuite(test_id, **self.opts) for test_id in Env.JS_TEST_ID_KEYS if test_id != 'jest-snapshot'] + @property + def _default_subsuites(self): + """ + Returns all JS test suites + """ + return [JsTestSubSuite(test_id, **self.opts) for test_id in Env.JS_TEST_ID_KEYS if test_id != 'jest-snapshot'] -# class JsTestSubSuite(TestSuite): -# """ -# Class for JS suites like cms, cms-squire, lms, common, -# common-requirejs and xmodule -# """ -# def __init__(self, *args, **kwargs): -# super().__init__(*args, **kwargs) -# self.test_id = args[0] -# self.run_under_coverage = kwargs.get('with_coverage', True) -# self.mode = kwargs.get('mode', 'run') -# self.port = kwargs.get('port') -# self.root = self.root + ' javascript' -# self.report_dir = Env.JS_REPORT_DIR +class JsTestSubSuite(TestSuite): + """ + Class for JS suites like cms, cms-squire, lms, common, + common-requirejs and xmodule + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.test_id = args[0] + self.run_under_coverage = kwargs.get('with_coverage', True) + self.mode = kwargs.get('mode', 'run') + self.port = kwargs.get('port') + self.root = self.root + ' javascript' + self.report_dir = Env.JS_REPORT_DIR -# try: -# self.test_conf_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(self.test_id)] -# except ValueError: -# self.test_conf_file = Env.KARMA_CONFIG_FILES[0] + try: + self.test_conf_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(self.test_id)] + except ValueError: + self.test_conf_file = Env.KARMA_CONFIG_FILES[0] -# self.coverage_report = self.report_dir / f'coverage-{self.test_id}.xml' -# self.xunit_report = self.report_dir / f'javascript_xunit-{self.test_id}.xml' + self.coverage_report = self.report_dir / f'coverage-{self.test_id}.xml' + self.xunit_report = self.report_dir / f'javascript_xunit-{self.test_id}.xml' -# @property -# def cmd(self): -# """ -# Run the tests using karma runner. -# """ -# cmd = [ -# "node", -# "--max_old_space_size=4096", -# "node_modules/.bin/karma", -# "start", -# self.test_conf_file, -# "--single-run={}".format('false' if self.mode == 'dev' else 'true'), -# "--capture-timeout=60000", -# f"--junitreportpath={self.xunit_report}", -# f"--browsers={Env.KARMA_BROWSER}", -# ] + @property + def cmd(self): + """ + Run the tests using karma runner. + """ + cmd = [ + "node", + "--max_old_space_size=4096", + "node_modules/.bin/karma", + "start", + self.test_conf_file, + "--single-run={}".format('false' if self.mode == 'dev' else 'true'), + "--capture-timeout=60000", + f"--junitreportpath={self.xunit_report}", + f"--browsers={Env.KARMA_BROWSER}", + ] -# if self.port: -# cmd.append(f"--port={self.port}") + if self.port: + cmd.append(f"--port={self.port}") -# if self.run_under_coverage: -# cmd.extend([ -# "--coverage", -# f"--coveragereportpath={self.coverage_report}", -# ]) + if self.run_under_coverage: + cmd.extend([ + "--coverage", + f"--coveragereportpath={self.coverage_report}", + ]) -# return cmd + return cmd class JestSnapshotTestSuite(TestSuite): From 2b6a6f867e94bac3f66ab04321b2d39434495eff Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 4 Oct 2023 15:57:33 +0500 Subject: [PATCH 17/35] chore: remove plugin which is not using --- .../pytest_plugin.py | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 openedx/testing/coverage_context_listener/pytest_plugin.py diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py deleted file mode 100644 index a3679027e8..0000000000 --- a/openedx/testing/coverage_context_listener/pytest_plugin.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -A pytest plugin that reports test contexts to coverage running in another process. -""" - -import pytest - - -class RemoteContextPlugin: - """ - Pytest plugin for reporting pytests contexts to coverage running in another process - """ - def __init__(self, config): - self.config = config - self.active = config.getoption("pytest-contexts") - - -@pytest.hookimpl(tryfirst=True) -def pytest_configure(config): - config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") - - -def pytest_addoption(parser): - group = parser.getgroup("coverage") - group.addoption( - "--pytest-remote-contexts", - action="store_true", - dest="pytest-contexts", - help="Capture the pytest contexts that coverage is being captured in in another process", - ) From 936a6c0983f63d9e25d242f9b70121ad43fc21ff Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 4 Oct 2023 19:02:16 +0500 Subject: [PATCH 18/35] chore: resolve conflicts --- lms/envs/bok_choy_docker.yml | 184 ----------------------------------- scripts/reset-test-db.sh | 141 --------------------------- 2 files changed, 325 deletions(-) delete mode 100644 lms/envs/bok_choy_docker.yml delete mode 100755 scripts/reset-test-db.sh diff --git a/lms/envs/bok_choy_docker.yml b/lms/envs/bok_choy_docker.yml deleted file mode 100644 index b68beb648d..0000000000 --- a/lms/envs/bok_choy_docker.yml +++ /dev/null @@ -1,184 +0,0 @@ -# ingested edx-platform/lms/envs/bok_choy_docker.auth.json -# ingested edx-platform/lms/envs/bok_choy_docker.env.json -ACTIVATION_EMAIL_SUPPORT_LINK: https://support.example.com/activation-email-help.html -ANALYTICS_DASHBOARD_URL: '' -AWS_ACCESS_KEY_ID: '' -AWS_SECRET_ACCESS_KEY: '' -BUGS_EMAIL: bugs@example.com -BULK_EMAIL_DEFAULT_FROM_EMAIL: no-reply@example.com -CACHES: - celery: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_celery - LOCATION: ['edx.devstack.memcached:11211'] - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - default: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: sandbox_default - LOCATION: ['edx.devstack.memcached:11211'] - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - general: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: sandbox_general - LOCATION: ['edx.devstack.memcached:11211'] - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - mongo_metadata_inheritance: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_mongo_metadata_inheritance - LOCATION: ['edx.devstack.memcached:11211'] - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true - staticfiles: - BACKEND: django.core.cache.backends.memcached.PyMemcacheCache - KEY_FUNCTION: common.djangoapps.util.memcache.safe_key - KEY_PREFIX: integration_static_files - LOCATION: ['edx.devstack.memcached:11211'] - OPTIONS: - no_delay: true - ignore_exc: true - use_pooling: true -CELERY_BROKER_HOSTNAME: localhost -CELERY_BROKER_PASSWORD: celery -CELERY_BROKER_TRANSPORT: amqp -CELERY_BROKER_USER: celery -CERT_QUEUE: certificates -CMS_BASE: '** OVERRIDDEN **' -CODE_JAIL: - limits: {REALTIME: 3, VMEM: 0} -COMMENTS_SERVICE_KEY: password -COMMENTS_SERVICE_URL: http://edx.devstack.lms:4567 -CONTACT_EMAIL: info@example.com -CONTENTSTORE: - DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [edx.devstack.mongo] - port: 27017 - ENGINE: xmodule.contentstore.mongo.MongoContentStore - OPTIONS: - db: test - host: [edx.devstack.mongo] - port: 27017 -DATABASES: - default: {ENGINE: django.db.backends.mysql, HOST: edx.devstack.mysql80, NAME: edxtest, - PASSWORD: '', PORT: '3306', USER: root} - student_module_history: {ENGINE: django.db.backends.mysql, HOST: edx.devstack.mysql80, - NAME: student_module_history_test, PASSWORD: '', PORT: '3306', USER: root} -DEFAULT_FEEDBACK_EMAIL: feedback@example.com -DEFAULT_FROM_EMAIL: registration@example.com -DJFS: {aws_access_key_id: test, aws_secret_access_key: test, bucket: test, prefix: test, - type: s3fs} -DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [edx.devstack.mongo] - port: 27017 -EMAIL_BACKEND: django.core.mail.backends.dummy.EmailBackend -EVENT_TRACKING_BACKENDS: - mongo: - ENGINE: eventtracking.backends.mongodb.MongoBackend - OPTIONS: - collection: events - database: test - host: [edx.devstack.mongo] - port: 27017 -FEATURES: {ALLOW_AUTOMATED_SIGNUPS: true, AUTOMATIC_AUTH_FOR_TESTING: true, - AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING: true, CERTIFICATES_HTML_VIEW: true, - CERTIFICATES_INSTRUCTOR_GENERATION: true, CUSTOM_COURSES_EDX: true, - ENABLE_COURSE_DISCOVERY: true, ENABLE_DISCUSSION_SERVICE: true, ENABLE_GRADE_DOWNLOADS: true, - ENABLE_SPECIAL_EXAMS: true, ENABLE_THIRD_PARTY_AUTH: true, - ENABLE_VERIFIED_CERTIFICATES: true, EXPOSE_CACHE_PROGRAMS_ENDPOINT: true, MODE_CREATION_FOR_TESTING: true, - PREVIEW_LMS_BASE: 'preview.localhost:8003', RESTRICT_AUTOMATIC_AUTH: false, SHOW_HEADER_LANGUAGE_SELECTOR: true} -GITHUB_REPO_ROOT: '** OVERRIDDEN **' -JWT_AUTH: {JWT_PRIVATE_SIGNING_JWK: '{"e": "AQAB", "d": "HIiV7KNjcdhVbpn3KT-I9n3JPf5YbGXsCIedmPqDH1d4QhBofuAqZ9zebQuxkRUpmqtYMv0Zi6ECSUqH387GYQF_XvFUFcjQRPycISd8TH0DAKaDpGr-AYNshnKiEtQpINhcP44I1AYNPCwyoxXA1fGTtmkKChsuWea7o8kytwU5xSejvh5-jiqu2SF4GEl0BEXIAPZsgbzoPIWNxgO4_RzNnWs6nJZeszcaDD0CyezVSuH9QcI6g5QFzAC_YuykSsaaFJhZ05DocBsLczShJ9Omf6PnK9xlm26I84xrEh_7x4fVmNBg3xWTLh8qOnHqGko93A1diLRCrKHOvnpvgQ", - "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ", - "q": "3T3DEtBUka7hLGdIsDlC96Uadx_q_E4Vb1cxx_4Ss_wGp1Loz3N3ZngGyInsKlmbBgLo1Ykd6T9TRvRNEWEtFSOcm2INIBoVoXk7W5RuPa8Cgq2tjQj9ziGQ08JMejrPlj3Q1wmALJr5VTfvSYBu0WkljhKNCy1KB6fCby0C9WE", - "p": "vUqzWPZnDG4IXyo-k5F0bHV0BNL_pVhQoLW7eyFHnw74IOEfSbdsMspNcPSFIrtgPsn7981qv3lN_staZ6JflKfHayjB_lvltHyZxfl0dvruShZOx1N6ykEo7YrAskC_qxUyrIvqmJ64zPW3jkuOYrFs7Ykj3zFx3Zq1H5568G0", - "kid": "BTZ9HA6K", "kty": "RSA"}', JWT_PUBLIC_SIGNING_JWK_SET: '{"keys": [{"kid": - "BTZ9HA6K", "e": "AQAB", "kty": "RSA", "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ"}]}', - JWT_SECRET_KEY: super-secret-key} -LMS_BASE: http://edx.devstack.lms:18003 -LMS_ROOT_URL: http://edx.devstack.lms:18003 -LOCAL_LOGLEVEL: INFO -LOGGING_ENV: sandbox -LOG_DIR: '** OVERRIDDEN **' -MEDIA_URL: /media/ -MKTG_URL_LINK_MAP: {ABOUT: about, BLOG: blog, CAREERS: careers, CONTACT: contact, - COURSES: courses, DONATE: donate, HELP_CENTER: help-center, HONOR: honor, NEWS: news, - PRESS: press, PRIVACY: privacy, ROOT: root, SITEMAP.XML: sitemap_xml, TOS: tos, - WHAT_IS_VERIFIED_CERT: verified-certificate} -MODULESTORE: - default: - ENGINE: xmodule.modulestore.mixed.MixedModuleStore - OPTIONS: - mappings: {} - stores: - - DOC_STORE_CONFIG: - collection: modulestore - db: test - host: [edx.devstack.mongo] - port: 27017 - ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore - NAME: draft - OPTIONS: - collection: modulestore - db: test - default_class: xmodule.hidden_block.HiddenBlock - fs_root: '** OVERRIDDEN **' - host: [edx.devstack.mongo] - port: 27017 - render_template: common.djangoapps.edxmako.shortcuts.render_to_string - - ENGINE: xmodule.modulestore.xml.XMLModuleStore - NAME: xml - OPTIONS: {data_dir: '** OVERRIDDEN **', default_class: xmodule.hidden_block.HiddenBlock} -PASSWORD_RESET_SUPPORT_LINK: https://support.example.com/password-reset-help.html -REGISTRATION_EXTENSION_FORM: openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm -REGISTRATION_EXTRA_FIELDS: {city: hidden, country: required, gender: optional, goals: optional, - honor_code: required, level_of_education: optional, mailing_address: optional, terms_of_service: hidden, - year_of_birth: optional} -SECRET_KEY: 'bokchoy_docker_secret_key' -SERVER_EMAIL: devops@example.com -SESSION_COOKIE_DOMAIN: null -SITE_NAME: localhost:8003 -SOCIAL_SHARING_SETTINGS: {CERTIFICATE_FACEBOOK: true, CERTIFICATE_FACEBOOK_TEXT: 'Testing - facebook feature:', CUSTOM_COURSE_URLS: true, DASHBOARD_FACEBOOK: true, DASHBOARD_TWITTER: true, - DASHBOARD_TWITTER_TEXT: 'Testing feature:'} -STATIC_URL_BASE: /static/ -SUPPORT_SITE_LINK: https://support.example.com -SYSLOG_SERVER: '' -TECH_SUPPORT_EMAIL: technical@example.com -THIRD_PARTY_AUTH_BACKENDS: [social_core.backends.google.GoogleOAuth2, social_core.backends.linkedin.LinkedinOAuth2, - social_core.backends.facebook.FacebookOAuth2, common.djangoapps.third_party_auth.dummy.DummyBackend, - common.djangoapps.third_party_auth.saml.SAMLAuthBackend] -TIME_ZONE: America/New_York -TRACKING_BACKENDS: - mongo: - ENGINE: common.djangoapps.track.backends.mongodb.MongoBackend - OPTIONS: - collection: events - database: test - host: [edx.devstack.mongo] - port: 27017 -WIKI_ENABLED: true -XQUEUE_INTERFACE: - basic_auth: [edx, edx] - django_auth: {password: password, username: lms} - url: '** OVERRIDDEN **' -ZENDESK_API_KEY: '' -ZENDESK_USER: '' - diff --git a/scripts/reset-test-db.sh b/scripts/reset-test-db.sh deleted file mode 100755 index 4700f5a99a..0000000000 --- a/scripts/reset-test-db.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env bash - -############################################################################ -# -# reset-test-db.sh -# -# Resets the MySQL test database for the bok-choy acceptance tests. -# -# If it finds a cached schema and migration history, it will start -# from the cached version to speed up migrations. -# -# If no cached database exists, it will create one. This can be -# checked into the repo to speed up future tests. -# -# Note that we do NOT want to re-use the cache between test runs! -# A newer commit could introduce migrations that do not exist -# in other commits, which could cause migrations to fail in the other -# commits. -# -# For this reason, we always use a cache that was committed to master -# at the time the branch was created. -# -############################################################################ - -# Fail fast -set -e - -DB_CACHE_DIR="common/test/db_cache" - -if [[ -z "$BOK_CHOY_HOSTNAME" ]]; then - MYSQL_HOST="" - SETTINGS="bok_choy" -else - MYSQL_HOST="--host=edx.devstack.mysql80" - SETTINGS="bok_choy_docker" -fi - -for i in "$@"; do - case $i in - -r|--rebuild_cache) - REBUILD_CACHE=true - ;; - -m|--migrations) - APPLY_MIGRATIONS=true - ;; - -c|--calculate_migrations) - CALCULATE_MIGRATIONS=true - ;; - -u|--use-existing-db) - USE_EXISTING_DB=true - ;; - esac -done - -declare -A databases -declare -a database_order -databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test") -database_order=("default" "student_module_history") - -calculate_migrations() { - echo "Calculating migrations for fingerprinting." - output_file="common/test/db_cache/bok_choy_${db}_migrations.yaml" - # Redirect stdout to /dev/null because the script will print - # out all migrations to both stdout and the output file. - ./manage.py lms --settings "$SETTINGS" show_unapplied_migrations --database "$db" --output_file "$output_file" 1>/dev/null -} - -run_migrations() { - echo "Running the lms migrations on the $db bok_choy DB." - ./manage.py lms --settings "$SETTINGS" migrate --database "$db" --traceback --noinput - echo "Running the cms migrations on the $db bok_choy DB." - ./manage.py cms --settings "$SETTINGS" migrate --database "$db" --traceback --noinput -} - -load_cache_into_db() { - echo "Loading the schema from the filesystem into the $db MySQL DB." - mysql "$MYSQL_HOST" -u root "${databases["$db"]}" < "$DB_CACHE_DIR/bok_choy_schema_$db.sql" - echo "Loading the fixture data from the filesystem into the $db MySQL DB." - ./manage.py lms --settings "$SETTINGS" loaddata --database "$db" "$DB_CACHE_DIR/bok_choy_data_$db.json" - echo "Loading the migration data from the filesystem into the $db MySQL DB." - mysql "$MYSQL_HOST" -u root "${databases["$db"]}" < "$DB_CACHE_DIR/bok_choy_migrations_data_$db.sql" -} - -rebuild_cache_for_db() { - # Make sure the DB has all migrations applied - run_migrations - - # Dump the schema and data to the cache - echo "Using the dumpdata command to save the $db fixture data to the filesystem." - ./manage.py lms --settings "$SETTINGS" dumpdata --database "$db" > "$DB_CACHE_DIR/bok_choy_data_$db.json" --exclude=api_admin.Catalog - echo "Saving the schema of the $db bok_choy DB to the filesystem." - mysqldump "$MYSQL_HOST" -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > "$DB_CACHE_DIR/bok_choy_schema_$db.sql" - - # dump_data does not dump the django_migrations table so we do it separately. - echo "Saving the django_migrations table of the $db bok_choy DB to the filesystem." - mysqldump $MYSQL_HOST -u root --no-create-info --skip-comments --skip-dump-date "${databases["$db"]}" django_migrations > "$DB_CACHE_DIR/bok_choy_migrations_data_$db.sql" -} - -for db in "${database_order[@]}"; do - if ! [[ $USE_EXISTING_DB ]]; then - echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql $MYSQL_HOST -u root - - # Clear out the test database - # - # We are using the reset_db command which uses "DROP DATABASE" and - # "CREATE DATABASE" in case the tests are being run in an environment (e.g. devstack - # or a jenkins worker environment) that already ran tests on another commit that had - # different migrations that created, dropped, or altered tables. - echo "Issuing a reset_db command to the $db bok_choy MySQL database." - ./manage.py lms --settings "$SETTINGS" reset_db --traceback --router "$db" - fi - - if ! [[ $CALCULATE_MIGRATIONS ]]; then - # If there are cached database schemas/data, then load them. - # If they are missing, then we will want to build new cache files even if - # not explicitly directed to do so via arguments passed to this script. - if [[ ! -f $DB_CACHE_DIR/bok_choy_schema_$db.sql || ! -f $DB_CACHE_DIR/bok_choy_data_$db.json || ! -f $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql ]]; then - REBUILD_CACHE=true - else - load_cache_into_db - fi - fi -done - -if [[ $REBUILD_CACHE ]]; then - echo "Cleaning the DB cache directory and building new files." - mkdir -p $DB_CACHE_DIR && rm -f $DB_CACHE_DIR/bok_choy* - - for db in "${database_order[@]}"; do - rebuild_cache_for_db - done -elif [[ $APPLY_MIGRATIONS ]]; then - for db in "${database_order[@]}"; do - run_migrations - done -elif [[ $CALCULATE_MIGRATIONS ]]; then - for db in "${database_order[@]}"; do - calculate_migrations - done -fi - From 01a4908562d56f03814dcae31506efe3202c7f05 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Wed, 4 Oct 2023 19:41:57 +0500 Subject: [PATCH 19/35] chore: fix tests --- .../pytest_plugin.py | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 openedx/testing/coverage_context_listener/pytest_plugin.py diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py deleted file mode 100644 index a3679027e8..0000000000 --- a/openedx/testing/coverage_context_listener/pytest_plugin.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -A pytest plugin that reports test contexts to coverage running in another process. -""" - -import pytest - - -class RemoteContextPlugin: - """ - Pytest plugin for reporting pytests contexts to coverage running in another process - """ - def __init__(self, config): - self.config = config - self.active = config.getoption("pytest-contexts") - - -@pytest.hookimpl(tryfirst=True) -def pytest_configure(config): - config.pluginmanager.register(RemoteContextPlugin(config), "remotecontextplugin") - - -def pytest_addoption(parser): - group = parser.getgroup("coverage") - group.addoption( - "--pytest-remote-contexts", - action="store_true", - dest="pytest-contexts", - help="Capture the pytest contexts that coverage is being captured in in another process", - ) From 5bc3dfa6fef97bec287fc129e885687d6c99247d Mon Sep 17 00:00:00 2001 From: salman2013 Date: Fri, 6 Oct 2023 11:21:11 +0500 Subject: [PATCH 20/35] chore: uncomment the code --- pavelib/utils/test/suites/js_suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pavelib/utils/test/suites/js_suite.py b/pavelib/utils/test/suites/js_suite.py index d96d95f39c..65c5feaf84 100644 --- a/pavelib/utils/test/suites/js_suite.py +++ b/pavelib/utils/test/suites/js_suite.py @@ -25,7 +25,7 @@ class JsTestSuite(TestSuite): self.opts = kwargs suite = args[0] - # self.subsuites = self._default_subsuites if suite == 'all' else [JsTestSubSuite(*args, **kwargs)] + self.subsuites = self._default_subsuites if suite == 'all' else [JsTestSubSuite(*args, **kwargs)] def __enter__(self): super().__enter__() From cda018e67bb82f8bbd9e6310dab6746e092a5e4e Mon Sep 17 00:00:00 2001 From: salman2013 Date: Fri, 6 Oct 2023 15:10:44 +0500 Subject: [PATCH 21/35] chore: fix test case --- pavelib/utils/envs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 4cfbec7790..69a9e06075 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -77,7 +77,7 @@ class Env: PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" # Detect if in a Docker container, and if so which one - # USING_DOCKER = SERVER_HOST != '0.0.0.0' + USING_DOCKER = SERVER_HOST != '0.0.0.0' DEVSTACK_SETTINGS = 'devstack' TEST_SETTINGS = 'test' @@ -85,6 +85,10 @@ class Env: TEST_DIR = REPO_ROOT / ".testids" # Configured browser to use for the js test suites + SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') + if USING_DOCKER: + KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' + else: KARMA_BROWSER = 'FirefoxNoUpdates' # Files used to run each of the js test suites From 7049cb2d2cb90ec810f0acca4c4c8bdfb09fc563 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Fri, 6 Oct 2023 15:21:27 +0500 Subject: [PATCH 22/35] chore: fix test case --- pavelib/utils/envs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 69a9e06075..af21118bb8 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -89,7 +89,7 @@ class Env: if USING_DOCKER: KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' else: - KARMA_BROWSER = 'FirefoxNoUpdates' + KARMA_BROWSER = 'FirefoxNoUpdates' # Files used to run each of the js test suites # TODO: Store this as a dict. Order seems to matter for some From 8dddb538fff56211f18c8a5d31d2d90b3df7492a Mon Sep 17 00:00:00 2001 From: salman2013 Date: Fri, 6 Oct 2023 15:27:15 +0500 Subject: [PATCH 23/35] chore: fix test case --- pavelib/utils/envs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index af21118bb8..1b82f9be75 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -85,6 +85,7 @@ class Env: TEST_DIR = REPO_ROOT / ".testids" # Configured browser to use for the js test suites + SERVER_HOST = '0.0.0.0' SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') if USING_DOCKER: KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' From 9d543254b0c3cb22ae3c4e63a8c08aff61d08f1e Mon Sep 17 00:00:00 2001 From: salman2013 Date: Fri, 6 Oct 2023 15:36:48 +0500 Subject: [PATCH 24/35] chore: fix test case --- pavelib/utils/envs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 1b82f9be75..eca00713ea 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -77,6 +77,7 @@ class Env: PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" # Detect if in a Docker container, and if so which one + SERVER_HOST = '0.0.0.0' USING_DOCKER = SERVER_HOST != '0.0.0.0' DEVSTACK_SETTINGS = 'devstack' TEST_SETTINGS = 'test' @@ -84,8 +85,7 @@ class Env: # Test Ids Directory TEST_DIR = REPO_ROOT / ".testids" - # Configured browser to use for the js test suites - SERVER_HOST = '0.0.0.0' + # Configured browser to use for the js test suites SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') if USING_DOCKER: KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' From 2d78b2d952b0ee9c37b9b45fcfc787770dcc2a0a Mon Sep 17 00:00:00 2001 From: salman2013 Date: Fri, 6 Oct 2023 16:56:17 +0500 Subject: [PATCH 25/35] chore: fix test case --- pavelib/utils/test/suites/js_suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pavelib/utils/test/suites/js_suite.py b/pavelib/utils/test/suites/js_suite.py index 65c5feaf84..8651ea4961 100644 --- a/pavelib/utils/test/suites/js_suite.py +++ b/pavelib/utils/test/suites/js_suite.py @@ -25,7 +25,7 @@ class JsTestSuite(TestSuite): self.opts = kwargs suite = args[0] - self.subsuites = self._default_subsuites if suite == 'all' else [JsTestSubSuite(*args, **kwargs)] + #self.subsuites = self._default_subsuites if suite == 'all' else [JsTestSubSuite(*args, **kwargs)] def __enter__(self): super().__enter__() From 52aea3529a4930c5183da8d1ae44262771fb764e Mon Sep 17 00:00:00 2001 From: salman2013 Date: Mon, 9 Oct 2023 11:29:57 +0500 Subject: [PATCH 26/35] chore: fix test case --- .github/workflows/js-tests.yml | 8 ++++++++ pavelib/utils/test/suites/js_suite.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index e4903af83b..c26e4e79fc 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -30,6 +30,14 @@ jobs: - name: Setup npm run: npm i -g npm@8.5.x + - name: Install Firefox 61.0 + run: | + sudo apt-get purge firefox + wget "https://ftp.mozilla.org/pub/firefox/releases/61.0/linux-x86_64/en-US/firefox-61.0.tar.bz2" + tar -xjf firefox-61.0.tar.bz2 + sudo mv firefox /opt/firefox + sudo ln -s /opt/firefox/firefox /usr/bin/firefox + - name: Install Required System Packages run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb diff --git a/pavelib/utils/test/suites/js_suite.py b/pavelib/utils/test/suites/js_suite.py index 8651ea4961..65c5feaf84 100644 --- a/pavelib/utils/test/suites/js_suite.py +++ b/pavelib/utils/test/suites/js_suite.py @@ -25,7 +25,7 @@ class JsTestSuite(TestSuite): self.opts = kwargs suite = args[0] - #self.subsuites = self._default_subsuites if suite == 'all' else [JsTestSubSuite(*args, **kwargs)] + self.subsuites = self._default_subsuites if suite == 'all' else [JsTestSubSuite(*args, **kwargs)] def __enter__(self): super().__enter__() From cdeaaa54584a9db9f14e784c279d937125a4b193 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Mon, 9 Oct 2023 19:12:10 +0500 Subject: [PATCH 27/35] chore: fix test case --- common/static/common/js/karma.common.conf.js | 43 ++++++++++++++++++++ pavelib/utils/envs.py | 2 +- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/common/static/common/js/karma.common.conf.js b/common/static/common/js/karma.common.conf.js index 274d4efa7e..09f6a21848 100644 --- a/common/static/common/js/karma.common.conf.js +++ b/common/static/common/js/karma.common.conf.js @@ -45,6 +45,7 @@ var appRoot = path.join(__dirname, '../../../../'); // eslint-disable-next-line import/no-extraneous-dependencies var webdriver = require('selenium-webdriver'); // eslint-disable-next-line import/no-extraneous-dependencies +var firefox = require('selenium-webdriver/firefox'); var webpackConfig = require(path.join(appRoot, 'webpack.dev.config.js')); @@ -296,6 +297,8 @@ function getBaseConfig(config, useRequireJs) { 'karma-requirejs', 'karma-junit-reporter', 'karma-coverage', + 'karma-chrome-launcher', + 'karma-firefox-launcher', 'karma-spec-reporter', 'karma-selenium-webdriver-launcher', 'karma-webpack', @@ -333,6 +336,46 @@ function getBaseConfig(config, useRequireJs) { // enable / disable watching file and executing tests whenever any file changes autoWatch: false, + + // start these browsers + // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher + browsers: ['FirefoxNoUpdates'], + + customLaunchers: { + // Firefox configuration that doesn't perform auto-updates + FirefoxNoUpdates: { + base: 'Firefox', + prefs: { + 'app.update.auto': false, + 'app.update.enabled': false + } + }, + ChromeDocker: { + base: 'SeleniumWebdriver', + browserName: 'chrome', + getDriver: function() { + return new webdriver.Builder() + .forBrowser('chrome') + .usingServer('http://edx.devstack.chrome:4444/wd/hub') + .build(); + } + }, + FirefoxDocker: { + base: 'SeleniumWebdriver', + browserName: 'firefox', + getDriver: function() { + var options = new firefox.Options(), + profile = new firefox.Profile(); + profile.setPreference('focusmanager.testmode', true); + options.setProfile(profile); + return new webdriver.Builder() + .forBrowser('firefox') + .usingServer('http://edx.devstack.firefox:4444/wd/hub') + .setFirefoxOptions(options) + .build(); + } + } + }, // Continuous Integration mode // if true, Karma captures browsers, runs the tests and exits diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index eca00713ea..20764a613e 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -85,7 +85,7 @@ class Env: # Test Ids Directory TEST_DIR = REPO_ROOT / ".testids" - # Configured browser to use for the js test suites + # Configured browser to use for the js test suites SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') if USING_DOCKER: KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' From 8bd5be2ab49f849cd33c660052449769e83c5799 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Mon, 9 Oct 2023 22:38:13 +0500 Subject: [PATCH 28/35] chore: fix test case --- pavelib/utils/envs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 20764a613e..53288780bd 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -79,7 +79,7 @@ class Env: # Detect if in a Docker container, and if so which one SERVER_HOST = '0.0.0.0' USING_DOCKER = SERVER_HOST != '0.0.0.0' - DEVSTACK_SETTINGS = 'devstack' + DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' TEST_SETTINGS = 'test' # Test Ids Directory From 8f97ac90cf86749a47729fab01559b4cbee45504 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Tue, 10 Oct 2023 09:56:28 +0500 Subject: [PATCH 29/35] chore: fix test case --- pavelib/utils/envs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 53288780bd..9ffe8887ff 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -82,6 +82,9 @@ class Env: DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' TEST_SETTINGS = 'test' + # Mongo databases that will be dropped before/after the tests run + MONGO_HOST = 'edx.devstack.mongo' if USING_DOCKER else 'localhost' + # Test Ids Directory TEST_DIR = REPO_ROOT / ".testids" From 2a4ce5fa48df7f07702b4e67ea46239a558b9a04 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Thu, 12 Oct 2023 20:11:25 +0500 Subject: [PATCH 30/35] chore: fix comments, remove un-used variable --- pavelib/paver_tests/test_utils.py | 1 - pavelib/prereqs.py | 10 ---------- pavelib/utils/envs.py | 11 +++-------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/pavelib/paver_tests/test_utils.py b/pavelib/paver_tests/test_utils.py index 7a157b04ea..ff468ecc9b 100644 --- a/pavelib/paver_tests/test_utils.py +++ b/pavelib/paver_tests/test_utils.py @@ -12,7 +12,6 @@ from pavelib.utils.envs import Env from pavelib.utils.test.utils import MINIMUM_FIREFOX_VERSION, check_firefox_version -@unittest.skipIf(Env.USING_DOCKER, 'Firefox version check works differently under Docker Devstack') class TestUtils(unittest.TestCase): """ Test utils.py under pavelib/utils/test diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py index a1b14c5445..f925fe1662 100644 --- a/pavelib/prereqs.py +++ b/pavelib/prereqs.py @@ -354,13 +354,3 @@ def install_prereqs(): def log_installed_python_prereqs(): """ Logs output of pip freeze for debugging. """ sh("pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log")) - - -def print_devstack_warning(): # lint-amnesty, pylint: disable=missing-function-docstring - if Env.USING_DOCKER: # pragma: no cover - print("********************************************************************************") - print("* WARNING: Mac users should run this from both the lms and studio shells") - print("* in docker devstack to avoid startup errors that kill your CPU.") - print("* For more details, see:") - print("* https://github.com/openedx/devstack#docker-is-using-lots-of-cpu-time-when-it-should-be-idle") - print("********************************************************************************") diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 9ffe8887ff..2202b34e19 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -77,23 +77,18 @@ class Env: PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" # Detect if in a Docker container, and if so which one - SERVER_HOST = '0.0.0.0' - USING_DOCKER = SERVER_HOST != '0.0.0.0' - DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' + DEVSTACK_SETTINGS = 'devstack' TEST_SETTINGS = 'test' # Mongo databases that will be dropped before/after the tests run - MONGO_HOST = 'edx.devstack.mongo' if USING_DOCKER else 'localhost' + MONGO_HOST = 'localhost' # Test Ids Directory TEST_DIR = REPO_ROOT / ".testids" # Configured browser to use for the js test suites SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') - if USING_DOCKER: - KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' - else: - KARMA_BROWSER = 'FirefoxNoUpdates' + KARMA_BROWSER = 'FirefoxNoUpdates' # Files used to run each of the js test suites # TODO: Store this as a dict. Order seems to matter for some From 2ff9ca8c1b2259ba053de9e6008d3787d4148112 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Mon, 23 Oct 2023 12:13:30 +0500 Subject: [PATCH 31/35] chore: fix comments, remove un-used selenium browser --- pavelib/utils/envs.py | 1 - tox.ini | 3 --- 2 files changed, 4 deletions(-) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 2202b34e19..47227eb7d8 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -87,7 +87,6 @@ class Env: TEST_DIR = REPO_ROOT / ".testids" # Configured browser to use for the js test suites - SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') KARMA_BROWSER = 'FirefoxNoUpdates' # Files used to run each of the js test suites diff --git a/tox.ini b/tox.ini index ce00efa870..96188f1780 100644 --- a/tox.ini +++ b/tox.ini @@ -40,9 +40,6 @@ passenv = NPM_CONFIG_PREFIX PYTHON_VERSION SCRAPY_SETTINGS_MODULE - SELENIUM_BROWSER - SELENIUM_HOST - SELENIUM_PORT SHARD SKIP_NPM_INSTALL SSH_AUTH_SOCK From e2560e163fc71ad59b8614a47d02eb5f8ffd122d Mon Sep 17 00:00:00 2001 From: salman2013 Date: Tue, 24 Oct 2023 09:17:33 +0500 Subject: [PATCH 32/35] chore: fix tests --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index 96188f1780..ce00efa870 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,9 @@ passenv = NPM_CONFIG_PREFIX PYTHON_VERSION SCRAPY_SETTINGS_MODULE + SELENIUM_BROWSER + SELENIUM_HOST + SELENIUM_PORT SHARD SKIP_NPM_INSTALL SSH_AUTH_SOCK From f71e650b2fe693e3150fdd62972c6a1c349cafc4 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Fri, 27 Oct 2023 09:09:55 +0500 Subject: [PATCH 33/35] chore: update bok-choy hostname and revert the old logic to chose browser for karma --- pavelib/utils/envs.py | 10 ++++++++-- tox.ini | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 47227eb7d8..cc20470d68 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -77,7 +77,9 @@ class Env: PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" # Detect if in a Docker container, and if so which one - DEVSTACK_SETTINGS = 'devstack' + SERVER_HOST = os.environ.get('SERVER_HOSTNAME', '0.0.0.0') + USING_DOCKER = SERVER_HOST != '0.0.0.0' + DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' TEST_SETTINGS = 'test' # Mongo databases that will be dropped before/after the tests run @@ -87,7 +89,11 @@ class Env: TEST_DIR = REPO_ROOT / ".testids" # Configured browser to use for the js test suites - KARMA_BROWSER = 'FirefoxNoUpdates' + SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') + if USING_DOCKER: + KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' + else: + KARMA_BROWSER = 'FirefoxNoUpdates' # Files used to run each of the js test suites # TODO: Store this as a dict. Order seems to matter for some diff --git a/tox.ini b/tox.ini index ce00efa870..0b8d8f967f 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,9 @@ setenv = PYTHONHASHSEED=0 TOXENV={envname} passenv = + SERVER_CMS_PORT + SERVER_HOSTNAME + SERVER_LMS_PORT DISABLE_COURSEENROLLMENT_HISTORY DISPLAY DJANGO_SETTINGS_MODULE From de935e146f9575e78b09c0584e38830644e852a3 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Mon, 30 Oct 2023 09:46:16 +0500 Subject: [PATCH 34/35] chore: update variable for frontend test server hostname --- pavelib/utils/envs.py | 4 ++-- tox.ini | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index cc20470d68..ed2553a42f 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -77,8 +77,8 @@ class Env: PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" # Detect if in a Docker container, and if so which one - SERVER_HOST = os.environ.get('SERVER_HOSTNAME', '0.0.0.0') - USING_DOCKER = SERVER_HOST != '0.0.0.0' + FRONTEND_TEST_SERVER_HOST = os.environ.get('FRONTEND_TEST_SERVER_HOSTNAME', '0.0.0.0') + USING_DOCKER = FRONTEND_TEST_SERVER_HOST != '0.0.0.0' DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' TEST_SETTINGS = 'test' diff --git a/tox.ini b/tox.ini index 0b8d8f967f..5000ca897b 100644 --- a/tox.ini +++ b/tox.ini @@ -26,9 +26,9 @@ setenv = PYTHONHASHSEED=0 TOXENV={envname} passenv = - SERVER_CMS_PORT - SERVER_HOSTNAME - SERVER_LMS_PORT + FRONTEND_TEST_CMS_PORT + FRONTEND_TEST_SERVER_HOSTNAME + FRONTEND_TEST_LMS_PORT DISABLE_COURSEENROLLMENT_HISTORY DISPLAY DJANGO_SETTINGS_MODULE From a97845294f2bb8420aa49ac0c3f21145136de1a8 Mon Sep 17 00:00:00 2001 From: salman2013 Date: Tue, 31 Oct 2023 15:11:42 +0500 Subject: [PATCH 35/35] chore: update lms,cms port variable names --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 5000ca897b..80eb6b5097 100644 --- a/tox.ini +++ b/tox.ini @@ -26,9 +26,9 @@ setenv = PYTHONHASHSEED=0 TOXENV={envname} passenv = - FRONTEND_TEST_CMS_PORT + FRONTEND_TEST_SERVER_CMS_PORT FRONTEND_TEST_SERVER_HOSTNAME - FRONTEND_TEST_LMS_PORT + FRONTEND_TEST_SERVER_LMS_PORT DISABLE_COURSEENROLLMENT_HISTORY DISPLAY DJANGO_SETTINGS_MODULE