diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index f7cd11bf2c..838d0fe261 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -99,7 +99,6 @@ "openedx/core/djangoapps/course_apps/", "openedx/core/djangoapps/course_date_signals/", "openedx/core/djangoapps/course_groups/", - "openedx/core/djangoapps/coursegraph/", "openedx/core/djangoapps/courseware_api/", "openedx/core/djangoapps/crawlers/", "openedx/core/djangoapps/credentials/", @@ -181,7 +180,6 @@ "openedx/core/djangoapps/course_apps/", "openedx/core/djangoapps/course_date_signals/", "openedx/core/djangoapps/course_groups/", - "openedx/core/djangoapps/coursegraph/", "openedx/core/djangoapps/courseware_api/", "openedx/core/djangoapps/crawlers/", "openedx/core/djangoapps/credentials/", @@ -240,6 +238,7 @@ "paths": [ "cms/djangoapps/api/", "cms/djangoapps/cms_user_tasks/", + "cms/djangoapps/coursegraph/", "cms/djangoapps/course_creators/", "cms/djangoapps/export_course_metadata/", "cms/djangoapps/maintenance/", diff --git a/cms/djangoapps/contentstore/admin.py b/cms/djangoapps/contentstore/admin.py index cd70873b4b..12f9dfcf6f 100644 --- a/cms/djangoapps/contentstore/admin.py +++ b/cms/djangoapps/contentstore/admin.py @@ -10,7 +10,7 @@ from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME from django.utils.translation import gettext as _ from edx_django_utils.admin.mixins import ReadOnlyAdminMixin -from cms.djangoapps.contentstore.models import VideoUploadConfig +from cms.djangoapps.contentstore.models import BackfillCourseTabsConfig, VideoUploadConfig from cms.djangoapps.contentstore.outlines_regenerate import CourseOutlineRegenerate from openedx.core.djangoapps.content.learning_sequences.api import key_supports_outlines @@ -78,5 +78,6 @@ class CourseOutlineRegenerateAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): return super().changelist_view(request, extra_context) +admin.site.register(BackfillCourseTabsConfig, ConfigurationModelAdmin) admin.site.register(VideoUploadConfig, ConfigurationModelAdmin) admin.site.register(CourseOutlineRegenerate, CourseOutlineRegenerateAdmin) diff --git a/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py b/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py index bcbea418f2..878a8dabaa 100644 --- a/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py +++ b/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py @@ -12,11 +12,12 @@ Search for the error message to detect any issues. import logging from django.core.management.base import BaseCommand - from xmodule.tabs import CourseTabList from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore +from cms.djangoapps.contentstore.models import BackfillCourseTabsConfig + logger = logging.getLogger(__name__) @@ -37,12 +38,19 @@ class Command(BaseCommand): if there are any new default course tabs. Else, makes no updates. """ store = modulestore() - course_keys = sorted( + all_course_keys = sorted( (course.id for course in store.get_course_summaries()), key=str # Different types of CourseKeys can't be compared without this. ) - logger.info(f'{len(course_keys)} courses read from modulestore.') + config = BackfillCourseTabsConfig.current() + start = config.start_index if config.enabled and config.start_index >= 0 else 0 + end = (start + config.count) if config.enabled and config.count > 0 else len(all_course_keys) + course_keys = all_course_keys[start:end] + + logger.info(f'{len(all_course_keys)} courses read from modulestore. Processing {start} to {end}.') + + error_keys = [] for course_key in course_keys: try: course = store.get_course(course_key, depth=1) @@ -59,3 +67,10 @@ class Command(BaseCommand): except Exception as err: # pylint: disable=broad-except logger.exception(err) logger.error(f'Course {course_key} encountered an Exception while trying to update.') + error_keys.append(course_key) + + if error_keys: + msg = 'The following courses encountered errors and were not updated:\n' + for error_key in error_keys: + msg += f' - {error_key}\n' + logger.info(msg) diff --git a/cms/djangoapps/contentstore/management/commands/tests/test_backfill_course_tabs.py b/cms/djangoapps/contentstore/management/commands/tests/test_backfill_course_tabs.py index 322f8f5064..258f00a133 100644 --- a/cms/djangoapps/contentstore/management/commands/tests/test_backfill_course_tabs.py +++ b/cms/djangoapps/contentstore/management/commands/tests/test_backfill_course_tabs.py @@ -3,13 +3,16 @@ Tests for `backfill_course_outlines` Studio (cms) management command. """ from unittest import mock +import ddt from django.core.management import call_command - from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory +from cms.djangoapps.contentstore.models import BackfillCourseTabsConfig + +@ddt.ddt class BackfillCourseTabsTest(ModuleStoreTestCase): """ Test `backfill_course_tabs` @@ -75,7 +78,7 @@ class BackfillCourseTabsTest(ModuleStoreTestCase): assert len(course.tabs) == 7 assert 'dates' in {tab.type for tab in course.tabs} assert 'progress' in {tab.type for tab in course.tabs} - mock_logger.info.assert_any_call('4 courses read from modulestore.') + mock_logger.info.assert_any_call('4 courses read from modulestore. Processing 0 to 4.') mock_logger.info.assert_any_call(f'Updating tabs for {course.id}.') mock_logger.info.assert_any_call(f'Successfully updated tabs for {course.id}.') assert mock_logger.info.call_count == 3 @@ -109,7 +112,7 @@ class BackfillCourseTabsTest(ModuleStoreTestCase): assert len(course_2.tabs) == 7 assert 'dates' in {tab.type for tab in course_1.tabs} assert 'progress' in {tab.type for tab in course_2.tabs} - mock_logger.info.assert_any_call('2 courses read from modulestore.') + mock_logger.info.assert_any_call('2 courses read from modulestore. Processing 0 to 2.') mock_logger.info.assert_any_call(f'Updating tabs for {course_1.id}.') mock_logger.info.assert_any_call(f'Successfully updated tabs for {course_1.id}.') mock_logger.info.assert_any_call(f'Updating tabs for {course_2.id}.') @@ -149,9 +152,29 @@ class BackfillCourseTabsTest(ModuleStoreTestCase): # Course wasn't updated due to the ValueError assert error_course_tabs_before == error_course_tabs_after - mock_logger.info.assert_any_call('2 courses read from modulestore.') + mock_logger.info.assert_any_call('2 courses read from modulestore. Processing 0 to 2.') mock_logger.info.assert_any_call(f'Successfully updated tabs for {updated_course.id}.') mock_logger.exception.assert_called() mock_logger.error.assert_called_once_with( f'Course {error_course.id} encountered an Exception while trying to update.' ) + + @ddt.data( + (1, 2, [False, True, True, False]), + (1, 0, [False, True, True, True]), + (-1, -1, [True, True, True, True]), + ) + @ddt.unpack + def test_arguments_batching(self, start, count, expected_tabs_modified): + courses = CourseFactory.create_batch(4) + for course in courses: + course.tabs = [tab for tab in course.tabs if tab.type in ('course_info', 'courseware')] + course = self.update_course(course, ModuleStoreEnum.UserID.test) + assert len(course.tabs) == 2 + + BackfillCourseTabsConfig.objects.create(enabled=True, start_index=start, count=count) + call_command('backfill_course_tabs') + + for i, course in enumerate(courses): + course = self.store.get_course(course.id) + assert len(course.tabs) == (7 if expected_tabs_modified[i] else 2), f'Wrong tabs for course index {i}' diff --git a/cms/djangoapps/contentstore/migrations/0007_backfillcoursetabsconfig.py b/cms/djangoapps/contentstore/migrations/0007_backfillcoursetabsconfig.py new file mode 100644 index 0000000000..2123798d83 --- /dev/null +++ b/cms/djangoapps/contentstore/migrations/0007_backfillcoursetabsconfig.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.12 on 2022-03-18 13:49 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('contentstore', '0006_courseoutlineregenerate'), + ] + + operations = [ + migrations.CreateModel( + name='BackfillCourseTabsConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')), + ('enabled', models.BooleanField(default=False, verbose_name='Enabled')), + ('start_index', models.IntegerField(default=0, help_text='Index of first course to start backfilling (in an alphabetically sorted list of courses)')), + ('count', models.IntegerField(default=0, help_text='How many courses to backfill in this run (or zero for all courses)')), + ('changed_by', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL, verbose_name='Changed by')), + ], + options={ + 'verbose_name': 'Arguments for backfill_course_tabs', + 'verbose_name_plural': 'Arguments for backfill_course_tabs', + }, + ), + ] diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index f6817ac59f..66d884d8dd 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -4,7 +4,7 @@ Models for contentstore from config_models.models import ConfigurationModel -from django.db.models.fields import TextField +from django.db.models.fields import IntegerField, TextField class VideoUploadConfig(ConfigurationModel): @@ -22,3 +22,24 @@ class VideoUploadConfig(ConfigurationModel): def get_profile_whitelist(cls): """Get the list of profiles to include in the encoding download""" return [profile for profile in cls.current().profile_whitelist.split(",") if profile] + + +class BackfillCourseTabsConfig(ConfigurationModel): + """ + Manages configuration for a run of the backfill_course_tabs management command. + + .. no_pii: + """ + + class Meta: + verbose_name = 'Arguments for backfill_course_tabs' + verbose_name_plural = 'Arguments for backfill_course_tabs' + + start_index = IntegerField( + help_text='Index of first course to start backfilling (in an alphabetically sorted list of courses)', + default=0, + ) + count = IntegerField( + help_text='How many courses to backfill in this run (or zero for all courses)', + default=0, + ) diff --git a/cms/djangoapps/contentstore/signals/handlers.py b/cms/djangoapps/contentstore/signals/handlers.py index a67f958381..9945277a13 100644 --- a/cms/djangoapps/contentstore/signals/handlers.py +++ b/cms/djangoapps/contentstore/signals/handlers.py @@ -5,6 +5,7 @@ import logging from datetime import datetime from functools import wraps +from django.conf import settings from django.core.cache import cache from django.dispatch import receiver from pytz import UTC @@ -55,6 +56,9 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable= update_search_index, update_special_exams_and_publish ) + from cms.djangoapps.coursegraph.tasks import ( + dump_course_to_neo4j + ) # register special exams asynchronously course_key_str = str(course_key) @@ -64,6 +68,10 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable= # Push the course outline to learning_sequences asynchronously. update_outline_from_modulestore_task.delay(course_key_str) + if settings.COURSEGRAPH_DUMP_COURSE_ON_PUBLISH: + # Push the course out to CourseGraph asynchronously. + dump_course_to_neo4j.delay(course_key_str) + # Finally, call into the course search subsystem # to kick off an indexing action if CoursewareSearchIndexer.indexing_is_enabled() and CourseAboutSearchIndexer.indexing_is_enabled(): diff --git a/cms/djangoapps/contentstore/tests/test_admin.py b/cms/djangoapps/contentstore/tests/test_admin.py index d0a306ba11..43e3c23369 100644 --- a/cms/djangoapps/contentstore/tests/test_admin.py +++ b/cms/djangoapps/contentstore/tests/test_admin.py @@ -6,9 +6,6 @@ This is not inside a django app because it is a global property of the system. import ddt from django.test import TestCase from django.urls import reverse -from edx_toggles.toggles.testutils import override_waffle_flag - -from openedx.core.djangoapps.user_authn.config.waffle import ADMIN_AUTH_REDIRECT_TO_LMS @ddt.ddt @@ -16,17 +13,8 @@ class TestAdminView(TestCase): """ Tests of the admin view. """ - @override_waffle_flag(ADMIN_AUTH_REDIRECT_TO_LMS, True) @ddt.data('/admin/', '/admin/login', reverse('admin:login')) def test_admin_login_redirect(self, admin_url): """Admin login will redirect towards the site login page.""" response = self.client.get(admin_url, follow=True) assert any('/login/edx-oauth2/?next=' in r[0] for r in response.redirect_chain) - - def test_admin_login_default(self): - """Without flag Admin login will redirect towards the admin default login page.""" - response = self.client.get('/admin/', follow=True) - assert response.status_code == 200 - self.assertIn('/admin/login/?next=/admin/', response.redirect_chain[0]) - assert len(response.redirect_chain) == 1 - assert response.template_name == ['admin/login.html'] diff --git a/cms/djangoapps/contentstore/views/public.py b/cms/djangoapps/contentstore/views/public.py index 823bcd4154..2d5e69bc23 100644 --- a/cms/djangoapps/contentstore/views/public.py +++ b/cms/djangoapps/contentstore/views/public.py @@ -7,10 +7,8 @@ from django.conf import settings from django.shortcuts import redirect from urllib.parse import quote_plus # lint-amnesty, pylint: disable=wrong-import-order from waffle.decorators import waffle_switch -from django.contrib import admin from common.djangoapps.edxmako.shortcuts import render_to_response -from openedx.core.djangoapps.user_authn.config.waffle import ADMIN_AUTH_REDIRECT_TO_LMS from ..config import waffle @@ -48,10 +46,7 @@ def redirect_to_lms_login_for_admin(request): """ This view redirect the admin/login url to the site's login page. """ - if ADMIN_AUTH_REDIRECT_TO_LMS.is_enabled(): - return redirect('/login?next=/admin') - else: - return admin.site.login(request) + return redirect('/login?next=/admin') def _build_next_param(request): diff --git a/cms/djangoapps/coursegraph/README.rst b/cms/djangoapps/coursegraph/README.rst new file mode 100644 index 0000000000..18f0f60bdd --- /dev/null +++ b/cms/djangoapps/coursegraph/README.rst @@ -0,0 +1,120 @@ + +CourseGraph Support +------------------- + +This app exists to write data to "CourseGraph", a tool enabling Open edX developers and support specialists to inspect their platform instance's learning content. CourseGraph itself is simply an instance of `Neo4j`_, which is an open-source graph database with a Web interface. + +.. _Neo4j: https://neo4j.com + +Deploying Coursegraph +===================== + +There are two ways to deploy CourseGraph: + +* For operators using Tutor, there is a `CourseGraph plugin for Tutor`_ that is currently released as "Beta". Nutmeg is the earliest Open edX release that the plugin will work alongside. + +* For operators still using the old Ansible installation pathway, there exists a `neo4j Ansible playbook`_. Be warned that this method is not well-documented nor officially supported. + +In order for CourseGraph to have queryable, up-to-date data, learning content from CMS must be written to CourseGraph regularly. That is where this Django app comes into play. For details on the various ways to write CMS data to CourseGraph, visit the `operations section of the CourseGraph Tutor plugin docs`_. + +**Please note**: Access to a populated CourseGraph instance confers access to all the learning content in the associated Open edX CMS (Studio). The basic authentication provided by Neo4j may or may not be sufficient for your security needs. Consider taking additional security measures, such as restricting CourseGraph access to only users on a private VPN. + +.. _neo4j Ansible playbook: https://github.com/edx/configuration/blob/master/playbooks/neo4j.yml + +.. _CourseGraph plugin for Tutor: https://github.com/openedx/tutor-contrib-coursegraph/ + +.. _operations section of the CourseGraph Tutor plugin docs: https://github.com/openedx/tutor-contrib-coursegraph/#managing-data + +Running CourseGraph locally +=========================== + +In some circumstances, you may want to run CourseGraph locally, connected to a development-mode Open edX instance. You can do this in both Tutor and Devstack. + +Tutor +***** + +The `CourseGraph plugin for Tutor`_ makes it easy to install, configure, and run CourseGraph for local development. + +Devstack +******** + +CourseGraph is included as an "extra" component in the `Open edX Devstack`_. That is, it is not run or provisioned by default, but can be enabled on-demand. + +To provision Devstack CourseGraph with data from Devstack LMS, run:: + + make dev.provision.coursegraph + +CourseGraph should now be accessible at http://localhost:7474 with the username ``neo4j`` and the password ``edx``. + +Under the hood, the provisioning command just invokes ``dump_to_neo4j`` on your LMS, pointed at your CourseGraph. The provisioning command can be run again at any point in the future to refresh CourseGraph with new LMS data. The data in CourseGraph will persist unless you explicitly destroy it (as noted below). + +Other Devstack CourseGraph commands include:: + + make dev.up.coursegraph # Bring up the container (without re-provisioning). + make dev.down.coursegraph # Stop and remove the container. + make dev.shell.coursegraph # Start a shell session in the container. + make dev.attach.coursegraph # Attach to the container. + make dev.destroy.coursegraph # Stop the container and destroy its database. + +The above commands should be run in your ``devstack`` folder, and they assume that LMS is already properly provisioned. See the `Devstack interface`_ for more details. + +.. _Open edX Devstack: https://github.com/edx/devstack/ +.. _Devstack interface: https://edx.readthedocs.io/projects/open-edx-devstack/en/latest/devstack_interface.html + + +Querying Coursegraph +==================== + +CourseGraph is queryable using the `Cypher`_ query language. Open edX learning content is represented in Neo4j using a straightforward scheme: + +* A node is an XBlock usage. + +* Nodes are tagged with their ``block_type``, such as: + + * ``course`` + * ``chapter`` + * ``sequential`` + * ``vertical`` + * ``problem`` + * ``html`` + * etc. + +* Every node is also tagged with ``item``. + +* Parent-child relationships in the course hierarchy are reflected in the ``PARENT_OF`` relationship. + +* Ordered sibling relationships in the course hierarchy are reflected in the ``PRECEDES`` relationship. + +* Fields on each XBlock usage (``.display_name``, ``.data``, etc) are available on the corresponding node. + +.. _Cypher: https://neo4j.com/developer/cypher/ + + +Example Queries +*************** + +How many XBlocks exist in the LMS, by type? :: + + MATCH + (c:course) -[:PARENT_OF*]-> (n:item) + RETURN + distinct(n.block_type) as block_type, + count(n) as number + order by + number DESC + + +In a given course, which units contain problems with custom Python grading code? :: + + MATCH + (c:course) -[:PARENT_OF*]-> (u:vertical) -[:PARENT_OF*]-> (p:problem) + WHERE + p.data CONTAINS 'loncapa/python' + AND + c.course_key = '' + RETURN + u.location + +You can see many more examples of useful CourseGraph queries on the `query archive wiki page`_. + +.. _query archive wiki page: https://openedx.atlassian.net/wiki/spaces/COMM/pages/3273228388/Useful+CourseGraph+Queries diff --git a/openedx/core/djangoapps/coursegraph/__init__.py b/cms/djangoapps/coursegraph/__init__.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/__init__.py rename to cms/djangoapps/coursegraph/__init__.py diff --git a/cms/djangoapps/coursegraph/admin.py b/cms/djangoapps/coursegraph/admin.py new file mode 100644 index 0000000000..f79fa909d2 --- /dev/null +++ b/cms/djangoapps/coursegraph/admin.py @@ -0,0 +1,123 @@ +""" +Admin site bindings for coursegraph +""" +import logging + +from django.contrib import admin, messages +from django.utils.translation import gettext as _ +from edx_django_utils.admin.mixins import ReadOnlyAdminMixin + +from .models import CourseGraphCourseDump +from .tasks import ModuleStoreSerializer + +log = logging.getLogger(__name__) + + +@admin.action( + permissions=['change'], + description=_("Dump courses to CourseGraph (respect cache)"), +) +def dump_courses(modeladmin, request, queryset): + """ + Admin action to enqueue Dump-to-CourseGraph tasks for a set of courses, + excluding courses that haven't been published since they were last dumped. + + queryset is a QuerySet of CourseGraphCourseDump objects, which are just + CourseOverview objects under the hood. + """ + all_course_keys = queryset.values_list('id', flat=True) + serializer = ModuleStoreSerializer(all_course_keys) + try: + submitted, skipped = serializer.dump_courses_to_neo4j() + # Unfortunately there is no unified base class for the reasonable + # exceptions we could expect from py2neo (connection unavailable, bolt protocol + # error, and so on), so we just catch broadly, show a generic error banner, + # and then log the exception for site operators to look at. + except Exception as err: # pylint: disable=broad-except + log.exception( + "Failed to enqueue CourseGraph dumps to Neo4j (respecting cache): %s", + ", ".join(str(course_key) for course_key in all_course_keys), + ) + modeladmin.message_user( + request, + _("Error enqueueing dumps for {} course(s): {}").format( + len(all_course_keys), str(err) + ), + level=messages.ERROR, + ) + return + if submitted: + modeladmin.message_user( + request, + _( + "Enqueued dumps for {} course(s). Skipped {} unchanged course(s)." + ).format(len(submitted), len(skipped)), + level=messages.SUCCESS, + ) + else: + modeladmin.message_user( + request, + _( + "Skipped all {} course(s), as they were unchanged.", + ).format(len(skipped)), + level=messages.WARNING, + ) + + +@admin.action( + permissions=['change'], + description=_("Dump courses to CourseGraph (override cache)") +) +def dump_courses_overriding_cache(modeladmin, request, queryset): + """ + Admin action to enqueue Dump-to-CourseGraph tasks for a set of courses + (whether or not they have been published recently). + + queryset is a QuerySet of CourseGraphCourseDump objects, which are just + CourseOverview objects under the hood. + """ + all_course_keys = queryset.values_list('id', flat=True) + serializer = ModuleStoreSerializer(all_course_keys) + try: + submitted, _skipped = serializer.dump_courses_to_neo4j(override_cache=True) + # Unfortunately there is no unified base class for the reasonable + # exceptions we could expect from py2neo (connection unavailable, bolt protocol + # error, and so on), so we just catch broadly, show a generic error banner, + # and then log the exception for site operators to look at. + except Exception as err: # pylint: disable=broad-except + log.exception( + "Failed to enqueue CourseGraph Neo4j course dumps (overriding cache): %s", + ", ".join(str(course_key) for course_key in all_course_keys), + ) + modeladmin.message_user( + request, + _("Error enqueueing dumps for {} course(s): {}").format( + len(all_course_keys), str(err) + ), + level=messages.ERROR, + ) + return + modeladmin.message_user( + request, + _("Enqueued dumps for {} course(s).").format(len(submitted)), + level=messages.SUCCESS, + ) + + +@admin.register(CourseGraphCourseDump) +class CourseGraphCourseDumpAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): + """ + Model admin for "Course graph course dumps". + + Just a read-only table with some useful metadata, allowing admin users to + select courses to be dumped to CourseGraph. + """ + list_display = [ + 'id', + 'display_name', + 'modified', + 'enrollment_start', + 'enrollment_end', + ] + search_fields = ['id', 'display_name'] + actions = [dump_courses, dump_courses_overriding_cache] diff --git a/openedx/core/djangoapps/coursegraph/apps.py b/cms/djangoapps/coursegraph/apps.py similarity index 65% rename from openedx/core/djangoapps/coursegraph/apps.py rename to cms/djangoapps/coursegraph/apps.py index ecedf33d19..95d7873fce 100644 --- a/openedx/core/djangoapps/coursegraph/apps.py +++ b/cms/djangoapps/coursegraph/apps.py @@ -12,6 +12,6 @@ class CoursegraphConfig(AppConfig): """ AppConfig for courseware app """ - name = 'openedx.core.djangoapps.coursegraph' + name = 'cms.djangoapps.coursegraph' - from openedx.core.djangoapps.coursegraph import tasks + from cms.djangoapps.coursegraph import tasks diff --git a/openedx/core/djangoapps/coursegraph/management/__init__.py b/cms/djangoapps/coursegraph/management/__init__.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/management/__init__.py rename to cms/djangoapps/coursegraph/management/__init__.py diff --git a/openedx/core/djangoapps/coursegraph/management/commands/__init__.py b/cms/djangoapps/coursegraph/management/commands/__init__.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/management/commands/__init__.py rename to cms/djangoapps/coursegraph/management/commands/__init__.py diff --git a/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py new file mode 100644 index 0000000000..40afe7ffbe --- /dev/null +++ b/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py @@ -0,0 +1,114 @@ +""" +This file contains a management command for exporting the modulestore to +Neo4j, a graph database. + +Example usages: + + # Dump all courses published since last dump. + # Use connection parameters from `settings.COURSEGRAPH_SETTINGS`. + python manage.py cms dump_to_neo4j + + # Dump all courses published since last dump. + # Use custom connection parameters. + python manage.py cms dump_to_neo4j --host localhost --port 7473 \ + --secure --user user --password password + + # Specify certain courses instead of dumping all of them. + # Use connection parameters from `settings.COURSEGRAPH_SETTINGS`. + python manage.py cms dump_to_neo4j --courses 'course-v1:A+B+1' 'course-v1:A+B+2' +""" + + +import logging +from textwrap import dedent + +from django.core.management.base import BaseCommand + +from cms.djangoapps.coursegraph.tasks import ModuleStoreSerializer + +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + Dump recently-published course(s) over to a CourseGraph (Neo4j) instance. + """ + help = dedent(__doc__).strip() + + def add_arguments(self, parser): + parser.add_argument( + '--host', + type=str, + help="the hostname of the Neo4j server", + ) + parser.add_argument( + '--port', + type=int, + help="the port on the Neo4j server that accepts Bolt requests", + ) + parser.add_argument( + '--secure', + action='store_true', + help="connect to server over Bolt/TLS instead of plain unencrypted Bolt", + ) + parser.add_argument( + '--user', + type=str, + help="the username of the Neo4j user", + ) + parser.add_argument( + '--password', + type=str, + help="the password of the Neo4j user", + ) + parser.add_argument( + '--courses', + metavar='KEY', + type=str, + nargs='*', + help="keys of courses to serialize; if omitted all courses in system are serialized", + ) + parser.add_argument( + '--skip', + metavar='KEY', + type=str, + nargs='*', + help="keys of courses to NOT to serialize", + ) + parser.add_argument( + '--override', + action='store_true', + help="dump all courses regardless of when they were last published", + ) + + def handle(self, *args, **options): + """ + Iterates through each course, serializes them into graphs, and saves + those graphs to neo4j. + """ + + mss = ModuleStoreSerializer.create(options['courses'], options['skip']) + connection_overrides = { + key: options[key] + for key in ["host", "port", "secure", "user", "password"] + } + submitted_courses, skipped_courses = mss.dump_courses_to_neo4j( + connection_overrides=connection_overrides, + override_cache=options['override'], + ) + + log.info( + "%d courses submitted for export to neo4j. %d courses skipped.", + len(submitted_courses), + len(skipped_courses), + ) + + if not submitted_courses: + print("No courses submitted for export to neo4j at all!") + return + + if submitted_courses: + print( + "These courses were submitted for export to neo4j successfully:\n\t" + + "\n\t".join(submitted_courses) + ) diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/__init__.py b/cms/djangoapps/coursegraph/management/commands/tests/__init__.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/management/commands/tests/__init__.py rename to cms/djangoapps/coursegraph/management/commands/tests/__init__.py diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py similarity index 83% rename from openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py rename to cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py index 61349eda8a..32fef0a887 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py +++ b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py @@ -8,15 +8,16 @@ from datetime import datetime from unittest import mock import ddt from django.core.management import call_command +from django.test.utils import override_settings from edx_toggles.toggles.testutils import override_waffle_switch from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory import openedx.core.djangoapps.content.block_structure.config as block_structure_config from openedx.core.djangoapps.content.block_structure.signals import update_block_structure_on_course_publish -from openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j import ModuleStoreSerializer -from openedx.core.djangoapps.coursegraph.management.commands.tests.utils import MockGraph, MockNodeMatcher -from openedx.core.djangoapps.coursegraph.tasks import ( +from cms.djangoapps.coursegraph.management.commands.dump_to_neo4j import ModuleStoreSerializer +from cms.djangoapps.coursegraph.management.commands.tests.utils import MockGraph, MockNodeMatcher +from cms.djangoapps.coursegraph.tasks import ( coerce_types, serialize_course, serialize_item, @@ -115,8 +116,8 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): Tests for the dump to neo4j management command """ - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') @ddt.data(1, 2) def test_dump_specific_courses(self, number_of_courses, mock_graph_class, mock_matcher_class): """ @@ -140,8 +141,8 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0 ) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') def test_dump_skip_course(self, mock_graph_class, mock_matcher_class): """ Test that you can skip courses. @@ -166,8 +167,8 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0, ) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') def test_dump_skip_beats_specifying(self, mock_graph_class, mock_matcher_class): """ Test that if you skip and specify the same course, you'll skip it. @@ -193,8 +194,8 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0, ) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') def test_dump_all_courses(self, mock_graph_class, mock_matcher_class): """ Test if you don't specify which courses to dump, then you'll dump @@ -219,6 +220,48 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0, ) + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.Graph', autospec=True) + @override_settings( + COURSEGRAPH_CONNECTION=dict( + protocol='bolt', + host='coursegraph.example.edu', + port=7777, + secure=True, + user="neo4j", + password="default-password", + ) + ) + def test_dump_to_neo4j_connection_defaults(self, mock_graph_class, mock_matcher_class): + """ + Test that user can override individual settings.COURSEGRAPH_CONNECTION parameters + by passing them to `dump_to_neo4j`, whilst falling back to the ones that they + don't override. + """ + self.setup_mock_graph( + mock_matcher_class, mock_graph_class + ) + call_command( + 'dump_to_neo4j', + courses=self.course_strings[:1], + port=7788, + secure=False, + password="overridden-password", + ) + assert mock_graph_class.call_args.args == () + assert mock_graph_class.call_args.kwargs == dict( + + # From settings: + protocol='bolt', + host='coursegraph.example.edu', + user="neo4j", + + # Overriden by command: + port=7788, + secure=False, + password="overridden-password", + ) + class SomeThing: """Just to test the stringification of an object.""" @@ -395,8 +438,8 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): coerced_value = coerce_types(original_value) assert coerced_value == coerced_expected - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') def test_dump_to_neo4j(self, mock_graph_constructor, mock_matcher_class): """ Tests the dump_to_neo4j method works against a mock @@ -423,8 +466,8 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): assert len(mock_graph.nodes) == 11 self.assertCountEqual(submitted, self.course_strings) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') def test_dump_to_neo4j_rollback(self, mock_graph_constructor, mock_matcher_class): """ Tests that the the dump_to_neo4j method handles the case where there's @@ -447,8 +490,8 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): self.assertCountEqual(submitted, self.course_strings) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') @ddt.data((True, 2), (False, 0)) @ddt.unpack def test_dump_to_neo4j_cache( @@ -480,8 +523,8 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): ) assert len(submitted) == expected_number_courses - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') def test_dump_to_neo4j_published(self, mock_graph_constructor, mock_matcher_class): """ Tests that we only dump those courses that have been published after @@ -506,14 +549,32 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): assert len(submitted) == 1 assert submitted[0] == str(self.course.id) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.get_course_last_published') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.get_command_last_run') + @mock.patch('cms.djangoapps.coursegraph.tasks.get_course_last_published') + @mock.patch('cms.djangoapps.coursegraph.tasks.get_command_last_run') @ddt.data( - (str(datetime(2016, 3, 30)), str(datetime(2016, 3, 31)), True), - (str(datetime(2016, 3, 31)), str(datetime(2016, 3, 30)), False), - (str(datetime(2016, 3, 31)), None, False), - (None, str(datetime(2016, 3, 30)), True), - (None, None, True), + ( + str(datetime(2016, 3, 30)), str(datetime(2016, 3, 31)), + (True, ( + 'course has been published since last neo4j update time - ' + 'update date 2016-03-30 00:00:00 < published date 2016-03-31 00:00:00' + )) + ), + ( + str(datetime(2016, 3, 31)), str(datetime(2016, 3, 30)), + (False, None) + ), + ( + str(datetime(2016, 3, 31)), None, + (False, None) + ), + ( + None, str(datetime(2016, 3, 30)), + (True, 'no record of the last neo4j update time for the course') + ), + ( + None, None, + (True, 'no record of the last neo4j update time for the course') + ), ) @ddt.unpack def test_should_dump_course( diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py b/cms/djangoapps/coursegraph/management/commands/tests/utils.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py rename to cms/djangoapps/coursegraph/management/commands/tests/utils.py diff --git a/cms/djangoapps/coursegraph/models.py b/cms/djangoapps/coursegraph/models.py new file mode 100644 index 0000000000..f053dc9993 --- /dev/null +++ b/cms/djangoapps/coursegraph/models.py @@ -0,0 +1,21 @@ +""" +(Proxy) models supporting CourseGraph. +""" + +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview + + +class CourseGraphCourseDump(CourseOverview): + """ + Proxy model for CourseOverview. + + Does *not* create/update/delete CourseOverview objects - only reads the objects. + Uses the course IDs of the CourseOverview objects to determine which courses + can be dumped to CourseGraph. + """ + class Meta: + proxy = True + + def __str__(self): + """Represent ourselves with the course key.""" + return str(self.id) diff --git a/openedx/core/djangoapps/coursegraph/tasks.py b/cms/djangoapps/coursegraph/tasks.py similarity index 76% rename from openedx/core/djangoapps/coursegraph/tasks.py rename to cms/djangoapps/coursegraph/tasks.py index f43384990f..e2d4bf5b09 100644 --- a/openedx/core/djangoapps/coursegraph/tasks.py +++ b/cms/djangoapps/coursegraph/tasks.py @@ -7,6 +7,7 @@ neo4j, a graph database. import logging from celery import shared_task +from django.conf import settings from django.utils import timezone from edx_django_utils.cache import RequestCache from edx_django_utils.monitoring import set_code_owner_attribute @@ -133,29 +134,26 @@ def get_command_last_run(course_key, graph): def get_course_last_published(course_key): """ - We use the CourseStructure table to get when this course was last - published. + Approximately when was a course last published? + + We use the 'modified' column in the CourseOverview table as a quick and easy + (although perhaps inexact) way of determining when a course was last + published. This works because CourseOverview rows are re-written upon + course publish. + Args: course_key: a CourseKey - Returns: The datetime the course was last published at, converted into - text, or None, if there's no record of the last time this course - was published. + Returns: The datetime the course was last published at, stringified. + Uses Python's default str(...) implementation for datetimes, which + is sortable and similar to ISO 8601: + https://docs.python.org/3/library/datetime.html#datetime.date.__str__ """ # Import is placed here to avoid model import at project startup. - from xmodule.modulestore.django import modulestore - from openedx.core.djangoapps.content.block_structure.models import BlockStructureModel - from openedx.core.djangoapps.content.block_structure.exceptions import BlockStructureNotFound + from openedx.core.djangoapps.content.course_overviews.models import CourseOverview - store = modulestore() - course_usage_key = store.make_course_usage_key(course_key) - try: - structure = BlockStructureModel.get(course_usage_key) - course_last_published_date = str(structure.modified) - except BlockStructureNotFound: - course_last_published_date = None - - return course_last_published_date + approx_last_published = CourseOverview.get_from_id(course_key).modified + return str(approx_last_published) def strip_branch_and_version(location): @@ -231,7 +229,9 @@ def should_dump_course(course_key, graph): course_key: a CourseKey object. graph: a py2neo Graph object. - Returns: bool of whether this course should be dumped to neo4j. + Returns: + - whether this course should be dumped to neo4j (bool) + - reason why course needs to be dumped (string, None if doesn't need to be dumped) """ last_this_command_was_run = get_command_last_run(course_key, graph) @@ -241,29 +241,39 @@ def should_dump_course(course_key, graph): # if we don't have a record of the last time this command was run, # we should serialize the course and dump it if last_this_command_was_run is None: - return True + return ( + True, + "no record of the last neo4j update time for the course" + ) # if we've serialized the course recently and we have no published # events, we will not dump it, and so we can skip serializing it # again here if last_this_command_was_run and course_last_published_date is None: - return False + return (False, None) # otherwise, serialize and dump the course if the command was run # before the course's last published event - return last_this_command_was_run < course_last_published_date + needs_update = last_this_command_was_run < course_last_published_date + update_reason = None + if needs_update: + update_reason = ( + f"course has been published since last neo4j update time - " + f"update date {last_this_command_was_run} < published date {course_last_published_date}" + ) + return (needs_update, update_reason) @shared_task @set_code_owner_attribute -def dump_course_to_neo4j(course_key_string, credentials): +def dump_course_to_neo4j(course_key_string, connection_overrides=None): """ Serializes a course and writes it to neo4j. Arguments: - course_key: course key for the course to be exported - credentials (dict): the necessary credentials to connect - to neo4j and create a py2neo `Graph` obje + course_key_string: course key for the course to be exported + connection_overrides (dict): overrides to Neo4j connection + parameters specified in `settings.COURSEGRAPH_CONNECTION`. """ course_key = CourseKey.from_string(course_key_string) nodes, relationships = serialize_course(course_key) @@ -274,7 +284,9 @@ def dump_course_to_neo4j(course_key_string, credentials): len(relationships), ) - graph = authenticate_and_create_graph(credentials) + graph = authenticate_and_create_graph( + connection_overrides=connection_overrides + ) transaction = graph.begin() course_string = str(course_key) @@ -334,13 +346,13 @@ class ModuleStoreSerializer: course_keys = [course_key for course_key in course_keys if course_key not in skip_keys] return cls(course_keys) - def dump_courses_to_neo4j(self, credentials, override_cache=False): + def dump_courses_to_neo4j(self, connection_overrides=None, override_cache=False): """ Method that iterates through a list of courses in a modulestore, serializes them, then submits tasks to write them to neo4j. Arguments: - credentials (dict): the necessary credentials to connect - to neo4j and create a py2neo `Graph` object + connection_overrides (dict): overrides to Neo4j connection + parameters specified in `settings.COURSEGRAPH_CONNECTION`. override_cache: serialize the courses even if they'be been recently serialized @@ -353,55 +365,56 @@ class ModuleStoreSerializer: submitted_courses = [] skipped_courses = [] - graph = authenticate_and_create_graph(credentials) + graph = authenticate_and_create_graph(connection_overrides) for index, course_key in enumerate(self.course_keys): # first, clear the request cache to prevent memory leaks RequestCache.clear_all_namespaces() - log.info( - "Now submitting %s for export to neo4j: course %d of %d total courses", - course_key, - index + 1, - total_number_of_courses, - ) - - if not (override_cache or should_dump_course(course_key, graph)): + (needs_dump, reason) = should_dump_course(course_key, graph) + if not (override_cache or needs_dump): log.info("skipping submitting %s, since it hasn't changed", course_key) skipped_courses.append(str(course_key)) continue + if override_cache: + reason = "override_cache is True" + + log.info( + "Now submitting %s for export to neo4j, because %s: course %d of %d total courses", + course_key, + reason, + index + 1, + total_number_of_courses, + ) + dump_course_to_neo4j.apply_async( - args=[str(course_key), credentials], + kwargs=dict( + course_key_string=str(course_key), + connection_overrides=connection_overrides, + ) ) submitted_courses.append(str(course_key)) return submitted_courses, skipped_courses -def authenticate_and_create_graph(credentials): +def authenticate_and_create_graph(connection_overrides=None): """ This function authenticates with neo4j and creates a py2neo graph object + Arguments: - credentials (dict): a dictionary of credentials used to authenticate, - and then create, a py2neo graph object. + connection_overrides (dict): overrides to Neo4j connection + parameters specified in `settings.COURSEGRAPH_CONNECTION`. Returns: a py2neo `Graph` object. """ - - host = credentials['host'] - port = credentials['port'] - secure = credentials['secure'] - neo4j_user = credentials['user'] - neo4j_password = credentials['password'] - - graph = Graph( - protocol='bolt', - password=neo4j_password, - user=neo4j_user, - address=host, - port=port, - secure=secure, - ) - - return graph + provided_overrides = { + key: value + for key, value in (connection_overrides or {}).items() + # Drop overrides whose values are `None`. Note that `False` is a + # legitimate override value that we don't want to drop here. + if value is not None + } + connection_with_overrides = {**settings.COURSEGRAPH_CONNECTION, **provided_overrides} + return Graph(**connection_with_overrides) diff --git a/openedx/core/__pycache__/apidocs.cpython-38.pyc.139860693926064 b/cms/djangoapps/coursegraph/tests/__init__.py similarity index 100% rename from openedx/core/__pycache__/apidocs.cpython-38.pyc.139860693926064 rename to cms/djangoapps/coursegraph/tests/__init__.py diff --git a/cms/djangoapps/coursegraph/tests/test_admin.py b/cms/djangoapps/coursegraph/tests/test_admin.py new file mode 100644 index 0000000000..21a26d8450 --- /dev/null +++ b/cms/djangoapps/coursegraph/tests/test_admin.py @@ -0,0 +1,227 @@ +""" +Shallow tests for CourseGraph dump-queueing Django admin interface. + +See ..management.commands.tests.test_dump_to_neo4j for more comprehensive +tests of dump_course_to_neo4j. +""" + +from unittest import mock + +import py2neo +from django.test import TestCase +from django.test.utils import override_settings +from freezegun import freeze_time + +from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview + +from .. import admin, tasks + + +_coursegraph_connection = { + "protocol": "bolt", + "secure": True, + "host": "example.edu", + "port": 7687, + "user": "neo4j", + "password": "fake-coursegraph-password", +} + +_configure_coursegraph_connection = override_settings( + COURSEGRAPH_CONNECTION=_coursegraph_connection, +) + +_patch_log_exception = mock.patch.object( + admin.log, 'exception', autospec=True +) + +_patch_apply_dump_task = mock.patch.object( + tasks.dump_course_to_neo4j, 'apply_async' +) + +_pretend_last_course_dump_was_may_2020 = mock.patch.object( + tasks, + 'get_command_last_run', + new=(lambda _key, _graph: "2020-05-01"), +) + +_patch_neo4j_graph = mock.patch.object( + tasks, 'Graph', autospec=True +) + +_make_neo4j_graph_raise = mock.patch.object( + tasks, 'Graph', side_effect=py2neo.ConnectionUnavailable( + 'we failed to connect or something!' + ) +) + + +class CourseGraphAdminActionsTestCase(TestCase): + """ + Test CourseGraph Django admin actions. + """ + + @classmethod + def setUpTestData(cls): + """ + Make course overviews with varying modification dates. + """ + super().setUpTestData() + cls.course_updated_in_april = CourseOverviewFactory(run='april_update') + cls.course_updated_in_june = CourseOverviewFactory(run='june_update') + cls.course_updated_in_july = CourseOverviewFactory(run='july_update') + cls.course_updated_in_august = CourseOverviewFactory(run='august_update') + + # For each course overview, make an arbitrary update and then save() + # so that its `.modified` date is set. + with freeze_time("2020-04-01"): + cls.course_updated_in_april.marketing_url = "https://example.org" + cls.course_updated_in_april.save() + with freeze_time("2020-06-01"): + cls.course_updated_in_june.marketing_url = "https://example.org" + cls.course_updated_in_june.save() + with freeze_time("2020-07-01"): + cls.course_updated_in_july.marketing_url = "https://example.org" + cls.course_updated_in_july.save() + with freeze_time("2020-08-01"): + cls.course_updated_in_august.marketing_url = "https://example.org" + cls.course_updated_in_august.save() + + @_configure_coursegraph_connection + @_pretend_last_course_dump_was_may_2020 + @_patch_neo4j_graph + @_patch_apply_dump_task + @_patch_log_exception + def test_dump_courses(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph): + """ + Test that dump_courses admin action dumps requested courses iff they have + been modified since the last dump to coursegraph. + """ + modeladmin_mock = mock.MagicMock() + + # Request all courses except the August-updated one + requested_course_keys = { + str(self.course_updated_in_april.id), + str(self.course_updated_in_june.id), + str(self.course_updated_in_july.id), + } + admin.dump_courses( + modeladmin=modeladmin_mock, + request=mock.MagicMock(), + queryset=CourseOverview.objects.filter(id__in=requested_course_keys), + ) + + # User should have been messaged + assert modeladmin_mock.message_user.call_count == 1 + assert modeladmin_mock.message_user.call_args.args[1] == ( + "Enqueued dumps for 2 course(s). Skipped 1 unchanged course(s)." + ) + + # For enqueueing, graph should've been authenticated once, using configured settings. + assert mock_neo4j_graph.call_count == 1 + assert mock_neo4j_graph.call_args.args == () + assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection + + # No errors should've been logged. + assert mock_log_exception.call_count == 0 + + # April course should have been skipped because the command was last run in May. + # Dumps for June and July courses should have been enqueued. + assert mock_apply_dump_task.call_count == 2 + actual_dumped_course_keys = { + call_args.kwargs['kwargs']['course_key_string'] + for call_args in mock_apply_dump_task.call_args_list + } + expected_dumped_course_keys = { + str(self.course_updated_in_june.id), + str(self.course_updated_in_july.id), + } + assert actual_dumped_course_keys == expected_dumped_course_keys + + @_configure_coursegraph_connection + @_pretend_last_course_dump_was_may_2020 + @_patch_neo4j_graph + @_patch_apply_dump_task + @_patch_log_exception + def test_dump_courses_overriding_cache(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph): + """ + Test that dump_coursese_overriding_cach admin action dumps requested courses + whether or not they been modified since the last dump to coursegraph. + """ + modeladmin_mock = mock.MagicMock() + + # Request all courses except the August-updated one + requested_course_keys = { + str(self.course_updated_in_april.id), + str(self.course_updated_in_june.id), + str(self.course_updated_in_july.id), + } + admin.dump_courses_overriding_cache( + modeladmin=modeladmin_mock, + request=mock.MagicMock(), + queryset=CourseOverview.objects.filter(id__in=requested_course_keys), + ) + + # User should have been messaged + assert modeladmin_mock.message_user.call_count == 1 + assert modeladmin_mock.message_user.call_args.args[1] == ( + "Enqueued dumps for 3 course(s)." + ) + + # For enqueueing, graph should've been authenticated once, using configured settings. + assert mock_neo4j_graph.call_count == 1 + assert mock_neo4j_graph.call_args.args == () + assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection + + # No errors should've been logged. + assert mock_log_exception.call_count == 0 + + # April, June, and July courses should have all been dumped. + assert mock_apply_dump_task.call_count == 3 + actual_dumped_course_keys = { + call_args.kwargs['kwargs']['course_key_string'] + for call_args in mock_apply_dump_task.call_args_list + } + expected_dumped_course_keys = { + str(self.course_updated_in_april.id), + str(self.course_updated_in_june.id), + str(self.course_updated_in_july.id), + } + assert actual_dumped_course_keys == expected_dumped_course_keys + + @_configure_coursegraph_connection + @_pretend_last_course_dump_was_may_2020 + @_make_neo4j_graph_raise + @_patch_apply_dump_task + @_patch_log_exception + def test_dump_courses_error(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph): + """ + Test that the dump_courses admin action dumps messages the user if an error + occurs when trying to enqueue course dumps. + """ + modeladmin_mock = mock.MagicMock() + + # Request dump of all four courses. + admin.dump_courses( + modeladmin=modeladmin_mock, + request=mock.MagicMock(), + queryset=CourseOverview.objects.all() + ) + + # Admin user should have been messaged about failure. + assert modeladmin_mock.message_user.call_count == 1 + assert modeladmin_mock.message_user.call_args.args[1] == ( + "Error enqueueing dumps for 4 course(s): we failed to connect or something!" + ) + + # For enqueueing, graph should've been authenticated once, using configured settings. + assert mock_neo4j_graph.call_count == 1 + assert mock_neo4j_graph.call_args.args == () + assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection + + # Exception should have been logged. + assert mock_log_exception.call_count == 1 + assert "Failed to enqueue" in mock_log_exception.call_args.args[0] + + # No courses should have been dumped. + assert mock_apply_dump_task.call_count == 0 diff --git a/cms/envs/common.py b/cms/envs/common.py index cd7eb4b9de..759f474d68 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -484,6 +484,19 @@ FEATURES = { # .. toggle_target_removal_date: None # .. toggle_tickets: 'https://openedx.atlassian.net/browse/MST-1348' 'ENABLE_INTEGRITY_SIGNATURE': False, + + # .. toggle_name: MARK_LIBRARY_CONTENT_BLOCK_COMPLETE_ON_VIEW + # .. toggle_implementation: DjangoSetting + # .. toggle_default: False + # .. toggle_description: If enabled, the Library Content Block is marked as complete when users view it. + # Otherwise (by default), all children of this block must be completed. + # .. toggle_use_cases: open_edx + # .. toggle_creation_date: 2022-03-22 + # .. toggle_target_removal_date: None + # .. toggle_tickets: https://github.com/edx/edx-platform/pull/28268 + # .. toggle_warnings: For consistency in user-experience, keep the value in sync with the setting of the same name + # in the LMS and CMS. + 'MARK_LIBRARY_CONTENT_BLOCK_COMPLETE_ON_VIEW': False, } # .. toggle_name: ENABLE_COPPA_COMPLIANCE @@ -518,6 +531,16 @@ LIBRARY_AUTHORING_MICROFRONTEND_URL = None # .. toggle_creation_date: 2021-12-03 # .. toggle_tickets: https://openedx.atlassian.net/browse/VAN-666 ENABLE_AUTHN_RESET_PASSWORD_HIBP_POLICY = False +# .. toggle_name: ENABLE_AUTHN_REGISTER_HIBP_POLICY +# .. toggle_implementation: DjangoSetting +# .. toggle_default: False +# .. toggle_description: When enabled, this toggle activates the use of the password validation +# HIBP Policy on Authn MFE's registration. +# .. toggle_use_cases: open_edx +# .. toggle_creation_date: 2022-03-25 +# .. toggle_tickets: https://openedx.atlassian.net/browse/VAN-669 +ENABLE_AUTHN_REGISTER_HIBP_POLICY = False +HIBP_REGISTRATION_PASSWORD_FREQUENCY_THRESHOLD = 3 ############################# SOCIAL MEDIA SHARING ############################# SOCIAL_SHARING_SETTINGS = { @@ -663,7 +686,7 @@ AUTHENTICATION_BACKENDS = [ 'auth_backends.backends.EdXOAuth2', 'rules.permissions.ObjectPermissionBackend', 'openedx.core.djangoapps.content_libraries.auth.LtiAuthenticationBackend', - 'openedx.core.djangoapps.oauth_dispatch.dot_overrides.backends.EdxRateLimitedAllowAllUsersModelBackend', + 'django.contrib.auth.backends.AllowAllUsersModelBackend', 'bridgekeeper.backends.RulePermissionBackend', ] @@ -752,16 +775,15 @@ XQUEUE_INTERFACE = { MIDDLEWARE = [ 'openedx.core.lib.x_forwarded_for.middleware.XForwardedForMiddleware', - 'crum.CurrentRequestUserMiddleware', - 'edx_django_utils.monitoring.DeploymentMonitoringMiddleware', - # A newer and safer request cache. + # Resets the request cache. 'edx_django_utils.cache.middleware.RequestCacheMiddleware', - 'edx_django_utils.monitoring.MonitoringMemoryMiddleware', - # Cookie monitoring - 'openedx.core.lib.request_utils.CookieMonitoringMiddleware', + # Various monitoring middleware + 'edx_django_utils.monitoring.CookieMonitoringMiddleware', + 'edx_django_utils.monitoring.DeploymentMonitoringMiddleware', + 'edx_django_utils.monitoring.MonitoringMemoryMiddleware', # Before anything that looks at cookies, especially the session middleware 'openedx.core.djangoapps.cookie_metadata.middleware.CookieNameChange', @@ -811,9 +833,6 @@ MIDDLEWARE = [ 'codejail.django_integration.ConfigureCodeJailMiddleware', - # catches any uncaught RateLimitExceptions and returns a 403 instead of a 500 - 'ratelimitbackend.middleware.RateLimitMiddleware', - # for expiring inactive sessions 'openedx.core.djangoapps.session_inactivity_timeout.middleware.SessionInactivityTimeout', @@ -1592,7 +1611,7 @@ INSTALLED_APPS = [ 'openedx.core.djangoapps.self_paced', # Coursegraph - 'openedx.core.djangoapps.coursegraph.apps.CoursegraphConfig', + 'cms.djangoapps.coursegraph.apps.CoursegraphConfig', # Credit courses 'openedx.core.djangoapps.credit.apps.CreditConfig', @@ -1689,8 +1708,6 @@ INSTALLED_APPS = [ # Learning Sequence Navigation 'openedx.core.djangoapps.content.learning_sequences.apps.LearningSequencesConfig', - 'ratelimitbackend', - # Database-backed Organizations App (http://github.com/edx/edx-organizations) 'organizations', @@ -2218,7 +2235,33 @@ POLICY_CHANGE_TASK_RATE_LIMIT = '300/h' # .. setting_default: value of LOW_PRIORITY_QUEUE # .. setting_description: The name of the Celery queue to which CourseGraph refresh # tasks will be sent -COURSEGRAPH_JOB_QUEUE = LOW_PRIORITY_QUEUE +COURSEGRAPH_JOB_QUEUE: str = LOW_PRIORITY_QUEUE + +# .. setting_name: COURSEGRAPH_CONNECTION +# .. setting_default: 'bolt+s://localhost:7687', in dictionary form. +# .. setting_description: Dictionary specifying Neo4j connection parameters for +# CourseGraph refresh. Accepted keys are protocol ('bolt' or 'http'), +# secure (bool), host (str), port (int), user (str), and password (str). +# See https://py2neo.org/2021.1/profiles.html#individual-settings for a +# a description of each of those keys. +COURSEGRAPH_CONNECTION: dict = { + "protocol": "bolt", + "secure": True, + "host": "localhost", + "port": 7687, + "user": "neo4j", + "password": None, +} + +# .. toggle_name: COURSEGRAPH_DUMP_COURSE_ON_PUBLISH +# .. toggle_implementation: DjangoSetting +# .. toggle_creation_date: 2022-01-27 +# .. toggle_use_cases: open_edx +# .. toggle_default: False +# .. toggle_description: Whether, upon publish, a course should automatically +# be exported to Neo4j via the connection parameters specified in +# `COURSEGRAPH_CONNECTION`. +COURSEGRAPH_DUMP_COURSE_ON_PUBLISH: bool = False ########## Settings for video transcript migration tasks ############ VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = DEFAULT_PRIORITY_QUEUE diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py index 6b632ddaf1..6c67d4d389 100644 --- a/cms/envs/devstack.py +++ b/cms/envs/devstack.py @@ -256,6 +256,17 @@ FEATURES['ENABLE_PREREQUISITE_COURSES'] = True # (ref MST-637) PROCTORING_USER_OBFUSCATION_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' +############## CourseGraph devstack settings ############################ + +COURSEGRAPH_CONNECTION: dict = { + "protocol": "bolt", + "secure": False, + "host": "edx.devstack.coursegraph", + "port": 7687, + "user": "neo4j", + "password": "edx", +} + #################### Webpack Configuration Settings ############################## WEBPACK_LOADER['DEFAULT']['TIMEOUT'] = 5 diff --git a/cms/envs/production.py b/cms/envs/production.py index 28895ba74f..82244acbde 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -83,6 +83,7 @@ with codecs.open(CONFIG_FILE, encoding='utf-8') as f: 'CELERY_QUEUES', 'MKTG_URL_LINK_MAP', 'MKTG_URL_OVERRIDES', + 'REST_FRAMEWORK', ] for key in KEYS_WITH_MERGED_VALUES: if key in __config_copy__: @@ -170,7 +171,7 @@ ENTERPRISE_CONSENT_API_URL = ENV_TOKENS.get('ENTERPRISE_CONSENT_API_URL', LMS_IN # Studio. Only applies to IDA for which the social auth flow uses DOT (Django OAuth Toolkit). IDA_LOGOUT_URI_LIST = ENV_TOKENS.get('IDA_LOGOUT_URI_LIST', []) -SITE_NAME = ENV_TOKENS['SITE_NAME'] +SITE_NAME = ENV_TOKENS.get('SITE_NAME', SITE_NAME) ALLOWED_HOSTS = [ # TODO: bbeggs remove this before prod, temp fix to get load testing running @@ -178,10 +179,10 @@ ALLOWED_HOSTS = [ CMS_BASE, ] -LOG_DIR = ENV_TOKENS['LOG_DIR'] +LOG_DIR = ENV_TOKENS.get('LOG_DIR', LOG_DIR) DATA_DIR = path(ENV_TOKENS.get('DATA_DIR', DATA_DIR)) -CACHES = ENV_TOKENS['CACHES'] +CACHES = ENV_TOKENS.get('CACHES', CACHES) # Cache used for location mapping -- called many times with the same key/value # in a given request. if 'loc_cache' not in CACHES: @@ -274,7 +275,7 @@ for app in ENV_TOKENS.get('ADDL_INSTALLED_APPS', []): INSTALLED_APPS.append(app) LOGGING = get_logger_config(LOG_DIR, - logging_env=ENV_TOKENS['LOGGING_ENV'], + logging_env=ENV_TOKENS.get('LOGGING_ENV', LOGGING_ENV), service_variant=SERVICE_VARIANT) # The following variables use (or) instead of the default value inside (get). This is to enforce using the Lazy Text @@ -315,11 +316,11 @@ CMS_SEGMENT_KEY = AUTH_TOKENS.get('SEGMENT_KEY') SECRET_KEY = AUTH_TOKENS['SECRET_KEY'] -AWS_ACCESS_KEY_ID = AUTH_TOKENS["AWS_ACCESS_KEY_ID"] +AWS_ACCESS_KEY_ID = AUTH_TOKENS.get("AWS_ACCESS_KEY_ID", AWS_ACCESS_KEY_ID) if AWS_ACCESS_KEY_ID == "": AWS_ACCESS_KEY_ID = None -AWS_SECRET_ACCESS_KEY = AUTH_TOKENS["AWS_SECRET_ACCESS_KEY"] +AWS_SECRET_ACCESS_KEY = AUTH_TOKENS.get("AWS_SECRET_ACCESS_KEY", AWS_SECRET_ACCESS_KEY) if AWS_SECRET_ACCESS_KEY == "": AWS_SECRET_ACCESS_KEY = None @@ -357,7 +358,7 @@ if COURSE_METADATA_EXPORT_BUCKET: else: COURSE_METADATA_EXPORT_STORAGE = DEFAULT_FILE_STORAGE -DATABASES = AUTH_TOKENS['DATABASES'] +DATABASES = AUTH_TOKENS.get('DATABASES', DATABASES) # The normal database user does not have enough permissions to run migrations. # Migrations are run with separate credentials, given as DB_MIGRATION_* @@ -385,8 +386,8 @@ XBLOCK_FIELD_DATA_WRAPPERS = ENV_TOKENS.get( XBLOCK_FIELD_DATA_WRAPPERS ) -CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE'] -DOC_STORE_CONFIG = AUTH_TOKENS['DOC_STORE_CONFIG'] +CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE) +DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG', DOC_STORE_CONFIG) ############################### BLOCKSTORE ##################################### BLOCKSTORE_API_URL = ENV_TOKENS.get('BLOCKSTORE_API_URL', None) # e.g. "https://blockstore.example.com/api/v1/" @@ -602,7 +603,7 @@ EXPLICIT_QUEUES = { 'queue': POLICY_CHANGE_GRADES_ROUTING_KEY}, 'cms.djangoapps.contentstore.tasks.update_search_index': { 'queue': UPDATE_SEARCH_INDEX_JOB_QUEUE}, - 'openedx.core.djangoapps.coursegraph.tasks.dump_course_to_neo4j': { + 'cms.djangoapps.coursegraph.tasks.dump_course_to_neo4j': { 'queue': COURSEGRAPH_JOB_QUEUE}, } @@ -629,3 +630,6 @@ DISCUSSIONS_MICROFRONTEND_URL = ENV_TOKENS.get('DISCUSSIONS_MICROFRONTEND_URL', ################### Discussions micro frontend Feedback URL################### DISCUSSIONS_MFE_FEEDBACK_URL = ENV_TOKENS.get('DISCUSSIONS_MFE_FEEDBACK_URL', DISCUSSIONS_MFE_FEEDBACK_URL) + +############## DRF overrides ############## +REST_FRAMEWORK.update(ENV_TOKENS.get('REST_FRAMEWORK', {})) diff --git a/cms/envs/test.py b/cms/envs/test.py index aaf7feb969..e58b8ea20b 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -283,7 +283,10 @@ FEATURES['ENABLE_TEAMS'] = True SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' ######### custom courses ######### -INSTALLED_APPS.append('openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig') +INSTALLED_APPS += [ + 'openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig', + 'common.djangoapps.third_party_auth.apps.ThirdPartyAuthConfig', +] FEATURES['CUSTOM_COURSES_EDX'] = True ########################## VIDEO IMAGE STORAGE ############################ diff --git a/cms/templates/admin/base_site.html b/cms/templates/admin/base_site.html index dd10b68f5f..0783178430 100644 --- a/cms/templates/admin/base_site.html +++ b/cms/templates/admin/base_site.html @@ -17,10 +17,7 @@ {% endif %} {% endif %} - {% flag "user_authn.admin_auth_redirect_to_lms" %} - {% trans 'Log out' as tmsg %} {{tmsg|force_escape}} - {% else %} - {% trans 'Log out' as tmsg %} {{tmsg|force_escape}} - {% endflag %} + {% trans 'Log out' as tmsg %} {{tmsg|force_escape}} + {% endblock %} diff --git a/cms/urls.py b/cms/urls.py index 3facddf0bf..541e2aee85 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -10,7 +10,7 @@ from django.urls import path, re_path from django.utils.translation import gettext_lazy as _ from auth_backends.urls import oauth2_urlpatterns from edx_api_doc_tools import make_docs_urls -from ratelimitbackend import admin +from django.contrib import admin import openedx.core.djangoapps.common_views.xblock import openedx.core.djangoapps.debug.views diff --git a/common/djangoapps/third_party_auth/migrations/0007_samlproviderconfig_was_valid_at.py b/common/djangoapps/third_party_auth/migrations/0007_samlproviderconfig_was_valid_at.py new file mode 100644 index 0000000000..06bc446277 --- /dev/null +++ b/common/djangoapps/third_party_auth/migrations/0007_samlproviderconfig_was_valid_at.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-03-21 14:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('third_party_auth', '0006_auto_20220314_1551'), + ] + + operations = [ + migrations.AddField( + model_name='samlproviderconfig', + name='was_valid_at', + field=models.DateTimeField(blank=True, help_text='Timestamped field that indicates a user has successfully logged in using this configuration at least once.', null=True), + ), + ] diff --git a/common/djangoapps/third_party_auth/migrations/0008_auto_20220324_1422.py b/common/djangoapps/third_party_auth/migrations/0008_auto_20220324_1422.py new file mode 100644 index 0000000000..653a9287db --- /dev/null +++ b/common/djangoapps/third_party_auth/migrations/0008_auto_20220324_1422.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.12 on 2022-03-24 14:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('third_party_auth', '0007_samlproviderconfig_was_valid_at'), + ] + + operations = [ + migrations.AddField( + model_name='ltiproviderconfig', + name='was_valid_at', + field=models.DateTimeField(blank=True, help_text='Timestamped field that indicates a user has successfully logged in using this configuration at least once.', null=True), + ), + migrations.AddField( + model_name='oauth2providerconfig', + name='was_valid_at', + field=models.DateTimeField(blank=True, help_text='Timestamped field that indicates a user has successfully logged in using this configuration at least once.', null=True), + ), + ] diff --git a/common/djangoapps/third_party_auth/models.py b/common/djangoapps/third_party_auth/models.py index 1421767cb8..f0a37616a3 100644 --- a/common/djangoapps/third_party_auth/models.py +++ b/common/djangoapps/third_party_auth/models.py @@ -223,6 +223,14 @@ class ProviderConfig(ConfigurationModel): ) ) + was_valid_at = models.DateTimeField( + blank=True, + null=True, + help_text=( + "Timestamped field that indicates a user has successfully logged in using this configuration at least once." + ) + ) + prefix = None # used for provider_id. Set to a string value in subclass backend_name = None # Set to a field or fixed value in subclass accepts_logins = True # Whether to display a sign-in button when the provider is enabled diff --git a/common/djangoapps/third_party_auth/samlproviderdata/tests/test_samlproviderdata.py b/common/djangoapps/third_party_auth/samlproviderdata/tests/test_samlproviderdata.py index 2ceb1e968e..7607ee5dd9 100644 --- a/common/djangoapps/third_party_auth/samlproviderdata/tests/test_samlproviderdata.py +++ b/common/djangoapps/third_party_auth/samlproviderdata/tests/test_samlproviderdata.py @@ -1,18 +1,20 @@ # pylint: disable=missing-module-docstring import copy -import pytz -from uuid import uuid4 # lint-amnesty, pylint: disable=wrong-import-order from datetime import datetime # lint-amnesty, pylint: disable=wrong-import-order +from unittest import mock +from uuid import uuid4 # lint-amnesty, pylint: disable=wrong-import-order + +import pytz from django.contrib.sites.models import Site from django.urls import reverse from django.utils.http import urlencode +from enterprise.constants import ENTERPRISE_ADMIN_ROLE, ENTERPRISE_LEARNER_ROLE +from enterprise.models import EnterpriseCustomer, EnterpriseCustomerIdentityProvider from rest_framework import status from rest_framework.test import APITestCase -from enterprise.models import EnterpriseCustomer, EnterpriseCustomerIdentityProvider -from enterprise.constants import ENTERPRISE_ADMIN_ROLE, ENTERPRISE_LEARNER_ROLE from common.djangoapps.student.tests.factories import UserFactory -from common.djangoapps.third_party_auth.models import SAMLProviderData, SAMLProviderConfig +from common.djangoapps.third_party_auth.models import SAMLProviderConfig, SAMLProviderData from common.djangoapps.third_party_auth.tests.samlutils import set_jwt_cookie from common.djangoapps.third_party_auth.tests.utils import skip_unless_thirdpartyauth from common.djangoapps.third_party_auth.utils import convert_saml_slug_provider_id @@ -180,3 +182,35 @@ class SAMLProviderDataTests(APITestCase): set_jwt_cookie(self.client, self.user, [(ENTERPRISE_ADMIN_ROLE, BAD_ENTERPRISE_ID)]) response = self.client.get(url, format='json') assert response.status_code == status.HTTP_403_FORBIDDEN + + @mock.patch('common.djangoapps.third_party_auth.samlproviderdata.views.fetch_metadata_xml') + @mock.patch('common.djangoapps.third_party_auth.samlproviderdata.views.parse_metadata_xml') + def test_sync_one_provider_data_success(self, mock_parse, mock_fetch): + """ + POST auth/saml/v0/provider_data/sync_provider_data -d data + """ + mock_fetch.return_value = 'tag' + public_key = 'askdjf;sakdjfs;adkfjas;dkfjas;dkfjas;dlkfj' + sso_url = 'https://fake-test.id' + expires_at = datetime.now() + mock_parse.return_value = (public_key, sso_url, expires_at) + url = reverse('saml_provider_data-sync-provider-data') + data = { + 'entity_id': 'http://entity-id-1', + 'metadata_url': 'http://a-url', + 'enterprise_customer_uuid': ENTERPRISE_ID, + } + SAMLProviderData.objects.all().delete() + orig_count = SAMLProviderData.objects.count() + + response = self.client.post(url, data) + + assert response.status_code == status.HTTP_201_CREATED + assert response.data == " Created new record for SAMLProviderData for entityID http://entity-id-1" + assert SAMLProviderData.objects.count() == orig_count + 1 + + # should only update this time + response = self.client.post(url, data) + assert response.status_code == status.HTTP_200_OK + assert response.data == (" Updated existing SAMLProviderData for entityID http://entity-id-1") + assert SAMLProviderData.objects.count() == orig_count + 1 diff --git a/common/djangoapps/third_party_auth/samlproviderdata/views.py b/common/djangoapps/third_party_auth/samlproviderdata/views.py index 43c24db812..c3551cd656 100644 --- a/common/djangoapps/third_party_auth/samlproviderdata/views.py +++ b/common/djangoapps/third_party_auth/samlproviderdata/views.py @@ -1,21 +1,32 @@ """ Viewset for auth/saml/v0/samlproviderdata """ +import logging -from django.shortcuts import get_object_or_404 from django.http import Http404 +from django.shortcuts import get_object_or_404 from edx_rbac.mixins import PermissionRequiredMixin from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication -from rest_framework import permissions, viewsets -from rest_framework.authentication import SessionAuthentication -from rest_framework.exceptions import ParseError - from enterprise.models import EnterpriseCustomerIdentityProvider -from common.djangoapps.third_party_auth.utils import validate_uuid4_string, convert_saml_slug_provider_id +from rest_framework import permissions, status, viewsets +from rest_framework.authentication import SessionAuthentication +from rest_framework.decorators import action +from rest_framework.exceptions import ParseError +from rest_framework.response import Response + +from common.djangoapps.third_party_auth.utils import ( + convert_saml_slug_provider_id, + create_or_update_saml_provider_data, + fetch_metadata_xml, + parse_metadata_xml, + validate_uuid4_string +) from ..models import SAMLProviderConfig, SAMLProviderData from .serializers import SAMLProviderDataSerializer +log = logging.getLogger(__name__) + class SAMLProviderDataMixin: authentication_classes = [JwtAuthentication, SessionAuthentication] @@ -36,6 +47,7 @@ class SAMLProviderDataViewSet(PermissionRequiredMixin, SAMLProviderDataMixin, vi POST /auth/saml/v0/provider_data/ -d postData (must contain 'enterprise_customer_uuid') DELETE /auth/saml/v0/provider_data/:pk -d postData (must contain 'enterprise_customer_uuid') PATCH /auth/saml/v0/provider_data/:pk -d postData (must contain 'enterprise_customer_uuid') + POST /auth/saml/v0/provider_data/sync_provider_data (fetches metadata info from metadata url provided) """ permission_required = 'enterprise.can_access_admin_dashboard' @@ -81,3 +93,37 @@ class SAMLProviderDataViewSet(PermissionRequiredMixin, SAMLProviderDataMixin, vi Retrieve an EnterpriseCustomer to do auth against """ return self.requested_enterprise_uuid + + @action(detail=False, methods=['post']) + def sync_provider_data(self, request): + """ + Creates or updates a SAMProviderData record using info fetched from remote SAML metadata + For now we will require entityID but in future we will enhance this to try and extract entityID + from the metadata file, and make entityId optional, and return error response if there are + multiple entityIDs listed so that the user can choose and retry with a specified entityID + """ + entity_id = request.POST.get('entity_id') + metadata_url = request.POST.get('metadata_url') + if not entity_id: + return Response('entity_id is required!', status.HTTP_400_BAD_REQUEST) + if not metadata_url: + return Response('metadata_url is required!', status.HTTP_400_BAD_REQUEST) + + # part 1: fetch information from remote metadata based on metadataUrl in samlproviderconfig + xml = fetch_metadata_xml(metadata_url) + + # part 2: create/update samlproviderdata + log.info("Processing IdP with entityID %s", entity_id) + public_key, sso_url, expires_at = parse_metadata_xml(xml, entity_id) + changed = create_or_update_saml_provider_data(entity_id, public_key, sso_url, expires_at) + if changed: + str_message = f" Created new record for SAMLProviderData for entityID {entity_id}" + log.info(str_message) + response = str_message + http_status = status.HTTP_201_CREATED + else: + str_message = f" Updated existing SAMLProviderData for entityID {entity_id}" + log.info(str_message) + response = str_message + http_status = status.HTTP_200_OK + return Response(response, status=http_status) diff --git a/common/djangoapps/third_party_auth/tasks.py b/common/djangoapps/third_party_auth/tasks.py index 2b29ce20cc..88b118a689 100644 --- a/common/djangoapps/third_party_auth/tasks.py +++ b/common/djangoapps/third_party_auth/tasks.py @@ -7,13 +7,16 @@ import logging import requests from celery import shared_task -from django.utils.timezone import now from edx_django_utils.monitoring import set_code_owner_attribute from lxml import etree from requests import exceptions -from common.djangoapps.third_party_auth.models import SAMLConfiguration, SAMLProviderConfig, SAMLProviderData -from common.djangoapps.third_party_auth.utils import MetadataParseError, parse_metadata_xml +from common.djangoapps.third_party_auth.models import SAMLConfiguration, SAMLProviderConfig +from common.djangoapps.third_party_auth.utils import ( + MetadataParseError, + create_or_update_saml_provider_data, + parse_metadata_xml, +) log = logging.getLogger(__name__) @@ -85,7 +88,7 @@ def fetch_saml_metadata(): for entity_id in entity_ids: log.info("Processing IdP with entityID %s", entity_id) public_key, sso_url, expires_at = parse_metadata_xml(xml, entity_id) - changed = _update_data(entity_id, public_key, sso_url, expires_at) + changed = create_or_update_saml_provider_data(entity_id, public_key, sso_url, expires_at) if changed: log.info(f"→ Created new record for SAMLProviderData for entityID {entity_id}") num_updated += 1 @@ -124,28 +127,3 @@ def fetch_saml_metadata(): # Return counts for total, skipped, attempted, updated, and failed, along with any failure messages return num_total, num_skipped, num_attempted, num_updated, len(failure_messages), failure_messages - - -def _update_data(entity_id, public_key, sso_url, expires_at): - """ - Update/Create the SAMLProviderData for the given entity ID. - Return value: - False if nothing has changed and existing data's "fetched at" timestamp is just updated. - True if a new record was created. (Either this is a new provider or something changed.) - """ - data_obj = SAMLProviderData.current(entity_id) - fetched_at = now() - if data_obj and (data_obj.public_key == public_key and data_obj.sso_url == sso_url): - data_obj.expires_at = expires_at - data_obj.fetched_at = fetched_at - data_obj.save() - return False - else: - SAMLProviderData.objects.create( - entity_id=entity_id, - fetched_at=fetched_at, - expires_at=expires_at, - sso_url=sso_url, - public_key=public_key, - ) - return True diff --git a/common/djangoapps/third_party_auth/tests/test_admin.py b/common/djangoapps/third_party_auth/tests/test_admin.py index 82a93e8db8..c5481a3a09 100644 --- a/common/djangoapps/third_party_auth/tests/test_admin.py +++ b/common/djangoapps/third_party_auth/tests/test_admin.py @@ -64,10 +64,11 @@ class Oauth2ProviderConfigAdminTest(testutil.TestCase): # Remove the icon_image from the POST data, to simulate unchanged icon_image post_data = models.model_to_dict(provider1) del post_data['icon_image'] - # Remove max_session_length and organization. A default null value must be POSTed + # Remove max_session_length, was_valid_at and organization. A default null value must be POSTed # back as an absent value, rather than as a "null-like" included value. del post_data['max_session_length'] del post_data['organization'] + del post_data['was_valid_at'] # Change the name, to verify POST post_data['name'] = 'Another name' diff --git a/common/djangoapps/third_party_auth/utils.py b/common/djangoapps/third_party_auth/utils.py index 3d411bb63a..8517af0328 100644 --- a/common/djangoapps/third_party_auth/utils.py +++ b/common/djangoapps/third_party_auth/utils.py @@ -3,29 +3,68 @@ Utility functions for third_party_auth """ import datetime +import logging from uuid import UUID import dateutil.parser import pytz +import requests from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user +from django.utils.timezone import now from enterprise.models import EnterpriseCustomerIdentityProvider, EnterpriseCustomerUser from lxml import etree from onelogin.saml2.utils import OneLogin_Saml2_Utils +from requests import exceptions from social_core.pipeline.social_auth import associate_by_email -from common.djangoapps.third_party_auth.models import OAuth2ProviderConfig +from common.djangoapps.third_party_auth.models import OAuth2ProviderConfig, SAMLProviderData from openedx.core.djangolib.markup import Text from . import provider SAML_XML_NS = 'urn:oasis:names:tc:SAML:2.0:metadata' # The SAML Metadata XML namespace +log = logging.getLogger(__name__) + class MetadataParseError(Exception): """ An error occurred while parsing the SAML metadata from an IdP """ pass # lint-amnesty, pylint: disable=unnecessary-pass +def fetch_metadata_xml(url): + """ + Fetches IDP metadata from provider url + Returns: xml document + """ + try: + log.info("Fetching %s", url) + if not url.lower().startswith('https'): + log.warning("This SAML metadata URL is not secure! It should use HTTPS. (%s)", url) + response = requests.get(url, verify=True) # May raise HTTPError or SSLError or ConnectionError + response.raise_for_status() # May raise an HTTPError + + try: + parser = etree.XMLParser(remove_comments=True) + xml = etree.fromstring(response.content, parser) + except etree.XMLSyntaxError: # lint-amnesty, pylint: disable=try-except-raise + raise + # TODO: Can use OneLogin_Saml2_Utils to validate signed XML if anyone is using that + return xml + except (exceptions.SSLError, exceptions.HTTPError, exceptions.RequestException, MetadataParseError) as error: + # Catch and process exception in case of errors during fetching and processing saml metadata. + # Here is a description of each exception. + # SSLError is raised in case of errors caused by SSL (e.g. SSL cer verification failure etc.) + # HTTPError is raised in case of unexpected status code (e.g. 500 error etc.) + # RequestException is the base exception for any request related error that "requests" lib raises. + # MetadataParseError is raised if there is error in the fetched meta data (e.g. missing @entityID etc.) + log.exception(str(error), exc_info=error) + raise error + except etree.XMLSyntaxError as error: + log.exception(str(error), exc_info=error) + raise error + + def parse_metadata_xml(xml, entity_id): """ Given an XML document containing SAML 2.0 metadata, parse it and return a tuple of @@ -125,6 +164,31 @@ def get_user_from_email(details): return None +def create_or_update_saml_provider_data(entity_id, public_key, sso_url, expires_at): + """ + Update/Create the SAMLProviderData for the given entity ID. + Return value: + False if nothing has changed and existing data's "fetched at" timestamp is just updated. + True if a new record was created. (Either this is a new provider or something changed.) + """ + data_obj = SAMLProviderData.current(entity_id) + fetched_at = now() + if data_obj and (data_obj.public_key == public_key and data_obj.sso_url == sso_url): + data_obj.expires_at = expires_at + data_obj.fetched_at = fetched_at + data_obj.save() + return False + else: + SAMLProviderData.objects.create( + entity_id=entity_id, + fetched_at=fetched_at, + expires_at=expires_at, + sso_url=sso_url, + public_key=public_key, + ) + return True + + def convert_saml_slug_provider_id(provider): # lint-amnesty, pylint: disable=redefined-outer-name """ Provider id is stored with the backend type prefixed to it (ie "saml-") diff --git a/common/djangoapps/util/request_rate_limiter.py b/common/djangoapps/util/request_rate_limiter.py deleted file mode 100644 index cc2ad663e0..0000000000 --- a/common/djangoapps/util/request_rate_limiter.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -A utility class which wraps the RateLimitMixin 3rd party class to do bad request counting -which can be used for rate limiting -""" - -from ratelimitbackend.backends import RateLimitMixin - - -class RequestRateLimiter(RateLimitMixin): - """ - Use the 3rd party RateLimitMixin to help do rate limiting. - """ - def is_rate_limit_exceeded(self, request): - """ - Returns if the client has been rated limited - """ - counts = self.get_counters(request) - return sum(counts.values()) >= self.requests - - def tick_request_counter(self, request): - """ - Ticks any counters used to compute when rate limt has been reached - """ - self.cache_incr(self.get_cache_key(request)) - - -class BadRequestRateLimiter(RequestRateLimiter): - """ - Default rate limit is 30 requests for every 5 minutes. - """ - pass # lint-amnesty, pylint: disable=unnecessary-pass diff --git a/common/lib/xmodule/xmodule/library_content_module.py b/common/lib/xmodule/xmodule/library_content_module.py index d3bce47bb8..e2b43387a8 100644 --- a/common/lib/xmodule/xmodule/library_content_module.py +++ b/common/lib/xmodule/xmodule/library_content_module.py @@ -10,6 +10,8 @@ from copy import copy from gettext import ngettext import bleach +from django.conf import settings +from django.utils.functional import classproperty from lazy import lazy from lxml import etree from lxml.etree import XMLSyntaxError @@ -116,7 +118,18 @@ class LibraryContentBlock( show_in_read_only_mode = True - completion_mode = XBlockCompletionMode.AGGREGATOR + # noinspection PyMethodParameters + @classproperty + def completion_mode(cls): # pylint: disable=no-self-argument + """ + Allow overriding the completion mode with a feature flag. + + This is a property, so it can be dynamically overridden in tests, as it is not evaluated at runtime. + """ + if settings.FEATURES.get('MARK_LIBRARY_CONTENT_BLOCK_COMPLETE_ON_VIEW', False): + return XBlockCompletionMode.COMPLETABLE + + return XBlockCompletionMode.AGGREGATOR display_name = String( display_name=_("Display Name"), diff --git a/docs/docs_settings.py b/docs/docs_settings.py index 3b6df8c41a..8c1b813df1 100644 --- a/docs/docs_settings.py +++ b/docs/docs_settings.py @@ -7,8 +7,10 @@ import all the Studio code. import os -from lms.envs.devstack import * # lint-amnesty, pylint: disable=wildcard-import -from cms.envs.devstack import ( # lint-amnesty, pylint: disable=unused-import +from openedx.core.lib.derived import derive_settings + +from lms.envs.common import * # lint-amnesty, pylint: disable=wildcard-import +from cms.envs.common import ( # lint-amnesty, pylint: disable=unused-import ADVANCED_PROBLEM_TYPES, COURSE_IMPORT_EXPORT_STORAGE, GIT_EXPORT_DEFAULT_IDENT, @@ -16,6 +18,7 @@ from cms.envs.devstack import ( # lint-amnesty, pylint: disable=unused-import SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE, VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE, UPDATE_SEARCH_INDEX_JOB_QUEUE, + FRONTEND_REGISTER_URL, ) # Turn on all the boolean feature flags, so that conditionally included @@ -26,13 +29,18 @@ for key, value in FEATURES.items(): # Settings that will fail if we enable them, and we don't need them for docs anyway. FEATURES['RUN_AS_ANALYTICS_SERVER_ENABLED'] = False +FEATURES['ENABLE_SOFTWARE_SECURE_FAKE'] = False +FEATURES['ENABLE_MKTG_SITE'] = False INSTALLED_APPS.extend([ 'cms.djangoapps.contentstore.apps.ContentstoreConfig', 'cms.djangoapps.course_creators', 'cms.djangoapps.xblock_config.apps.XBlockConfig', - 'lms.djangoapps.lti_provider' + 'lms.djangoapps.lti_provider', + 'user_tasks', ]) COMMON_TEST_DATA_ROOT = '' + +derive_settings(__name__) diff --git a/docs/guides/conf.py b/docs/guides/conf.py index f3038eb462..213d131ce3 100644 --- a/docs/guides/conf.py +++ b/docs/guides/conf.py @@ -235,7 +235,7 @@ def update_settings_module(service='lms'): Set the "DJANGO_SETTINGS_MODULE" environment variable appropriately for the module sphinx-apidoc is about to be run on. """ - if os.environ['EDX_PLATFORM_SETTINGS'] == 'devstack_docker': + if os.environ.get('EDX_PLATFORM_SETTINGS') == 'devstack_docker': settings_module = f'{service}.envs.devstack_docker' else: settings_module = f'{service}.envs.devstack' diff --git a/docs/guides/testing/test_pyramid.png b/docs/guides/testing/test_pyramid.png index f5d359bcfd..6100353fb4 100644 Binary files a/docs/guides/testing/test_pyramid.png and b/docs/guides/testing/test_pyramid.png differ diff --git a/docs/guides/testing/testing.rst b/docs/guides/testing/testing.rst index a2c62cf45c..9baf24aee7 100644 --- a/docs/guides/testing/testing.rst +++ b/docs/guides/testing/testing.rst @@ -9,8 +9,7 @@ Testing Overview ======== -We maintain three kinds of tests: unit tests, integration tests, and -acceptance tests. +We maintain two kinds of tests: unit tests and integration tests. Overall, you want to write the tests that **maximize coverage** while **minimizing maintenance**. In practice, this usually means investing @@ -22,8 +21,8 @@ the code base. Test Pyramid -The pyramid above shows the relative number of unit tests, integration -tests, and acceptance tests. Most of our tests are unit tests or +The pyramid above shows the relative number of unit tests and integration +tests. Most of our tests are unit tests or integration tests. Test Types @@ -67,19 +66,6 @@ Integration Tests .. _Django test client: https://docs.djangoproject.com/en/dev/topics/testing/overview/ - -UI Acceptance Tests -~~~~~~~~~~~~~~~~~~~ - -- There should be very few UI acceptance tests since they are generally slow and - flaky. Use these to test only bare minimum happy paths for necessary features. - -- We use `Bok Choy`_ to write end-user acceptance tests directly in Python, - using the framework to maximize reliability and maintainability. - -.. _Bok Choy: https://bok-choy.readthedocs.org/en/latest/tutorial.html - - Test Locations -------------- @@ -94,14 +80,6 @@ Test Locations the test for ``src/views/module.js`` should be written in ``spec/views/module_spec.js``. -- UI acceptance tests: - - - Set up and helper methods, and stubs for external services: - ``common/djangoapps/terrain`` - - Bok Choy Acceptance Tests: located under ``common/test/acceptance/tests`` - - Bok Choy Accessibility Tests: located under ``common/test/acceptance/tests`` and tagged with ``@attr("a11y")`` - - Bok Choy PageObjects: located under ``common/test/acceptance/pages`` - Running Tests ============= @@ -109,8 +87,7 @@ You can run all of the unit-level tests using this command:: paver test -This includes python, JavaScript, and documentation tests. It does not, -however, run any acceptance tests. +This includes python, JavaScript, and documentation tests. Note - `paver` is a scripting tool. To get information about various options, you can run the this command:: @@ -310,226 +287,6 @@ Note: the port is also output to the console that you ran the tests from if you These paver commands call through to Karma. For more info, see `karma-runner.github.io `__. -Running Bok Choy Acceptance Tests ---------------------------------- - -We use `Bok Choy`_ for acceptance testing. Bok Choy is a UI-level acceptance -test framework for writing robust `Selenium`_ tests in `Python`_. Bok Choy -makes your acceptance tests reliable and maintainable by utilizing the Page -Object and Promise design patterns. - -**Prerequisites**: - -These prerequisites are all automatically installed and available in -`Devstack`_, the supported development enviornment for the Open edX platform. - -* Chromedriver and Chrome - -* Mongo - -* Memcache - -* mySQL - -To run all the bok choy acceptance tests run this command:: - - paver test_bokchoy - -Once the database has been set up and the static files collected, you -can use the 'fast' option to skip those tasks. This option can also be -used with any of the test specs below:: - - paver test_bokchoy --fasttest - -For example to run a single test, specify the name of the test file:: - - paver test_bokchoy -t lms/test_lms.py - -Notice the test file location is relative to -common/test/acceptance/tests. This is another example:: - - paver test_bokchoy -t studio/test_studio_bad_data.py - -To run a single test faster by not repeating setup tasks use the ``--fasttest`` option:: - - paver test_bokchoy -t studio/test_studio_bad_data.py --fasttest - -To test only a certain feature, specify the file and the testcase class:: - - paver test_bokchoy -t studio/test_studio_bad_data.py::BadComponentTest - -To execute only a certain test case, specify the file name, class, and -test case method:: - - paver test_bokchoy -t lms/test_lms.py::RegistrationTest::test_register - -During acceptance test execution, log files and also screenshots of -failed tests are captured in test\_root/log. - -Use this command to put a temporary debugging breakpoint in a test. -If you check this in, your tests will hang on jenkins:: - - breakpoint() - -By default, all bokchoy tests are run with the 'split' ModuleStore. To -override the modulestore that is used, use the default\_store option. -The currently supported stores are: 'split' -(xmodule.modulestore.split\_mongo.split\_draft.DraftVersioningModuleStore) -and 'draft' (xmodule.modulestore.mongo.DraftMongoModuleStore). This is an example -for the 'draft' store:: - - paver test_bokchoy --default_store='draft' - -Running Bok Choy Accessibility Tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -We use Bok Choy for `automated accessibility testing`_. Bok Choy, a UI-level -acceptance test framework for writing robust `Selenium`_ tests in `Python`_, -includes the ability to perform accessibility audits on web pages using `Google -Accessibility Developer Tools`_ or `Deque's aXe Core`_. For more details about -how to write accessibility tests, please read the `Bok Choy documentation`_ and -the `Automated Accessibility Tests`_ Open edX Confluence page. - -.. _automated accessibility testing: https://bok-choy.readthedocs.org/en/latest/accessibility.html -.. _Selenium: http://docs.seleniumhq.org/ -.. _Python: https://www.python.org/ -.. _Google Accessibility Developer Tools: https://github.com/GoogleChrome/accessibility-developer-tools/ -.. _Deque's aXe Core: https://github.com/dequelabs/axe-core/ -.. _Bok Choy documentation: https://bok-choy.readthedocs.org/en/latest/accessibility.html -.. _Automated Accessibility Tests: https://openedx.atlassian.net/wiki/display/TE/Automated+Accessibility+Tests - - -**Prerequisites**: - -These prerequisites are all automatically installed and available in -`Devstack`_ (since the Cypress release), the supported development environment -for the Open edX platform. - -.. _Devstack: https://github.com/edx/configuration/wiki/edX-Developer-Stack - -* Mongo - -* Memcache - -* mySQL - -To run all the bok choy accessibility tests use this command:: - - paver test_a11y - -To run specific tests, use the ``-t`` flag to specify a pytest-style test spec -relative to the ``common/test/acceptance/tests`` directory. This is an example for it:: - - paver test_a11y -t lms/test_lms_dashboard.py::LmsDashboardA11yTest::test_dashboard_course_listings_a11y - -**Coverage**: - -To generate the coverage report for the views run during accessibility tests:: - - paver a11y_coverage - -Note that this coverage report is just a guideline to find areas that -are missing tests. If the view isn't 'covered', there definitely -isn't a test for it. If it is 'covered', we are loading that page -during the tests but not necessarily calling ``page.a11y_audit.check_for_accessibility_errors`` on it. - - -Options for Faster Development Cycles in Bok-Choy Tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following are ways in which a developer could shorten the development -cycle for faster feedback. The options below can often be used together. - -**Multiprocessing Mode** - -Bok-choy tests can be threaded using the `-n` switch. Using 2 threads generally -reduces test cycles by 33%. The recommendation is to make sure the -number of threads is no more than the number of processors available. For -example, the Cypress release of devstack is provisioned by default with 2 -processors. In that case, to run tests in multiprocess mode:: - - paver test_bokchoy -n 2 - -*Caveat*: Not all tests have been designed with multiprocessing in mind; some -testcases (approx 10%) will fail in multiprocess mode for various reasons -(e.g., shared fixtures, unexpected state, etc). If you have tests that fail -in multiprocessing mode, it may be worthwhile to run them in single-stream mode -to understand if you are encountering such a failure. With that noted, this -can speed development for most test classes. - -**Leave Your Servers Running** - -There are two additional switches available in the `paver test_bokchoy` task. -Used together, they can shorten the cycle between test runs. Similar to above, -there are a handful of tests that won't work with this approach, due to insufficient -teardown and other unmanaged state. - -1. Start your servers in one terminal/ssh session:: - - paver test_bokchoy --serversonly - - Note if setup has already been done, you can run:: - - paver test_bokchoy --serversonly --fasttest - -2. Run your tests only in another terminal/ssh session:: - - paver test_bokchoy --testsonly --fasttest - -You must run BOTH `--testsonly` and `--fasttest`. - -3. When done, you can kill your servers in the first terminal/ssh session with -Control-C. *Warning*: Only hit Control-C one time so the pytest framework can -properly clean up. - -Acceptance Test Techniques -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -1. **Element existence on the page**: Do not use splinter's built-in browser - methods directly for determining if elements exist. Use the - world.is\_css\_present and world.is\_css\_not\_present wrapper - functions instead. Otherwise errors can arise if checks for the css - are performed before the page finishes loading. Also these wrapper - functions are optimized for the amount of wait time spent in both - cases of positive and negative expectation. - -2. **Dealing with alerts**: Chrome can hang on javascripts alerts. If a - javascript alert/prompt/confirmation is expected, use the step 'I - will confirm all alerts', 'I will cancel all alerts' or 'I will anser - all prompts with "(.\*)"' before the step that causes the alert in - order to properly deal with it. - -3. **Dealing with stale element reference exceptions**: These exceptions - happen if any part of the page is refreshed in between finding an - element and accessing the element. When possible, use any of the css - functions in common/djangoapps/terrain/ui\_helpers.py as they will - retry the action in case of this exception. If the functionality is - not there, wrap the function with world.retry\_on\_exception. This - function takes in a function and will retry and return the result of - the function if there was an exception. - -4. **Scenario Level Constants**: If you want an object to be available for - the entire scenario, it can be stored in world.scenario\_dict. This - object is a dictionary that gets refreshed at the beginning on the - scenario. Currently, the current logged in user and the current - created course are stored under 'COURSE' and 'USER'. This will help - prevent strings from being hard coded so the acceptance tests can - become more flexible. - -5. **Internal edX Jenkins considerations**: Acceptance tests are run in - Jenkins as part of the edX development workflow. They are broken into - shards and split across workers. Therefore if you add a new .feature - file, you need to define what shard they should be run in or else - they will not get executed. See someone from TestEng to help you - determine where they should go. - - Also, the test results are rolled up in Jenkins for ease of - understanding, with the acceptance tests under the top level of "CMS" - and "LMS" when they follow this convention: name your feature in the - .feature file CMS or LMS with a single period and then no other - periods in the name. The name can contain spaces. E.g. "CMS.Sign Up" - - Testing internationalization with dummy translations ---------------------------------------------------- @@ -645,7 +402,7 @@ Other Testing Tips Connecting to Browser --------------------- -If you want to see the browser being automated for JavaScript or bok-choy tests, +If you want to see the browser being automated for JavaScript, you can connect to the container running it via VNC. +------------------------+----------------------+ diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 148ac02010..48ca6e9ac6 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2527,7 +2527,6 @@ paths: as an object with the following fields: * uri: The location of the image * name: Name of the course - * number: Catalog number of the course * offer: An object detailing upgrade discount information * code: (str) Checkout code * expiration_date: (str) Expiration of offer, in ISO 8601 notation @@ -2535,7 +2534,6 @@ paths: * discounted_price: (str) Upgrade price with checkout code; includes currency symbol * percentage: (int) Amount of discount * upgrade_url: (str) Checkout URL - * org: Name of the organization that owns the course * related_programs: A list of objects that contains program data related to the given course including: * progress: An object containing program progress: * complete: (int) Number of complete courses in the program (a course is completed if the user has @@ -2556,8 +2554,6 @@ paths: * `"empty"`: no start date is specified * pacing: Course pacing. Possible values: instructor, self * user_timezone: User's chosen timezone setting (or null for browser default) - * is_staff: Whether the effective user has staff access to the course - * original_user_is_staff: Whether the original user has staff access to the course * can_view_legacy_courseware: Indicates whether the user is able to see the legacy courseware view * user_has_passing_grade: Whether or not the effective user's grade is equal to or above the courses minimum passing grade diff --git a/lms/djangoapps/bulk_email/api.py b/lms/djangoapps/bulk_email/api.py index 5c2e40ee64..f89a27065b 100644 --- a/lms/djangoapps/bulk_email/api.py +++ b/lms/djangoapps/bulk_email/api.py @@ -4,11 +4,12 @@ Python APIs exposed by the bulk_email app to other in-process apps. """ # Public Bulk Email Functions - +import logging from django.conf import settings from django.urls import reverse +from lms.djangoapps.bulk_email.models import CourseEmail from lms.djangoapps.bulk_email.models_api import ( is_bulk_email_disabled_for_course, is_bulk_email_enabled_for_course, @@ -18,6 +19,8 @@ from lms.djangoapps.bulk_email.models_api import ( from lms.djangoapps.discussion.notification_prefs.views import UsernameCipher from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers +log = logging.getLogger(__name__) + def get_emails_enabled(user, course_id): """ @@ -48,3 +51,39 @@ def get_unsubscribed_link(username, course_id): optout_url = reverse('bulk_email_opt_out', kwargs={'token': token, 'course_id': course_id}) url = f'{lms_root_url}{optout_url}' return url + + +def create_course_email(course_id, sender, targets, subject, html_message, text_message=None, template_name=None, + from_addr=None): + """ + Python API for creating a new CourseEmail instance. + + Args: + course_id (CourseKey): The CourseKey of the course. + sender (String): Email author. + targets (Target): Recipient groups the message should be sent to (e.g. SEND_TO_MYSELF) + subject (String)): Email subject. + html_message (String): Email body. Includes HTML markup. + text_message (String, optional): Plaintext version of email body. Defaults to None. + template_name (String, optional): Name of custom email template to use. Defaults to None. + from_addr (String, optional): Custom sending address, if desired. Defaults to None. + + Returns: + CourseEmail: Returns the created CourseEmail instance. + """ + try: + course_email = CourseEmail.create( + course_id, + sender, + targets, + subject, + html_message, + text_message=text_message, + template_name=template_name, + from_addr=from_addr + ) + + return course_email + except ValueError as err: + log.exception(f"Cannot create course email for {course_id} requested by user {sender} for targets {targets}") + raise ValueError from err diff --git a/lms/djangoapps/bulk_email/data.py b/lms/djangoapps/bulk_email/data.py new file mode 100644 index 0000000000..a0f50c6c6b --- /dev/null +++ b/lms/djangoapps/bulk_email/data.py @@ -0,0 +1,22 @@ +""" +Bulk Email Data + +This provides Data models to represent Bulk Email data. +""" + + +class BulkEmailTargetChoices: + """ + Enum for the available targets (recipient groups) of an email authored with the bulk course email tool. + + SEND_TO_MYSELF - Message intended for author of the message + SEND_TO_STAFF - Message intended for all course staff + SEND_TO_LEARNERS - Message intended for all enrolled learners + SEND_TO_COHORT - Message intended for a specific cohort + SEND_TO_TRACK - Message intended for all learners in a specific track (e.g. audit or verified) + """ + SEND_TO_MYSELF = "myself" + SEND_TO_STAFF = "staff" + SEND_TO_LEARNERS = "learners" + SEND_TO_COHORT = "cohort" + SEND_TO_TRACK = "track" diff --git a/lms/djangoapps/bulk_email/tasks.py b/lms/djangoapps/bulk_email/tasks.py index d6b8393f47..4d86edcc10 100644 --- a/lms/djangoapps/bulk_email/tasks.py +++ b/lms/djangoapps/bulk_email/tasks.py @@ -10,7 +10,7 @@ import re import time from collections import Counter from datetime import datetime -from smtplib import SMTPConnectError, SMTPDataError, SMTPException, SMTPServerDisconnected +from smtplib import SMTPConnectError, SMTPDataError, SMTPException, SMTPServerDisconnected, SMTPSenderRefused from time import sleep from boto.exception import AWSConnectionError @@ -87,12 +87,13 @@ LIMITED_RETRY_ERRORS = ( # An example is if email is being sent too quickly, but may succeed if sent # more slowly. When caught by a task, it triggers an exponential backoff and retry. # Retries happen continuously until the email is sent. -# Note that the SMTPDataErrors here are only those within the 4xx range. +# Note that the (SMTPDataErrors and SMTPSenderRefused) here are only those within the 4xx range. # Those not in this range (i.e. in the 5xx range) are treated as hard failures # and thus like SINGLE_EMAIL_FAILURE_ERRORS. INFINITE_RETRY_ERRORS = ( SESMaxSendingRateExceededError, # Your account's requests/second limit has been exceeded. SMTPDataError, + SMTPSenderRefused, ) # Errors that are known to indicate an inability to send any more emails, @@ -565,11 +566,11 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas f"{recipient_num}/{total_recipients}, Recipient UserId: {current_recipient['pk']}" ) message.send() - except SMTPDataError as exc: + except (SMTPDataError, SMTPSenderRefused) as exc: # According to SMTP spec, we'll retry error codes in the 4xx range. 5xx range indicates hard failure. total_recipients_failed += 1 log.exception( - f"BulkEmail ==> Status: Failed(SMTPDataError), Task: {parent_task_id}, SubTask: {task_id}, " + f"BulkEmail ==> Status: Failed({exc.smtp_error}), Task: {parent_task_id}, SubTask: {task_id}, " f"EmailId: {email_id}, Recipient num: {recipient_num}/{total_recipients}, Recipient UserId: " f"{current_recipient['pk']}" ) diff --git a/lms/djangoapps/bulk_email/tests/test_api.py b/lms/djangoapps/bulk_email/tests/test_api.py new file mode 100644 index 0000000000..f6c0804724 --- /dev/null +++ b/lms/djangoapps/bulk_email/tests/test_api.py @@ -0,0 +1,96 @@ +""" +Tests for the public Python API functions of the Bulk Email app. +""" +from testfixtures import LogCapture + +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + +from common.djangoapps.student.tests.factories import InstructorFactory +from lms.djangoapps.bulk_email.api import create_course_email +from lms.djangoapps.bulk_email.data import BulkEmailTargetChoices +from openedx.core.lib.html_to_text import html_to_text + + +class CreateCourseEmailTests(ModuleStoreTestCase): + """ + Tests for the `create_course_email` function of the bulk email app's public Python API. + """ + def setUp(self): + super().setUp() + self.course = CourseFactory.create() + self.instructor = InstructorFactory(course_key=self.course.id) + self.target = [BulkEmailTargetChoices.SEND_TO_MYSELF] + self.subject = "email subject" + self.html_message = "

test message

" + + def test_create_course_email(self): + """ + Happy path test for the `create_course_email` function. Verifies the creation of a CourseEmail instance with + the bare minimum information required for the function call. + """ + course_email = create_course_email( + self.course.id, + self.instructor, + self.target, + self.subject, + self.html_message, + ) + + assert course_email.sender.id == self.instructor.id + assert course_email.subject == self.subject + assert course_email.html_message == self.html_message + assert course_email.course_id == self.course.id + assert course_email.text_message == html_to_text(self.html_message) + + def test_create_course_email_with_optional_args(self): + """ + Additional testing to verify that optional data is used as expected when passed into the `create_course_email` + function. + """ + text_message = "everything is awesome!" + template_name = "gnarly_template" + from_addr = "blub@noreply.fish.com" + + course_email = create_course_email( + self.course.id, + self.instructor, + self.target, + self.subject, + self.html_message, + text_message=text_message, + template_name=template_name, + from_addr=from_addr + ) + + assert course_email.sender.id == self.instructor.id + assert course_email.subject == self.subject + assert course_email.html_message == self.html_message + assert course_email.course_id == self.course.id + assert course_email.text_message == text_message + assert course_email.template_name == template_name + assert course_email.from_addr == from_addr + + def test_create_course_email_expect_exception(self): + """ + Test to verify behavior when an exception occurs when calling teh `create_course_email` function. + """ + targets = ["humpty dumpty"] + + expected_messages = [ + f"Cannot create course email for {self.course.id} requested by user {self.instructor} for targets " + f"{targets}", + ] + + with self.assertRaises(ValueError): + with LogCapture() as log: + create_course_email( + self.course.id, + self.instructor, + targets, + self.subject, + self.html_message + ) + + for index, message in enumerate(expected_messages): + assert message in log.records[index].getMessage() diff --git a/lms/djangoapps/bulk_email/tests/test_tasks.py b/lms/djangoapps/bulk_email/tests/test_tasks.py index 51f7b4161c..e73db046aa 100644 --- a/lms/djangoapps/bulk_email/tests/test_tasks.py +++ b/lms/djangoapps/bulk_email/tests/test_tasks.py @@ -11,7 +11,7 @@ from datetime import datetime from dateutil.relativedelta import relativedelta import json # lint-amnesty, pylint: disable=wrong-import-order from itertools import chain, cycle, repeat # lint-amnesty, pylint: disable=wrong-import-order -from smtplib import SMTPAuthenticationError, SMTPConnectError, SMTPDataError, SMTPServerDisconnected # lint-amnesty, pylint: disable=wrong-import-order +from smtplib import SMTPAuthenticationError, SMTPConnectError, SMTPDataError, SMTPServerDisconnected, SMTPSenderRefused # lint-amnesty, pylint: disable=wrong-import-order from unittest.mock import Mock, patch # lint-amnesty, pylint: disable=wrong-import-order from uuid import uuid4 # lint-amnesty, pylint: disable=wrong-import-order import pytest @@ -411,6 +411,11 @@ class TestBulkEmailInstructorTask(InstructorTaskCourseTestCase): def test_retry_after_smtp_throttling_error(self): self._test_retry_after_unlimited_retry_error(SMTPDataError(455, "Throttling: Sending rate exceeded")) + def test_retry_after_smtp_sender_refused_error(self): + self._test_retry_after_unlimited_retry_error( + SMTPSenderRefused(421, "Throttling: Sending rate exceeded", self.instructor.email) + ) + def test_retry_after_ses_throttling_error(self): self._test_retry_after_unlimited_retry_error( SESMaxSendingRateExceededError(455, "Throttling: Sending rate exceeded") diff --git a/lms/djangoapps/courseware/tabs.py b/lms/djangoapps/courseware/tabs.py index 45c9a8ed65..f599e9b3df 100644 --- a/lms/djangoapps/courseware/tabs.py +++ b/lms/djangoapps/courseware/tabs.py @@ -332,7 +332,6 @@ class DatesTab(EnrolledTab): title = gettext_noop("Dates") priority = 30 view_name = "dates" - is_dynamic = True def __init__(self, tab_dict): def link_func(course, reverse_func): @@ -370,10 +369,6 @@ def get_course_tab_list(user, course): if tab.type == 'static_tab' and tab.course_staff_only and \ not bool(user and has_access(user, 'staff', course, course.id)): continue - # We are phasing this out in https://github.com/openedx/edx-platform/pull/30045/, but need this - # until the backfill course tabs command is completed - if tab.type == 'dates': - continue course_tab_list.append(tab) # Add in any dynamic tabs, i.e. those that are not persisted diff --git a/lms/djangoapps/courseware/tests/test_discussion_xblock.py b/lms/djangoapps/courseware/tests/test_discussion_xblock.py index 4e191f052c..aa77725c2c 100644 --- a/lms/djangoapps/courseware/tests/test_discussion_xblock.py +++ b/lms/djangoapps/courseware/tests/test_discussion_xblock.py @@ -12,9 +12,7 @@ import uuid from unittest import mock import ddt -from django.test import override_settings from django.urls import reverse -from edx_toggles.toggles.testutils import override_waffle_flag from opaque_keys.edx.keys import CourseKey from web_fragments.fragment import Fragment from xblock.field_data import DictFieldData @@ -26,7 +24,6 @@ from xmodule.modulestore.tests.factories import ItemFactory, ToyCourseFactory from lms.djangoapps.course_api.blocks.tests.helpers import deserialize_usage_key from lms.djangoapps.courseware.module_render import get_module_for_descriptor_internal from lms.djangoapps.courseware.tests.helpers import XModuleRenderingTestBase -from lms.djangoapps.discussion.toggles import ENABLE_DISCUSSIONS_MFE from common.djangoapps.student.tests.factories import CourseEnrollmentFactory, UserFactory @@ -308,34 +305,6 @@ class TestXBlockInCourse(SharedModuleStoreTestCase): assert 'data-user-create-comment="false"' in html assert 'data-user-create-subcomment="false"' in html - @override_settings(DISCUSSIONS_MICROFRONTEND_URL="http://test.url") - @override_waffle_flag(ENABLE_DISCUSSIONS_MFE, True) - def test_embed_mfe_in_course(self): - """ - Test that the xblock embeds the MFE UI when the flag is enabled - """ - discussion_xblock = get_module_for_descriptor_internal( - user=self.user, - descriptor=self.discussion, - student_data=mock.Mock(name='student_data'), - course_id=self.course.id, - track_function=mock.Mock(name='track_function'), - request_token='request_token', - ) - - fragment = discussion_xblock.render('student_view') - html = fragment.content - self.assertInHTML( - """ - " - ).format(src=mfe_url, title=_("Discussions"))) - fragment.add_css( - """ - #discussions-mfe-tab-embed { - width: 100%; - height: 800px; - border: none; - } - """ - ) - return fragment - self.add_resource_urls(fragment) login_msg = '' diff --git a/openedx/core/tests/test_admin_view.py b/openedx/core/tests/test_admin_view.py index b0a70ac965..1eafb9f0de 100644 --- a/openedx/core/tests/test_admin_view.py +++ b/openedx/core/tests/test_admin_view.py @@ -6,9 +6,8 @@ This is not inside a django app because it is a global property of the system. from django.test import Client, TestCase from django.urls import reverse -from edx_toggles.toggles.testutils import override_waffle_switch, override_waffle_flag +from edx_toggles.toggles.testutils import override_waffle_switch from common.djangoapps.student.tests.factories import UserFactory, TEST_PASSWORD -from openedx.core.djangoapps.user_authn.config.waffle import ADMIN_AUTH_REDIRECT_TO_LMS from openedx.core.djangoapps.user_authn.views.login import ENABLE_LOGIN_USING_THIRDPARTY_AUTH_ONLY @@ -44,16 +43,3 @@ class TestAdminView(TestCase): response = self.client.get(reverse('admin:login')) assert response.url == '/login?next=/admin' assert response.status_code == 302 - - with override_waffle_flag(ADMIN_AUTH_REDIRECT_TO_LMS, True): - response = self.client.get(reverse('admin:login')) - assert response.url == '/login?next=/admin' - assert response.status_code == 302 - - with override_waffle_switch(ENABLE_LOGIN_USING_THIRDPARTY_AUTH_ONLY, False): - response = self.client.get(reverse('admin:login')) - assert response.template_name == ['admin/login.html'] - - with override_waffle_flag(ADMIN_AUTH_REDIRECT_TO_LMS, False): - response = self.client.get(reverse('admin:login')) - assert response.template_name == ['admin/login.html'] diff --git a/openedx/features/course_experience/utils.py b/openedx/features/course_experience/utils.py index b343c81e3c..4c370e1f29 100644 --- a/openedx/features/course_experience/utils.py +++ b/openedx/features/course_experience/utils.py @@ -51,7 +51,7 @@ def get_course_outline_block_tree(request, course_id, user=None, allow_start_dat is_scored = block.get('has_score', False) and block.get('weight', 1) > 0 # Use a list comprehension to force the recursion over all children, rather than just stopping # at the first child that is scored. - children_scored = any(recurse_mark_scored(child) for child in block.get('children', [])) + children_scored = any(tuple(recurse_mark_scored(child) for child in block.get('children', []))) if is_scored or children_scored: block['scored'] = True return True diff --git a/openedx/tests/completion_integration/test_services.py b/openedx/tests/completion_integration/test_services.py index b24089c86e..0529ec7b71 100644 --- a/openedx/tests/completion_integration/test_services.py +++ b/openedx/tests/completion_integration/test_services.py @@ -7,6 +7,8 @@ import ddt from completion.models import BlockCompletion from completion.services import CompletionService from completion.test_utils import CompletionWaffleTestMixin +from django.conf import settings +from django.test import override_settings from opaque_keys.edx.keys import CourseKey from xmodule.library_tools import LibraryToolsService from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase @@ -181,6 +183,19 @@ class CompletionServiceTestCase(CompletionWaffleTestMixin, SharedModuleStoreTest assert self.completion_service.can_mark_block_complete_on_view(self.html) is True assert self.completion_service.can_mark_block_complete_on_view(self.problem) is False + @override_settings(FEATURES={**settings.FEATURES, 'MARK_LIBRARY_CONTENT_BLOCK_COMPLETE_ON_VIEW': True}) + def test_can_mark_library_content_complete_on_view(self): + library = LibraryFactory.create(modulestore=self.store) + lib_vertical = ItemFactory.create(parent=self.sequence, category='vertical', publish_item=False) + library_content_block = ItemFactory.create( + parent=lib_vertical, + category='library_content', + max_count=1, + source_library_id=str(library.location.library_key), + user_id=self.user.id, + ) + self.assertTrue(self.completion_service.can_mark_block_complete_on_view(library_content_block)) + def test_vertical_completion_with_library_content(self): library = LibraryFactory.create(modulestore=self.store) ItemFactory.create(parent=library, category='problem', publish_item=False, user_id=self.user.id) @@ -202,6 +217,9 @@ class CompletionServiceTestCase(CompletionWaffleTestMixin, SharedModuleStoreTest source_library_id=str(library.location.library_key), user_id=self.user.id, ) + # Library Content Block needs its children to be completed. + self.assertFalse(self.completion_service.can_mark_block_complete_on_view(library_content_block)) + library_content_block.refresh_children() lib_vertical = self.store.get_item(lib_vertical.location) self._bind_course_module(lib_vertical) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 799b062bba..fa3412bb01 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -31,7 +31,7 @@ django-storages<1.9 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==3.40.16 +edx-enterprise==3.41.10 # Newer versions need a more recent version of python-dateutil freezegun==0.3.12 diff --git a/requirements/edx-sandbox/py38.txt b/requirements/edx-sandbox/py38.txt index 0fc9c65db9..482679a6b7 100644 --- a/requirements/edx-sandbox/py38.txt +++ b/requirements/edx-sandbox/py38.txt @@ -89,5 +89,5 @@ sympy==1.6.2 # -c requirements/edx-sandbox/../constraints.txt # -r requirements/edx-sandbox/py38.in # openedx-calc -tqdm==4.63.0 +tqdm==4.63.1 # via nltk diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 9d6208fe20..27b8b7d34f 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -47,7 +47,9 @@ appdirs==1.4.4 asgiref==3.5.0 # via django async-timeout==4.0.2 - # via aiohttp + # via + # aiohttp + # redis attrs==21.4.0 # via # -r requirements/edx/base.in @@ -195,7 +197,6 @@ django==3.2.12 # django-mysql # django-oauth-toolkit # django-pyfs - # django-ratelimit-backend # django-sekizai # django-ses # django-splash @@ -330,15 +331,13 @@ django-pyfs==3.2.0 # via -r requirements/edx/base.in django-ratelimit==3.0.1 # via -r requirements/edx/base.in -django-ratelimit-backend @ git+https://github.com/edx/django-ratelimit-backend.git@6e1a0c6ea1d27062c16e9fb94d3c44475146877e - # via -r requirements/edx/github.in django-require @ git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc3211822a776 # via -r requirements/edx/github.in django-sekizai==3.0.1 # via # -r requirements/edx/base.in # django-wiki -django-ses==2.6.0 +django-ses==2.6.1 # via -r requirements/edx/base.in django-simple-history==3.0.0 # via @@ -428,9 +427,10 @@ edx-django-release-util==1.2.0 # via -r requirements/edx/base.in edx-django-sites-extensions==4.0.0 # via -r requirements/edx/base.in -edx-django-utils==4.5.0 +edx-django-utils==4.6.0 # via # -r requirements/edx/base.in + # -r requirements/edx/github.in # django-config-models # edx-drf-extensions # edx-enterprise @@ -452,7 +452,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.40.16 +edx-enterprise==3.41.10 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in @@ -552,7 +552,6 @@ fs-s3fs==0.1.8 # django-pyfs future==0.18.2 # via - # django-ses # edx-celeryutils # pyjwkest geoip2==4.5.0 @@ -625,7 +624,7 @@ libsass==0.10.0 # ora2 loremipsum==1.0.5 # via ora2 -lti-consumer-xblock==3.4.4 +lti-consumer-xblock==3.4.5 # via -r requirements/edx/base.in lxml==4.5.0 # via @@ -673,7 +672,7 @@ mock==4.0.3 # xblock-poll mongodbproxy @ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a # via -r requirements/edx/github.in -mongoengine==0.24.0 +mongoengine==0.24.1 # via -r requirements/edx/base.in monotonic==1.6 # via @@ -716,7 +715,7 @@ openedx-events==0.8.1 # via -r requirements/edx/base.in openedx-filters==0.5.0 # via -r requirements/edx/base.in -ora2==4.0.4 +ora2==4.0.6 # via -r requirements/edx/base.in packaging==21.3 # via @@ -849,7 +848,7 @@ python3-saml==1.9.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in -pytz==2021.3 +pytz==2022.1 # via # -r requirements/edx/base.in # babel @@ -913,7 +912,7 @@ ruamel-yaml==0.17.21 # via drf-yasg ruamel-yaml-clib==0.2.6 # via ruamel-yaml -rules==3.2.1 +rules==3.3 # via # -r requirements/edx/base.in # edx-enterprise @@ -1019,10 +1018,12 @@ testfixtures==6.18.5 # via edx-enterprise text-unidecode==1.3 # via python-slugify -tqdm==4.63.0 +tqdm==4.63.1 # via nltk typing-extensions==4.1.1 - # via django-countries + # via + # django-countries + # redis unicodecsv==0.14.1 # via # -r requirements/edx/base.in @@ -1031,7 +1032,7 @@ uritemplate==4.1.1 # via # coreapi # drf-yasg -urllib3==1.26.8 +urllib3==1.26.9 # via # -r requirements/edx/paver.txt # elasticsearch diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 5df20f13fb..37f2f3b1a2 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -74,6 +74,7 @@ async-timeout==4.0.2 # via # -r requirements/edx/testing.txt # aiohttp + # redis attrs==21.4.0 # via # -r requirements/edx/testing.txt @@ -279,7 +280,6 @@ django==3.2.12 # django-mysql # django-oauth-toolkit # django-pyfs - # django-ratelimit-backend # django-sekizai # django-ses # django-splash @@ -426,15 +426,13 @@ django-pyfs==3.2.0 # via -r requirements/edx/testing.txt django-ratelimit==3.0.1 # via -r requirements/edx/testing.txt -django-ratelimit-backend @ git+https://github.com/edx/django-ratelimit-backend.git@6e1a0c6ea1d27062c16e9fb94d3c44475146877e - # via -r requirements/edx/testing.txt django-require @ git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc3211822a776 # via -r requirements/edx/testing.txt django-sekizai==3.0.1 # via # -r requirements/edx/testing.txt # django-wiki -django-ses==2.6.0 +django-ses==2.6.1 # via -r requirements/edx/testing.txt django-simple-history==3.0.0 # via @@ -535,7 +533,7 @@ edx-django-release-util==1.2.0 # via -r requirements/edx/testing.txt edx-django-sites-extensions==4.0.0 # via -r requirements/edx/testing.txt -edx-django-utils==4.5.0 +edx-django-utils==4.6.0 # via # -r requirements/edx/testing.txt # django-config-models @@ -559,7 +557,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.40.16 +edx-enterprise==3.41.10 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt @@ -660,7 +658,7 @@ execnet==1.9.0 # pytest-xdist factory-boy==3.2.1 # via -r requirements/edx/testing.txt -faker==13.3.2 +faker==13.3.3 # via # -r requirements/edx/testing.txt # factory-boy @@ -699,7 +697,6 @@ fs-s3fs==0.1.8 future==0.18.2 # via # -r requirements/edx/testing.txt - # django-ses # edx-celeryutils # pyjwkest geoip2==4.5.0 @@ -841,7 +838,7 @@ loremipsum==1.0.5 # via # -r requirements/edx/testing.txt # ora2 -lti-consumer-xblock==3.4.4 +lti-consumer-xblock==3.4.5 # via -r requirements/edx/testing.txt lxml==4.5.0 # via @@ -904,7 +901,7 @@ mock==4.0.3 # xblock-poll mongodbproxy @ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a # via -r requirements/edx/testing.txt -mongoengine==0.24.0 +mongoengine==0.24.1 # via -r requirements/edx/testing.txt monotonic==1.6 # via @@ -956,7 +953,7 @@ openedx-events==0.8.1 # via -r requirements/edx/testing.txt openedx-filters==0.5.0 # via -r requirements/edx/testing.txt -ora2==4.0.4 +ora2==4.0.6 # via -r requirements/edx/testing.txt packaging==21.3 # via @@ -968,7 +965,7 @@ packaging==21.3 # redis # sphinx # tox -pact-python==1.5.1 +pact-python==1.5.2 # via -r requirements/edx/testing.txt pansi==2020.7.3 # via @@ -1134,7 +1131,7 @@ pysrt==1.1.2 # via # -r requirements/edx/testing.txt # edxval -pytest==7.1.0 +pytest==7.1.1 # via # -r requirements/edx/testing.txt # pylint-pytest @@ -1204,7 +1201,7 @@ python3-saml==1.9.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt -pytz==2021.3 +pytz==2022.1 # via # -r requirements/edx/testing.txt # babel @@ -1280,7 +1277,7 @@ ruamel-yaml-clib==0.2.6 # via # -r requirements/edx/testing.txt # ruamel-yaml -rules==3.2.1 +rules==3.3 # via # -r requirements/edx/testing.txt # edx-enterprise @@ -1472,7 +1469,7 @@ tox==3.24.5 # tox-battery tox-battery==0.6.1 # via -r requirements/edx/testing.txt -tqdm==4.63.0 +tqdm==4.63.1 # via # -r requirements/edx/testing.txt # nltk @@ -1486,6 +1483,7 @@ typing-extensions==4.1.1 # mypy # pydantic # pylint + # redis unicodecsv==0.14.1 # via # -r requirements/edx/testing.txt @@ -1497,7 +1495,7 @@ uritemplate==4.1.1 # -r requirements/edx/testing.txt # coreapi # drf-yasg -urllib3==1.26.8 +urllib3==1.26.9 # via # -r requirements/edx/testing.txt # elasticsearch @@ -1519,7 +1517,7 @@ vine==5.0.0 # amqp # celery # kombu -virtualenv==20.13.3 +virtualenv==20.13.4 # via # -r requirements/edx/testing.txt # tox diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 3000afd3a8..8f3930580c 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -52,7 +52,7 @@ python-slugify==4.0.1 # via # -c requirements/edx/../constraints.txt # code-annotations -pytz==2021.3 +pytz==2022.1 # via babel pyyaml==6.0 # via code-annotations @@ -84,7 +84,7 @@ stevedore==3.5.0 # via code-annotations text-unidecode==1.3 # via python-slugify -urllib3==1.26.8 +urllib3==1.26.9 # via requests zipp==3.7.0 # via importlib-metadata diff --git a/requirements/edx/github.in b/requirements/edx/github.in index 54d08e572a..50bfa1da6c 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -59,10 +59,6 @@ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 -e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme -# This is a temporary fork until https://github.com/brutasse/django-ratelimit-backend/pull/50 is merged -# back into the upstream code. -git+https://github.com/edx/django-ratelimit-backend.git@6e1a0c6ea1d27062c16e9fb94d3c44475146877e#egg=django-ratelimit-backend - # original repo is not maintained any more. git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc3211822a776#egg=django-require==1.0.12 @@ -75,3 +71,6 @@ git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc32118 git+https://github.com/open-craft/xblock-poll@v1.12.0#egg=xblock-poll==1.12.0 git+https://github.com/edx-solutions/xblock-drag-and-drop-v2@v2.3.5#egg=xblock-drag-and-drop-v2==2.3.5 + +# Temporary for testing edx-django-utils upgrade +git+https://github.com/edx/edx-django-utils.git@robrap/ARCHBOM-2054-move-cookie-monitoring-middleware#egg=edx_django_utils diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index cf24727b36..d71382c9e5 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -46,7 +46,7 @@ stevedore==3.5.0 # via # -r requirements/edx/paver.in # edx-opaque-keys -urllib3==1.26.8 +urllib3==1.26.9 # via requests watchdog==2.1.6 # via -r requirements/edx/paver.in diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 782b589874..c3a5c59f85 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -69,6 +69,7 @@ async-timeout==4.0.2 # via # -r requirements/edx/base.txt # aiohttp + # redis attrs==21.4.0 # via # -r requirements/edx/base.txt @@ -267,7 +268,6 @@ distlib==0.3.4 # django-mysql # django-oauth-toolkit # django-pyfs - # django-ratelimit-backend # django-sekizai # django-ses # django-splash @@ -412,15 +412,13 @@ django-pyfs==3.2.0 # via -r requirements/edx/base.txt django-ratelimit==3.0.1 # via -r requirements/edx/base.txt -django-ratelimit-backend @ git+https://github.com/edx/django-ratelimit-backend.git@6e1a0c6ea1d27062c16e9fb94d3c44475146877e - # via -r requirements/edx/base.txt django-require @ git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc3211822a776 # via -r requirements/edx/base.txt django-sekizai==3.0.1 # via # -r requirements/edx/base.txt # django-wiki -django-ses==2.6.0 +django-ses==2.6.1 # via -r requirements/edx/base.txt django-simple-history==3.0.0 # via @@ -519,7 +517,7 @@ edx-django-release-util==1.2.0 # via -r requirements/edx/base.txt edx-django-sites-extensions==4.0.0 # via -r requirements/edx/base.txt -edx-django-utils==4.5.0 +edx-django-utils==4.6.0 # via # -r requirements/edx/base.txt # django-config-models @@ -543,7 +541,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.40.16 +edx-enterprise==3.41.10 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -641,7 +639,7 @@ execnet==1.9.0 # via pytest-xdist factory-boy==3.2.1 # via -r requirements/edx/testing.in -faker==13.3.2 +faker==13.3.3 # via factory-boy fastapi==0.75.0 # via pact-python @@ -675,7 +673,6 @@ fs-s3fs==0.1.8 future==0.18.2 # via # -r requirements/edx/base.txt - # django-ses # edx-celeryutils # pyjwkest geoip2==4.5.0 @@ -800,7 +797,7 @@ loremipsum==1.0.5 # via # -r requirements/edx/base.txt # ora2 -lti-consumer-xblock==3.4.4 +lti-consumer-xblock==3.4.5 # via -r requirements/edx/base.txt lxml==4.5.0 # via @@ -856,7 +853,7 @@ mock==4.0.3 # xblock-poll mongodbproxy @ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a # via -r requirements/edx/base.txt -mongoengine==0.24.0 +mongoengine==0.24.1 # via -r requirements/edx/base.txt monotonic==1.6 # via @@ -904,7 +901,7 @@ openedx-events==0.8.1 # via -r requirements/edx/base.txt openedx-filters==0.5.0 # via -r requirements/edx/base.txt -ora2==4.0.4 +ora2==4.0.6 # via -r requirements/edx/base.txt packaging==21.3 # via @@ -915,7 +912,7 @@ packaging==21.3 # pytest # redis # tox -pact-python==1.5.1 +pact-python==1.5.2 # via -r requirements/edx/testing.in pansi==2020.7.3 # via @@ -1064,7 +1061,7 @@ pysrt==1.1.2 # via # -r requirements/edx/base.txt # edxval -pytest==7.1.0 +pytest==7.1.1 # via # -r requirements/edx/testing.in # pylint-pytest @@ -1132,7 +1129,7 @@ python3-saml==1.9.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -pytz==2021.3 +pytz==2022.1 # via # -r requirements/edx/base.txt # babel @@ -1204,7 +1201,7 @@ ruamel-yaml-clib==0.2.6 # via # -r requirements/edx/base.txt # ruamel-yaml -rules==3.2.1 +rules==3.3 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1360,7 +1357,7 @@ tox==3.24.5 # tox-battery tox-battery==0.6.1 # via -r requirements/edx/testing.in -tqdm==4.63.0 +tqdm==4.63.1 # via # -r requirements/edx/base.txt # nltk @@ -1373,6 +1370,7 @@ typing-extensions==4.1.1 # django-countries # pydantic # pylint + # redis unicodecsv==0.14.1 # via # -r requirements/edx/base.txt @@ -1384,7 +1382,7 @@ uritemplate==4.1.1 # -r requirements/edx/base.txt # coreapi # drf-yasg -urllib3==1.26.8 +urllib3==1.26.9 # via # -r requirements/edx/base.txt # elasticsearch @@ -1404,7 +1402,7 @@ vine==5.0.0 # amqp # celery # kombu -virtualenv==20.13.3 +virtualenv==20.13.4 # via tox voluptuous==0.12.2 # via diff --git a/scripts/xblock/requirements.txt b/scripts/xblock/requirements.txt index 77e8078094..f715aba5af 100644 --- a/scripts/xblock/requirements.txt +++ b/scripts/xblock/requirements.txt @@ -12,5 +12,5 @@ idna==3.3 # via requests requests==2.27.1 # via -r scripts/xblock/requirements.in -urllib3==1.26.8 +urllib3==1.26.9 # via requests