diff --git a/.github/actions/unit-tests/action.yml b/.github/actions/unit-tests/action.yml index 404ab2cf57..2a5429baff 100644 --- a/.github/actions/unit-tests/action.yml +++ b/.github/actions/unit-tests/action.yml @@ -27,7 +27,7 @@ runs: - name: save pytest warnings json file if: success() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: pytest-warnings-json path: | diff --git a/cms/envs/common.py b/cms/envs/common.py index eea4dae716..3b52ead574 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -2182,6 +2182,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'course_structure_cache': { @@ -2194,6 +2195,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'celery': { @@ -2206,6 +2208,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'mongo_metadata_inheritance': { @@ -2218,6 +2221,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'staticfiles': { @@ -2229,6 +2233,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'default': { @@ -2241,6 +2246,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'configuration': { @@ -2252,6 +2258,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'general': { @@ -2263,6 +2270,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, } diff --git a/cms/envs/devstack-experimental.yml b/cms/envs/devstack-experimental.yml index e33c024b0f..ad61260869 100644 --- a/cms/envs/devstack-experimental.yml +++ b/cms/envs/devstack-experimental.yml @@ -57,6 +57,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: celery LOCATION: @@ -68,6 +69,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: 78f87108afce LOCATION: @@ -78,6 +80,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: course_structure LOCATION: @@ -89,6 +92,7 @@ CACHES: ignore_exc: true no_delay: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: default LOCATION: @@ -100,6 +104,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: general LOCATION: @@ -110,6 +115,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: mongo_metadata_inheritance LOCATION: @@ -121,6 +127,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: 78f87108afce_general LOCATION: diff --git a/common/static/data/geoip/GeoLite2-Country.mmdb b/common/static/data/geoip/GeoLite2-Country.mmdb index d5a6c3e7b7..062db83db3 100644 Binary files a/common/static/data/geoip/GeoLite2-Country.mmdb and b/common/static/data/geoip/GeoLite2-Country.mmdb differ diff --git a/docs/concepts/testing/testing.rst b/docs/concepts/testing/testing.rst index 1297b0ee3b..9d448afd5b 100644 --- a/docs/concepts/testing/testing.rst +++ b/docs/concepts/testing/testing.rst @@ -114,10 +114,16 @@ For example, this command runs a single python unit test file:: pytest xmodule/tests/test_stringify.py Note - -edx-platorm has multiple services (lms, cms) in it. The environment for each service is different enough that we run some tests in both environments in jenkins. To make sure tests will pass in each of these environments (especially for tests in "common" directory), you will need to test in each seperately. Add --rootdir flag at end of your pytest call and specify the env you are testing in:: +edx-platorm has multiple services (lms, cms) in it. The environment for each service is different enough that we run some tests in both environments in Github Actions. +To test in each of these environments (especially for tests in "common" and "xmodule" directories), you will need to test in each seperately. +To specify that the tests are run with the relevant service as root, Add --rootdir flag at end of your pytest call and specify the env to test in:: pytest test --rootdir +Or, if you need django settings from a particular enviroment, add --ds flag to the end of your pytest call and specify the django settings object:: + + pytest test --ds= + Various tools like ddt create tests with very complex names, rather than figuring out the name yourself, you can: 1. Select tests to run based on their name, provide an expression to the `pytest -k option`_ which performs a substring match on test names:: diff --git a/docs/decisions/0019-oep-58-atlas-translations-design.rst b/docs/decisions/0019-oep-58-atlas-translations-design.rst new file mode 100644 index 0000000000..1b7f573e23 --- /dev/null +++ b/docs/decisions/0019-oep-58-atlas-translations-design.rst @@ -0,0 +1,362 @@ +Design for Refactoring Translations ``pull`` to use Atlas +########################################################## + +Status +====== + +Accepted + +Context +======= + +OEP-58 Translation Management overview +-------------------------------------- + +The `Translation Management update OEP-58`_ proposal has been merged with +the following changes to the way translations are managed in the Open edX platform: + +- Move Translation Files to the `openedx-translations repo`_ +- Add the `Transifex GitHub App `_ + to openedx Organization +- Connect the `openedx-translations repo`_ to the + `openedx-translations Transifex project`_ +- Copy `Transifex Translation Memory`_ into from the both of the + `edx-platform Transifex project`_ and the `xblocks Transifex project`_ into + the new `openedx-translations Transifex project`_ +- Utilize `openedx-atlas`_ to pull translations for development/deployment. + +If you're new to the `OEP-58`_ proposal, please +review the `OEP-58 Specifications`_ in addition to the +Key Metrics and Expected Results section in the +`Approach Memo and Technical Discovery - Translations Infrastructure Implementation`_ +document before continuing. + +Pre-OEP-58 Architecture/Implementation for XBlocks and Plugins +-------------------------------------------------------------- + +Before `OEP-58`_, Open edX XBlocks and Open edX plugins had the following: + +- Translations live in the GitHub repository. +- Translations are packaged with the rest of the code when published to pypi + +Pros: + +- Translations are always available after installation. + +Cons: + +- This can mean a complex integration with Transifex +- This can mean a lengthy manual PR review process up to a month such as in + the following example: + `Added French (Canada) and Japanese - xblock-drag-and-drop-v2 #220`_ + +XBlockI18nService +----------------- + +The `XBlockI18nService`_ loads translations for installed XBlocks via its +``__init__`` method. XBlock translations are only used during the +during the execution of the XBlock. + +The `XBlockI18nService implementation pull request`_ (2016) introduced +support for XBlock translations in ``edx-platform`` and has the full +context of the implementation. + +.. _js-translations: + +JavaScript Translations for XBocks +---------------------------------- + +As of September 2023, there is no centralized method to bundle JavaScript +translations in XBlocks. Non-XBlock plugins lack JavaScript translation +support altogether. + +The de-facto standard method for bundling JavaScript translations in XBlocks +is to use ``web_fragment`` and load the translations as part of the XBlock +frontend static files on every XBlock load. + +The LTI Consumer XBlock embeds the translations in its ``web_fragment`` via +the `LtiConsumerXBlock._get_statici18n_js_url`_ and +`LtiConsumerXBlock.student_view`_ methods. + +In order to separate the XBlock translations from the platform, it's isolated +in a separate ``gettext`` namespace. For example, the Drag and Drop XBlock +namespace is ``DragAndDropI18N`` which is hardcoded in multiple places such +as: + +- `XBlock Makefile compile_translations rule`_ +- `XBlock compiled JavaScript text.js translations`_ +- `XBlock main JavaScript file`_ + +`OEP-58`_ does not change this structure, it just makes the necessary changes +to pull translations from the `openedx-translations repo`_ via ``atlas`` +instead of having them live in the XBlock repository itself. + +Decisions +========= + +Proposed Design for edX Platform ``conf/locale`` translations +------------------------------------------------------------- + +We're going to use ``atlas`` in ``make pull_translations`` like we do in +`course-discovery atlas integration`_ and +`frontend-app-learning atlas integration`_. + +Proposed Design for XBlocks and Plugins +--------------------------------------- + +Instead of storing translation files for each XBlock and Plugin in their +respective repositories, +we will use `openedx-atlas`_ to pull them from the +`openedx-translations repo`_. + + +New ``atlas_pull_plugin_translations`` command +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Introduce new Django commands to the ``edx-platform``: + +- ``manage.py lms atlas_pull_plugin_translations --list``: List all XBlocks and + Plugins installed in the ``edx-platform`` virtual environment. This will + list the Python *module names* (as opposed to git repository names) of the + installed XBlocks and Plugins e.g.:: + + $ manage.py lms atlas_pull_plugin_translations --list + drag_and_drop_v2 + done + eox_tenant + + This list doesn't include plugins that are bundled within the + ``edx-platform`` repository itself. Translations for bundled plugins + are included in the ``edx-platform`` translation files. + +- ``manage.py lms atlas_pull_plugin_translations``: This command + will pull translations for installed XBlocks and Plugins by module name:: + + $ atlas pull --expand-glob \ + 'translations/*/drag_and_drop_v2/conf/locale:conf/plugins-locale/drag_and_drop_v2' \ + 'translations/*/done/conf/locale:conf/plugins-locale/done' \ + 'translations/*/edx_proctoring/conf/locale:conf/plugins-locale/edx_proctoring' + + Resulting in the following file tree:: + + $ tree conf/plugins-locale/ + conf/plugins-locale/ + ├── done + │ ├── ar + │ │ └── LC_MESSAGES + │ │ └── django.po + │ ├── de + │ │ └── LC_MESSAGES + │ │ └── django.po + │ ├── en + │ │ └── LC_MESSAGES + │ │ └── django.po + │ └── fr_CA + │ └── LC_MESSAGES + │ └── django.po + ├── drag_and_drop_v2 + │ ├── ar + │ │ └── LC_MESSAGES + │ │ └── django.po + │ ├── en + │ │ └── LC_MESSAGES + │ │ └── django.po + │ └── fr_CA + │ └── LC_MESSAGES + │ └── django.po + └── edx_proctoring + ├── ar + │ └── LC_MESSAGES + │ └── djangojs.po + ├── de + │ └── LC_MESSAGES + │ └── djangojs.po + ├── en + │ └── LC_MESSAGES + │ ├── djangojs.po + │ └── django.po + └── fr_CA + └── LC_MESSAGES + ├── djangojs.po + └── django.po + + +BlockI18nService support for ``atlas`` Python translations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``get_python_locale_directory`` will support two modes: + +#. If translations for the XBlock/plugin has been pulled by ``atlas`` + from the `openedx-translations repo`_, it will be used. For example, if the + ``edx-platform/conf/plugins-locale/drag_and_drop_v2/ar/LC_MESSAGES/django.po`` + path exists, it will be used for the Drag and Drop XBlock. + +#. Otherwise, the bundled translation files in the XBlock packages will be + used. The fallback path for the Drag and Drop XBlock will be + ``lib/python3.8/site-packages/drag_and_drop_v2/translations/ar/LC_MESSAGES/text.po``. + +This fallback is used to maintain backwards compatibility with existing +XBlocks that may or may not be included in the `openedx-translations repo`_. +Third-party XBlocks that are not included in the +`xblocks Transifex project`_, such as the `Lime Survey XBlock`_, +will benefit from this backwards compatibility. + +New ``compile_plugin_js_translations`` command +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An ``XBlock.i18n_js_namespace`` property will be added for +the ``compile_plugin_js_translations`` to generate JavaScript translations +in a centrally managed manner for installed XBlocks. + +A ``compile_plugin_js_translations`` command will loop over XBlock +modules that has the ``i18n_js_namespace`` +property set and compile the JavaScript translations via the `compilejsi18n`_ +command. + +For example if the Drag and Drop XBlock has +``i18n_js_namespace = 'DragAndDropI18N'``, the +``compile_plugin_js_translations`` command will execute the following +commands:: + + i18n_tool generate -v # Generate the .mo files + python manage.py compilejsi18n --namespace DragAndDropI18N --output conf/plugins-locale/drag_and_drop_v2/js/ + + +XBlockI18nService support for ``atlas`` JavaScript translations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A ``get_javascript_locale_path`` method will be added to the +``XBlockI18nService`` to provide XBlocks the +appropriate path to ``django.js`` translation files. This method +will allow XBlocks to utilize legacy packaged translations +or ``atlas``. + +A ``i18n_js_namespace`` property will be added +to generate JavaScript translations in a centrally managed manner for all +XBlocks as described in the :ref:`js-translations` section. + +For example, the `Drag and Drop XBlock get_static_i18n_js_url`_ will need to +be updated to support the new ``XBlockI18nService`` +``get_javascript_locale_path`` method and the namespace. + +.. code:: diff + + class DragAndDropBlock(XBlock): + + + i18n_js_namespace = 'DragAndDropI18N' + + @staticmethod + def _get_statici18n_js_url(): + """ + Returns the Javascript translation file for the currently selected language, if any found by + `pkg_resources` + """ + lang_code = translation.get_language() + if not lang_code: + return None + + + # TODO: Make this the default once OEP-58 is implemented. + + if hasattr(self.i18n_service, 'get_javascript_locale_path'): + + atlas_locale_path = self.i18n_service.get_javascript_locale_path() + + if atlas_locale_path: + + return atlas_locale_path + + text_js = 'public/js/translations/{lang_code}/text.js' + country_code = lang_code.split('-')[0] + for code in (translation.to_locale(lang_code), lang_code, country_code): + if pkg_resources.resource_exists(loader.module_name, text_js.format(lang_code=code)): + return text_js.format(lang_code=code) + return None + + +Dismissed Proposals +=================== + +XBlocks and plugins have their own "atlas pull" command +------------------------------------------------------- + +This dismissed proposal intends to have each XBlock and Plugin have their +own ``make pull_translations`` and be responsible for managing pulling their +own translations from the `openedx-translations repo`_. + +This proposal has been dismissed because it would require substantial work +to get into the details for the ``lib/python3.8/site-packages/`` directory +and ensure that the ``make pull_translations`` command won't corrupt the +virtual environment. + +This is a non-trivial task and appears to add more complexity than necessary +due to the fact that XBlocks and plugins won't be used outside the +context of ``edx-platform``. + + +Goals +===== +#. Use ``atlas pull`` for the ``edx-platform`` repo. +#. Use ``atlas pull`` for the XBlocks and Plugins. +#. Allow Tutor and other advanced uses to craft their own ``atlas pull`` + commands by making the the plugins list available via Django commands. +#. Allow ``atlas pull`` to use the Python module names instead of the + repository name of XBlocks and Plugins which is supported via the + `atlas pull --expand-glob`_ option. + +.. _non-goals: + +Non-Goals +========= + +The following are non-goals for this proposal, although some are going to +be tackled in the future as part of the +`Translation Management update OEP-58`_ proposal. + +#. Provide a fool-proof method for managing named-release translations. + This will be a separate discussion. +#. Discuss the merge/segment strategy of the ``edx-platform``. This is being + discussed in the + `decision no. 0018 `_. +#. Design a new XBlock frontend architecture. Instead this proposal works + with the existing architecture. +#. Provide a new translation method for theme translations. This will be + tackled later on. +#. Provide a new translation method for non-XBlock plugins such as + ``edx-val``. This will be tackled later on as part of the `OEP-58`_ + proposal. + +.. _OEP-58 Specifications: https://open-edx-proposals.readthedocs.io/en/latest/architectural-decisions/oep-0058-arch-translations-management.html#specification +.. _Translation Management update OEP-58: https://open-edx-proposals.readthedocs.io/en/latest/architectural-decisions/oep-0058-arch-translations-management.html#specification +.. _OEP-58: https://open-edx-proposals.readthedocs.io/en/latest/architectural-decisions/oep-0058-arch-translations-management.html#specification +.. _openedx-atlas: https://github.com/openedx/openedx-atlas +.. _openedx-translations repo: https://github.com/openedx/openedx-translations +.. _extract-translation-source-files.yml: https://github.com/openedx/openedx-translations/blob/2566e0c9a30d033e5dd8d05d4c12601c8e37b4ef/.github/workflows/extract-translation-source-files.yml#L36-L43 +.. _openedx-translations Transifex project: https://app.transifex.com/open-edx/openedx-translations/dashboard/ + +.. _Approach Memo and Technical Discovery - Translations Infrastructure Implementation: https://docs.google.com/document/d/11dFBCnbdHiCEdZp3pZeHdeH8m7Glla-XbIin7cnIOzU/edit +.. _Added French (Canada) and Japanese - xblock-drag-and-drop-v2 #220: https://github.com/openedx/xblock-drag-and-drop-v2/pull/220 +.. _XBlockI18nService: https://github.com/openedx/edx-platform/blob/6e28ba329e0a5354d7264ea834861bf0cae4ceb3/xmodule/modulestore/django.py#L359-L395 +.. _XBlockI18nService implementation pull request: https://github.com/openedx/edx-platform/pull/11575/files#diff-0bbcc6c13d9bfc9d88fbe2fdf4fd97f6066a7a0f0bfffb82bc942378b7cf33e0R248 + +.. _course-discovery atlas integration: https://github.com/openedx/course-discovery/pull/4037 +.. _frontend-app-learning atlas integration: https://github.com/openedx/frontend-app-learning/pull/1093 +.. _edx-platform pull_translations: https://github.com/openedx/edx-platform/blob/0137881b8199701b2af7d07c9a01200e358e3d86/Makefile#L55-L64 + +.. _drag-and-drop-v2 xblock: https://github.com/openedx/xblock-drag-and-drop-v2/ +.. _LTI Consumer XBlock: https://github.com/openedx/xblock-lti-consumer/ +.. _edx-val: https://github.com/openedx/edx-val + +.. _LtiConsumerXBlock._get_statici18n_js_url: https://github.com/openedx/xblock-lti-consumer/blob/7a142310a78ac393286c1e9e77c535ea520ab90b/lti_consumer/lti_xblock.py#L663-L677 +.. _LtiConsumerXBlock.student_view: https://github.com/openedx/xblock-lti-consumer/blob/7a142310a78ac393286c1e9e77c535ea520ab90b/lti_consumer/lti_xblock.py#L1215C24-L1217 +.. _Drag and Drop XBlock get_static_i18n_js_url: https://github.com/openedx/xblock-drag-and-drop-v2/blob/66e8d3517fe8c0db55c1a3907ff253c2a4562a7e/drag_and_drop_v2/drag_and_drop_v2.py#L318-L332 + +.. _XBlock compiled JavaScript text.js translations: https://github.com/openedx/xblock-drag-and-drop-v2/blob/b8ab1ecd9168ab1dba21f994ee4bfedb6a57d11f/drag_and_drop_v2/public/js/translations/tr/text.js#L3 +.. _XBlock Makefile compile_translations rule: https://github.com/openedx/xblock-drag-and-drop-v2/blob/66e8d3517fe8c0db55c1a3907ff253c2a4562a7e/Makefile#L41 +.. _XBlock main JavaScript file: https://github.com/openedx/xblock-drag-and-drop-v2/blob/b8ab1ecd9168ab1dba21f994ee4bfedb6a57d11f/drag_and_drop_v2/public/js/drag_and_drop.js#L6 + + +.. _translations/xblock-drag-and-drop-v2 directory: https://github.com/openedx/openedx-translations/tree/8a01424fd8f42e9e76aed34e235c82ab654cdfc5/translations/xblock-drag-and-drop-v2 +.. _atlas pull --expand-glob: https://github.com/openedx/openedx-atlas/blob/main/docs/decisions/0001-support-glob-pattern.rst + +.. _compilejsi18n: https://django-statici18n.readthedocs.io/en/latest/commands.html#compilejsi18n +.. _Transifex Translation Memory: https://help.transifex.com/en/articles/6224636-introduction-to-translation-memory +.. _edx-platform Transifex project: https://www.transifex.com/open-edx/edx-platform/ +.. _xblocks Transifex project: https://www.transifex.com/open-edx/xblocks/ + +.. _Lime Survey XBlock: https://github.com/eduNEXT/xblock-limesurvey diff --git a/lms/djangoapps/discussion/rest_api/discussions_notifications.py b/lms/djangoapps/discussion/rest_api/discussions_notifications.py index f3a5e3d61d..7913ce429f 100644 --- a/lms/djangoapps/discussion/rest_api/discussions_notifications.py +++ b/lms/djangoapps/discussion/rest_api/discussions_notifications.py @@ -52,6 +52,7 @@ class DiscussionNotificationSender: "replier_name": self.creator.username, "post_title": self.thread.title, "course_name": self.course.display_name, + "sender_id": self.creator.id, **extra_context, }, notification_type=notification_type, diff --git a/lms/djangoapps/discussion/rest_api/tests/test_tasks.py b/lms/djangoapps/discussion/rest_api/tests/test_tasks.py index 896e237f5f..20cef7a7c0 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_tasks.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_tasks.py @@ -307,6 +307,7 @@ class TestSendResponseNotifications(DiscussionAPIViewTestMixin, ModuleStoreTestC 'replier_name': self.user_2.username, 'post_title': 'test thread', 'course_name': self.course.display_name, + 'sender_id': self.user_2.id } self.assertDictEqual(args.context, expected_context) self.assertEqual( @@ -344,6 +345,7 @@ class TestSendResponseNotifications(DiscussionAPIViewTestMixin, ModuleStoreTestC 'post_title': self.thread.title, 'author_name': 'dummy\'s', 'course_name': self.course.display_name, + 'sender_id': self.user_3.id } self.assertDictEqual(args_comment.context, expected_context) self.assertEqual( @@ -359,6 +361,7 @@ class TestSendResponseNotifications(DiscussionAPIViewTestMixin, ModuleStoreTestC 'replier_name': self.user_3.username, 'post_title': self.thread.title, 'course_name': self.course.display_name, + 'sender_id': self.user_3.id } self.assertDictEqual(args_comment_on_response.context, expected_context) self.assertEqual( @@ -404,6 +407,7 @@ class TestSendResponseNotifications(DiscussionAPIViewTestMixin, ModuleStoreTestC 'post_title': self.thread.title, 'author_name': 'your', 'course_name': self.course.display_name, + 'sender_id': self.user_3.id, } self.assertDictEqual(args_comment.context, expected_context) self.assertEqual( @@ -440,6 +444,7 @@ class TestSendResponseNotifications(DiscussionAPIViewTestMixin, ModuleStoreTestC 'replier_name': self.user_2.username, 'post_title': 'test thread', 'course_name': self.course.display_name, + 'sender_id': self.user_2.id, } if parent_id: expected_context['author_name'] = 'dummy' diff --git a/lms/envs/common.py b/lms/envs/common.py index feebd335f4..b740af5000 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1127,6 +1127,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'course_structure_cache': { @@ -1139,6 +1140,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'celery': { @@ -1151,6 +1153,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'mongo_metadata_inheritance': { @@ -1163,6 +1166,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'staticfiles': { @@ -1174,6 +1178,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'default': { @@ -1186,6 +1191,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'configuration': { @@ -1197,6 +1203,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, 'general': { @@ -1208,6 +1215,7 @@ CACHES = { 'no_delay': True, 'ignore_exc': True, 'use_pooling': True, + 'connect_timeout': 0.5 } }, } diff --git a/lms/envs/devstack-experimental.yml b/lms/envs/devstack-experimental.yml index 50958af4d6..6d24b63cf9 100644 --- a/lms/envs/devstack-experimental.yml +++ b/lms/envs/devstack-experimental.yml @@ -76,6 +76,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: celery LOCATION: @@ -87,6 +88,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: 78f87108afce LOCATION: @@ -97,6 +99,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: course_structure LOCATION: @@ -108,6 +111,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: default LOCATION: @@ -119,6 +123,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: general LOCATION: @@ -129,6 +134,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: mongo_metadata_inheritance LOCATION: @@ -140,6 +146,7 @@ CACHES: no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 KEY_FUNCTION: common.djangoapps.util.memcache.safe_key KEY_PREFIX: 78f87108afce_general LOCATION: diff --git a/openedx/core/djangoapps/notifications/events.py b/openedx/core/djangoapps/notifications/events.py index 61db83d0e7..e76149f1c4 100644 --- a/openedx/core/djangoapps/notifications/events.py +++ b/openedx/core/djangoapps/notifications/events.py @@ -62,7 +62,8 @@ def notification_preferences_viewed_event(request, course_id): ) -def notification_generated_event(user_ids, app_name, notification_type, course_key, content_url, content): +def notification_generated_event(user_ids, app_name, notification_type, course_key, + content_url, content, sender_id=None): """ Emit an event when a notification is generated. """ @@ -78,6 +79,7 @@ def notification_generated_event(user_ids, app_name, notification_type, course_k 'notification_app': app_name, 'content_url': content_url, 'notification_content': content, + 'sender_id': sender_id, } with tracker.get_tracker().context(NOTIFICATION_GENERATED, context): tracker.emit( diff --git a/openedx/core/djangoapps/notifications/filters.py b/openedx/core/djangoapps/notifications/filters.py index 9176faa6ee..ea71f276d2 100644 --- a/openedx/core/djangoapps/notifications/filters.py +++ b/openedx/core/djangoapps/notifications/filters.py @@ -30,7 +30,7 @@ class NotificationFilter: course_time_limit = CourseDurationLimitConfig.current(course_key=course.id) if not verified_mode: logger.info( - "Course %s does not have a verified mode, so no users will be filtered out", + "NotificationFilter: Course %s does not have a verified mode, so no users will be filtered out", course.id, ) return user_ids @@ -41,10 +41,13 @@ class NotificationFilter: ) if course_time_limit.enabled_for_course(course.id): enrollments = enrollments.filter(created__gte=course_time_limit.enabled_as_of) - + logger.info("NotificationFilter: Number of audit enrollments for course %s: %s", course.id, enrollments.count()) for enrollment in enrollments: content_availability_date = max(enrollment.created, course.start) expiration_date = content_availability_date + access_duration + logger.info("NotificationFilter: content_availability_date: %s and access_duration: %s", + content_availability_date, access_duration + ) if expiration_date and timezone.now() > expiration_date: logger.info("User %s has expired audit access to course %s", enrollment.user_id, course.id) user_ids.remove(enrollment.user_id) @@ -59,7 +62,7 @@ class NotificationFilter: course = modulestore().get_course(course_key) for filter_name in applicable_filters: logger.info( - "Applying filter %s for notification type %s", + "NotificationFilter: Applying filter %s for notification type %s", filter_name, notification_type, ) diff --git a/openedx/core/djangoapps/notifications/tasks.py b/openedx/core/djangoapps/notifications/tasks.py index 4d9a240095..f38c6cebc0 100644 --- a/openedx/core/djangoapps/notifications/tasks.py +++ b/openedx/core/djangoapps/notifications/tasks.py @@ -92,12 +92,15 @@ def send_notifications(user_ids, course_key: str, app_name, notification_type, c course_key = CourseKey.from_string(course_key) if not ENABLE_NOTIFICATIONS.is_enabled(course_key): return + user_ids = list(set(user_ids)) batch_size = settings.NOTIFICATION_CREATION_BATCH_SIZE notifications_generated = False notification_content = '' - default_web_config = get_default_values_of_preference(app_name, notification_type).get('web', True) + sender_id = context.pop('sender_id', None) + default_web_config = get_default_values_of_preference(app_name, notification_type).get('web', False) + generated_notification_audience = [] for batch_user_ids in get_list_in_batches(user_ids, batch_size): # check if what is preferences of user and make decision to send notification or not preferences = CourseNotificationPreference.objects.filter( @@ -110,7 +113,7 @@ def send_notifications(user_ids, course_key: str, app_name, notification_type, c preferences = create_notification_pref_if_not_exists(batch_user_ids, preferences, course_key) if not preferences: - return + continue for preference in preferences: preference = update_user_preference(preference, preference.user_id, course_key) @@ -137,16 +140,19 @@ def send_notifications(user_ids, course_key: str, app_name, notification_type, c course_id=course_key, ) ) + generated_notification_audience.append(user_id) + # send notification to users but use bulk_create notification_objects = Notification.objects.bulk_create(notifications) if notification_objects and not notifications_generated: notifications_generated = True notification_content = notification_objects[0].content - if notifications_generated: - notification_generated_event( - batch_user_ids, app_name, notification_type, course_key, content_url, notification_content, - ) + if notifications_generated: + notification_generated_event( + generated_notification_audience, app_name, notification_type, course_key, content_url, + notification_content, sender_id=sender_id + ) def update_user_preference(preference: CourseNotificationPreference, user_id, course_id): diff --git a/openedx/core/djangoapps/notifications/tests/test_tasks.py b/openedx/core/djangoapps/notifications/tests/test_tasks.py index 339ce0e03f..98b90e7b4b 100644 --- a/openedx/core/djangoapps/notifications/tests/test_tasks.py +++ b/openedx/core/djangoapps/notifications/tests/test_tasks.py @@ -124,7 +124,12 @@ class SendNotificationsTest(ModuleStoreTestCase): content_url = 'https://example.com/' # Call the `send_notifications` function. - send_notifications([self.user.id], str(self.course_1.id), app_name, notification_type, context, content_url) + with patch('openedx.core.djangoapps.notifications.tasks.notification_generated_event') as event_mock: + send_notifications([self.user.id], str(self.course_1.id), app_name, notification_type, context, content_url) + assert event_mock.called + assert event_mock.call_args[0][0] == [self.user.id] + assert event_mock.call_args[0][1] == app_name + assert event_mock.call_args[0][2] == notification_type # Assert that `Notification` objects have been created for the users. notification = Notification.objects.filter(user_id=self.user.id).first() diff --git a/requirements/constraints.txt b/requirements/constraints.txt index e2ff4c0045..254655b4c2 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -26,7 +26,7 @@ django-storages==1.14 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==4.6.7 +edx-enterprise==4.6.9 # django-oauth-toolkit version >=2.0.0 has breaking changes. More details # mentioned on this issue https://github.com/openedx/edx-platform/issues/32884 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 4674aa2105..4c5aaa9b8f 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -482,7 +482,7 @@ edx-drf-extensions==8.12.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.6.7 +edx-enterprise==4.6.9 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 8e96616014..bfdfd6c64a 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -752,7 +752,7 @@ edx-drf-extensions==8.12.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.6.7 +edx-enterprise==4.6.9 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index f2491bceaf..cf936c8827 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -558,7 +558,7 @@ edx-drf-extensions==8.12.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.6.7 +edx-enterprise==4.6.9 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index e45735a63d..b0a27d8e6e 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -583,7 +583,7 @@ edx-drf-extensions==8.12.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.6.7 +edx-enterprise==4.6.9 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt