feat: update openedx-core: new catalog models + backfill migration (#38023)

* feat: use new version of openedx-core
* feat: Use openedx_catalog app, backfill it with all known courses
* feat: properly set "created" timestamp on course runs during backfill
* fix: better normalization of language codes
* feat: keep courses in sync with CourseRun/CatalogCourse
* feat: delete CourseRun/CatalogCourse when deleting a course
* refactor: course_id -> course_key, run -> run_code, display_name -> title
* fix: don't use SplitModulestoreCourseIndex for getting list of all courses

cms/envs/common.py

@@ -907,6 +907,9 @@ INSTALLED_APPS = [
 
     'openedx_events',
 
+    # Core models to represent courses
+    "openedx_catalog",
+
     # Core apps that power libraries
     "openedx_content",
     *openedx_content_backcompat_apps_to_install(),

lms/envs/common.py

@@ -2020,6 +2020,9 @@ INSTALLED_APPS = [
 
     'openedx_events',
 
+    # Core models to represent courses
+    "openedx_catalog",
+
     # Core apps that power libraries
     "openedx_content",
     *openedx_content_backcompat_apps_to_install(),

openedx/core/djangoapps/content/course_overviews/migrations/0030_backfill_new_catalog_courseruns.py

@@ -0,0 +1,138 @@
+"""
+Data migration to populate the new CourseRun and CatalogCourse models.
+"""
+
+# Generated by Django 5.2.11 on 2026-02-13 21:47
+import logging
+
+from django.conf import settings
+from django.db import migrations
+from organizations.api import ensure_organization, exceptions as org_exceptions
+
+log = logging.getLogger(__name__)
+
+# https://github.com/openedx/openedx-platform/issues/38036
+NORMALIZE_LANGUAGE_CODES = {
+    "zh-hans": "zh-cn",
+    "zh-hant": "zh-hk",
+    "ca@valencia": "ca-es-valencia",
+}
+
+
+def backfill_openedx_catalog(apps, schema_editor) -> None:
+    """
+    Populate the new CourseRun and CatalogCourse models.
+    """
+    CourseOverview = apps.get_model("course_overviews", "CourseOverview")
+    CatalogCourse = apps.get_model("openedx_catalog", "CatalogCourse")
+    CourseRun = apps.get_model("openedx_catalog", "CourseRun")
+
+    created_catalog_course_ids: set[int] = set()
+    all_course_runs = CourseOverview.objects.order_by("-created")
+    for course_overview in all_course_runs:
+        course_key = course_overview.id
+        org_code: str = course_key.org
+        course_code: str = course_key.course
+        run_code: str = course_key.run
+
+        # Ensure that the Organization exists.
+        try:
+            org_data = ensure_organization(org_code)
+        except org_exceptions.InvalidOrganizationException as exc:
+            # Note: IFF the org exists among the modulestore courses but not in the Organizations database table,
+            # and if auto-create is disabled (it's enabled by default), this will raise InvalidOrganizationException. It
+            # would be up to the operator to decide how they want to resolve that.
+            raise ValueError(
+                f'The organization short code "{org_code}" exists in modulestore ({course_key}) but '
+                "not the Organizations table, and auto-creating organizations is disabled. You can resolve this by "
+                "creating the Organization manually (e.g. from the Django admin) or turning on auto-creation. "
+                "You can set active=False to prevent this Organization from being used other than for historical data. "
+            ) from exc
+        if org_data["short_name"] != org_code:
+            # On most installations, the 'short_name' database column is case insensitive (unfortunately)
+            log.warning(
+                'The course with ID "%s" does not match its Organization.short_name "%s"',
+                course_key,
+                org_data["short_name"],
+            )
+
+        # Fetch the CourseOverview if it exists
+        try:
+            course_overview = CourseOverview.objects.get(id=course_key)
+        except CourseOverview.DoesNotExist:
+            course_overview = None  # Course exists in modulestore but details aren't cached into CourseOverview yet
+        title: str = (course_overview.display_name if course_overview else None) or course_code
+
+        # Determine the course language.
+        # Note that in Studio, the options for course language generally came from the ALL_LANGUAGES setting, which is
+        # mostly two-letter language codes with no locale, except it uses "zh_HANS" for Mandarin and "zh_HANT" for
+        # Cantonese. We normalize those to "zh-cn" and "zh-hk" for consistency with our platform UI languages /
+        # Transifex, but you can still access the "old" version using the CatalogCourse.language_short
+        # getter/setter for backwards compatbility. See https://github.com/openedx/openedx-platform/issues/38036
+        language = settings.LANGUAGE_CODE
+        if course_overview and course_overview.language:
+            language = course_overview.language.lower()
+            language = language.replace("_", "-")  # Ensure we use hyphens for consistency (`en-us` not `en_us`)
+            # Normalize this language code. The previous/non-normalized code will still be available via the
+            # "language_short" property for backwards compatibility.
+            language = NORMALIZE_LANGUAGE_CODES.get(language, language)
+            if len(language) > 2 and language[2] != "-":
+                # This seems like an invalid value; revert to the default:
+                log.warning(
+                    'The course with ID "%s" has invalid language "%s" - using default language "%s" instead.',
+                    course_key,
+                    language,
+                    settings.LANGUAGE_CODE,
+                )
+                language = settings.LANGUAGE_CODE
+
+        # Ensure that the CatalogCourse exists.
+        cc, cc_created = CatalogCourse.objects.get_or_create(
+            org_id=org_data["id"],
+            course_code=course_code,
+            defaults={
+                # The default title for the catalog course will be the same name as the newest run, since we iterate
+                # over "all_course_runs" in "-created" order.
+                "title": title,
+                "language": language,
+            },
+        )
+        if cc_created:
+            created_catalog_course_ids.add(cc.pk)
+
+        if cc.course_code != course_code:
+            raise ValueError(
+                f"The course {course_key} exists in modulestore with a different capitalization of its "
+                f'course code compared to other instances of the same run ("{course_code}" vs "{cc.course_code}"). '
+                "This really should not happen. To fix it, delete the inconsistent course runs (!). "
+            )
+
+        # Create the CourseRun
+        new_run, run_created = CourseRun.objects.get_or_create(
+            catalog_course=cc,
+            run_code=run_code,
+            course_key=course_key,
+            defaults={"title": title},
+        )
+
+        # Correct the "created" timestamp. Since it has auto_now_add=True, we can't set its value except using update()
+        # The CourseOverview should have the "created" date unless it's missing or the course was created before
+        # the CourseOverview model existed. In any case, it should be good enough. Otherwise use the default (now).
+        if course_overview:
+            if course_overview.created < cc.created and cc.pk in created_catalog_course_ids:
+                # Use the 'created' date from the oldest course run that we process.
+                CatalogCourse.objects.filter(pk=cc.pk).update(created=course_overview.created)
+            if run_created:
+                CourseRun.objects.filter(pk=new_run.pk).update(created=course_overview.created)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("openedx_catalog", "0001_initial"),
+        ("course_overviews", "0029_alter_historicalcourseoverview_options"),
+        ("split_modulestore_django", "0003_alter_historicalsplitmodulestorecourseindex_options"),
+    ]
+
+    operations = [
+        migrations.RunPython(backfill_openedx_catalog, reverse_code=migrations.RunPython.noop),
+    ]

openedx/core/djangoapps/content/course_overviews/signals.py

@@ -2,7 +2,6 @@
 Signal handler for invalidating cached course overviews
 """
 
-
 import logging
 
 from django.db import transaction
@@ -10,6 +9,8 @@ from django.db.models.signals import post_save
 from django.dispatch import Signal
 from django.dispatch.dispatcher import receiver
 
+from openedx_catalog import api as catalog_api
+from openedx_catalog.models_api import CourseRun
 from openedx.core.djangoapps.signals.signals import COURSE_CERT_DATE_CHANGE
 from xmodule.data import CertificatesDisplayBehaviors
 from xmodule.modulestore.django import SignalHandler
@@ -33,6 +34,8 @@ def _listen_for_course_publish(sender, course_key, **kwargs):  # pylint: disable
     """
     Catches the signal that a course has been published in Studio and updates the corresponding CourseOverview cache
     entry.
+
+    Also sync course data to the openedx_catalog CourseRun model.
     """
     try:
         previous_course_overview = CourseOverview.objects.get(id=course_key)
@@ -41,6 +44,51 @@ def _listen_for_course_publish(sender, course_key, **kwargs):  # pylint: disable
     updated_course_overview = CourseOverview.load_from_module_store(course_key)
     _check_for_course_changes(previous_course_overview, updated_course_overview)
 
+    # Currently, SplitModulestoreCourseIndex is the ultimate source of truth for
+    # which courses exist. When a course is published, we sync that data to
+    # CourseOverview, and from CourseOverview to CourseRun.
+
+    # In the future, CourseRun will be the "source of truth" and each CourseRun
+    # may optionally point to content and get synced to CourseOverview.
+
+    # Ensure a CourseRun exists for this course
+    try:
+        course_run = catalog_api.get_course_run(course_key)
+    except CourseRun.DoesNotExist:
+        # Presumably this is a newly-created course. Create the CourseRun.
+        course_run = catalog_api.create_course_run_for_modulestore_course_with(
+            course_key=course_key,
+            title=updated_course_overview.display_name,
+            language_short=updated_course_overview.language,
+        )
+
+    # Keep the CourseRun up to date as the course is edited:
+    if updated_course_overview.display_name != course_run.title:
+        catalog_api.sync_course_run_details(course_key, title=updated_course_overview.display_name)
+        # If this course is the only run in the CatalogCourse, should we update the title of
+        # the CatalogCourse to match the run's new title? Currently the only way to edit the name of
+        # a CatalogCourse is via the Django admin. But it's also not used anywhere yet.
+
+    if (
+        updated_course_overview.language
+        and updated_course_overview.language != course_run.catalog_course.language_short
+    ):
+        if course_run.catalog_course.runs.count() == 1:
+            # This is the only run in this CatalogCourse. Update the language of the CatalogCourse
+            catalog_api.update_catalog_course(
+                course_run.catalog_course,
+                language_short=updated_course_overview.language,
+            )
+        else:
+            LOG.warning(
+                'Course run "%s" language "%s" does not match its catalog course language, "%s"',
+                str(course_key),
+                updated_course_overview.language,
+                course_run.catalog_course.language_short,
+            )
+
+    # In the future, this will also sync schedule and other metadata to the CourseRun's related models
+
 
 @receiver(SignalHandler.course_deleted)
 def _listen_for_course_delete(sender, course_key, **kwargs):  # pylint: disable=unused-argument
@@ -56,6 +104,16 @@ def _listen_for_course_delete(sender, course_key, **kwargs):  # pylint: disable=
         sender=None,
         courserun_key=courserun_key,
     )
+    # Delete the openedx_catalog CourseRun to keep it in sync:
+    try:
+        course_run_obj = catalog_api.get_course_run(course_key)
+    except CourseRun.DoesNotExist:
+        pass
+    else:
+        catalog_course = course_run_obj.catalog_course
+        catalog_api.delete_course_run(course_key)
+        if catalog_course.runs.count() == 0:
+            catalog_api.delete_catalog_course(catalog_course)
 
 
 @receiver(post_save, sender=CourseOverview)

openedx/core/djangoapps/content/course_overviews/tests/test_sync_with_openedx_catalog.py

@@ -0,0 +1,162 @@
+"""
+Test that changes to courses get synced into the new openedx_catalog models.
+"""
+
+import pytest
+
+from openedx_catalog import api as catalog_api
+from openedx_catalog.models_api import CatalogCourse, CourseRun
+
+from openedx.core.djangolib.testing.utils import skip_unless_cms
+from xmodule.modulestore import ModuleStoreEnum
+from xmodule.modulestore.tests.django_utils import (
+    TEST_DATA_ONLY_SPLIT_MODULESTORE_DRAFT_PREFERRED,
+    ModuleStoreTestCase,
+    ImmediateOnCommitMixin,
+)
+from xmodule.modulestore.tests.factories import CourseFactory
+
+
+@skip_unless_cms
+class CourseOverviewSyncTestCase(ImmediateOnCommitMixin, ModuleStoreTestCase):
+    """
+    Test that changes to courses get synced into the new openedx_catalog models.
+    """
+
+    MODULESTORE = TEST_DATA_ONLY_SPLIT_MODULESTORE_DRAFT_PREFERRED
+    ENABLED_SIGNALS = ["course_deleted", "course_published"]
+
+    def test_courserun_creation(self) -> None:
+        """
+        Tests that when a course is created, the `CourseRun` record gets created.
+
+        (Also the corresponding `CatalogCourse`.)
+        """
+        course = CourseFactory.create(display_name="Intro to Testing", emit_signals=True)
+        course_key = course.location.context_key
+
+        run = catalog_api.get_course_run(course_key)
+        assert run.title == "Intro to Testing"
+        assert run.course_key == course_key
+        assert run.catalog_course.course_code == course_key.course
+        assert run.catalog_course.org_code == course_key.org
+
+    def test_courserun_sync(self) -> None:
+        """
+        Tests that when a course is updated, the catalog records get updated.
+
+        Because the "language" of a course cannot be set in Studio before you
+        create the course, when a Catalog Course has only a single run, we need
+        to keep the language of the catalog course in sync with any changes to
+        the language field of the course run. (Because authors necessarily
+        create a new course with the default language then edit it to have the
+        correct language that they actually intended to use for that [catalog]
+        course.) This is in contrast with display_name (title), which can
+        actually be set before creating a course.
+        """
+        # Create a course
+        course = CourseFactory.create(display_name="Intro to Testing", emit_signals=True)
+        course_id = course.location.context_key
+        run = catalog_api.get_course_run(course_id)
+        assert run.title == "Intro to Testing"
+        assert run.catalog_course.language_short == "en"
+
+        # Update the course's title and language:
+        course.language = "es"
+        course.display_name = "Introducción a las pruebas"
+        self.store.update_item(course, ModuleStoreEnum.UserID.test)
+
+        # Check if the catalog data is updated:
+        run.refresh_from_db()
+        assert run.title == "Introducción a las pruebas"
+        assert run.catalog_course.language_short == "es"
+        # Note: for now we don't update the title of the catalog course after it has been created.
+        # We _could_ decide to sync the name from run -> catalog course if there is only one run.
+        assert run.catalog_course.title == "Intro to Testing"
+
+    def test_courserun_of_many_sync(self) -> None:
+        """
+        Tests that when a course is updated, the catalog records get updated,
+        but if there are several runs of the same course, the changes don't
+        propagate to the `CatalogCourse` and only affect the `CourseRun.
+        """
+        # This import causes problems at top level when tests run on the LMS shard
+        from cms.djangoapps.contentstore.views.course import rerun_course
+
+        # Create a course
+        course = CourseFactory.create(display_name="Intro to Testing", emit_signals=True)
+        course_id = course.location.context_key
+        run = catalog_api.get_course_run(course_id)
+        assert run.title == "Intro to Testing"
+        assert run.catalog_course.language_short == "en"
+
+        # re-run the course:
+        new_run_course_id = rerun_course(
+            self.user,
+            source_course_key=course_id,
+            org=course_id.org,
+            number=course_id.course,
+            run="newRUN",
+            fields={"display_name": "Intro to Testing TEMPORARY NAME"},
+            background=False,
+        )
+
+        # Update the re-run's title (display_name) and language:
+        new_course = self.store.get_course(new_run_course_id)
+        new_course.language = "es"
+        new_course.display_name = "Introducción a las pruebas"
+        self.store.update_item(new_course, self.user.id)
+
+        # Check if the catalog data is updated correctly.
+        # The original CourseRun object should be unchanged:
+        run.refresh_from_db()
+        assert run.title == "Intro to Testing"
+        assert run.catalog_course.language_short == "en"
+        # The new CourseRun object should be created:
+        new_run = catalog_api.get_course_run(new_run_course_id)
+        assert new_run.title == "Introducción a las pruebas"
+        # Changing the language of the second run doesn't affect the lanugage of the overall catalog course (since the
+        # first run is still in English)
+        assert new_run.catalog_course.language_short == "en"
+
+    def test_courserun_deletion(self) -> None:
+        """
+        Tests that when a course run is deleted, the corresponding CourseRun is
+        deleted, and when it's the last run, the CatalogCourse is deleted too.
+        """
+        # This import causes problems at top level when tests run on the LMS shard
+        from cms.djangoapps.contentstore.views.course import rerun_course
+
+        # Create a course with two runs:
+        course = CourseFactory.create(display_name="Intro to Testing", emit_signals=True)
+        course_id1 = course.location.context_key
+        run1 = catalog_api.get_course_run(course_id1)
+        # re-run the course:
+        course_id2 = rerun_course(
+            self.user,
+            source_course_key=course_id1,
+            org=course_id1.org,
+            number=course_id1.course,
+            run="run2",
+            fields={"display_name": "ItT run2"},
+            background=False,
+        )
+        run2 = catalog_api.get_course_run(course_id2)
+        catalog_course = run1.catalog_course
+        assert catalog_course == run2.catalog_course  # Same for run1 and run2
+
+        self.store.delete_course(course_id1, ModuleStoreEnum.UserID.test)
+        with pytest.raises(CourseRun.DoesNotExist):
+            run1.refresh_from_db()
+
+        # run2 should still exist:
+        run2.refresh_from_db()
+        assert run2.catalog_course.title == "Intro to Testing"  # The catalog course still exists and works
+
+        # delete run 2:
+        self.store.delete_course(course_id2, ModuleStoreEnum.UserID.test)
+        with pytest.raises(CourseRun.DoesNotExist):
+            run2.refresh_from_db()
+        # With no runs left, the CatalogCourse also gets auto-deleted:
+        with pytest.raises(CatalogCourse.DoesNotExist):
+            catalog_course.refresh_from_db()

pyproject.toml

@@ -233,7 +233,7 @@ multi_line_output = 3
 skip = ["envs", "migrations"]
 
 [tool.importlinter]
-root_packages = ["lms", "cms", "openedx", "openedx_content"]
+root_packages = ["lms", "cms", "openedx", "openedx_content", "openedx_catalog"]
 include_external_packages = true
 # Our custom contract which checks that we're only importing from 'api.py'
 # for participating packages.
@@ -356,11 +356,13 @@ isolated_apps = [
     "openedx.core.djangoapps.olx_rest_api",
     "openedx.core.djangoapps.xblock",
     "openedx.core.lib.xblock_serializer",
+    "openedx_catalog",
 ]
 allowed_modules = [
     # Only imports from api.py and data.py are allowed elsewhere in the code
     # See https://open-edx-proposals.readthedocs.io/en/latest/best-practices/oep-0049-django-app-patterns.html#api-py
     "api",
+    "models_api",
     "data",
     "tests",
 ]

requirements/constraints.txt

@@ -65,7 +65,7 @@ numpy<2.0.0
 # breaking changes which openedx-core devs want to roll out manually. New patch versions
 # are OK to accept automatically.
 # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269
-openedx-core<0.36
+openedx-core<0.37
 
 # Date: 2023-11-29
 # Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise.

requirements/edx/base.txt

@@ -515,7 +515,9 @@ edx-opaque-keys[django]==3.1.0
     #   ora2
     #   xblocks-contrib
 edx-organizations==7.3.0
-    # via -r requirements/edx/kernel.in
+    # via
+    #   -r requirements/edx/kernel.in
+    #   openedx-core
 edx-proctoring==5.2.0
     # via -r requirements/edx/kernel.in
 edx-rbac==2.1.0
@@ -826,7 +828,7 @@ openedx-calc==4.0.3
     # via
     #   -r requirements/edx/kernel.in
     #   xblocks-contrib
-openedx-core==0.35.0
+openedx-core==0.36.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/edx/kernel.in

requirements/edx/development.txt

@@ -810,6 +810,7 @@ edx-organizations==7.3.0
     # via
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
+    #   openedx-core
 edx-proctoring==5.2.0
     # via
     #   -r requirements/edx/doc.txt
@@ -1382,7 +1383,7 @@ openedx-calc==4.0.3
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
     #   xblocks-contrib
-openedx-core==0.35.0
+openedx-core==0.36.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/edx/doc.txt

requirements/edx/doc.txt

@@ -608,7 +608,9 @@ edx-opaque-keys[django]==3.1.0
     #   ora2
     #   xblocks-contrib
 edx-organizations==7.3.0
-    # via -r requirements/edx/base.txt
+    # via
+    #   -r requirements/edx/base.txt
+    #   openedx-core
 edx-proctoring==5.2.0
     # via -r requirements/edx/base.txt
 edx-rbac==2.1.0
@@ -1006,7 +1008,7 @@ openedx-calc==4.0.3
     # via
     #   -r requirements/edx/base.txt
     #   xblocks-contrib
-openedx-core==0.35.0
+openedx-core==0.36.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/edx/base.txt

requirements/edx/testing.txt

@@ -629,7 +629,9 @@ edx-opaque-keys[django]==3.1.0
     #   ora2
     #   xblocks-contrib
 edx-organizations==7.3.0
-    # via -r requirements/edx/base.txt
+    # via
+    #   -r requirements/edx/base.txt
+    #   openedx-core
 edx-proctoring==5.2.0
     # via -r requirements/edx/base.txt
 edx-rbac==2.1.0
@@ -1056,7 +1058,7 @@ openedx-calc==4.0.3
     # via
     #   -r requirements/edx/base.txt
     #   xblocks-contrib
-openedx-core==0.35.0
+openedx-core==0.36.0
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/edx/base.txt