Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: upstream downstream link model [FC-0076] (#36111)

Browse Source 
Adds models, tasks, signal handlers and api's for adding, updating and deleting upstream->downstream links in database.
This commit is contained in:
Navin Karkera
2025-02-11 17:15:50 +00:00
committed by GitHub
parent 1ced859b77
commit 2598084946
17 changed files with 940 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
cms/djangoapps/contentstore/admin.py
View File

@@ -13,6 +13,8 @@ from edx_django_utils.admin.mixins import ReadOnlyAdminMixin
from cms.djangoapps.contentstore.models import (
BackfillCourseTabsConfig,
CleanStaleCertificateAvailabilityDatesConfig,
LearningContextLinksStatus,
PublishableEntityLink,
VideoUploadConfig
)
from cms.djangoapps.contentstore.outlines_regenerate import CourseOutlineRegenerate
@@ -86,6 +88,71 @@ class CleanStaleCertificateAvailabilityDatesConfigAdmin(ConfigurationModelAdmin)
pass
@admin.register(PublishableEntityLink)
class PublishableEntityLinkAdmin(admin.ModelAdmin):
"""
PublishableEntityLink admin.
"""
fields = (
"uuid",
"upstream_block",
"upstream_usage_key",
"upstream_context_key",
"downstream_usage_key",
"downstream_context_key",
"version_synced",
"version_declined",
"created",
"updated",
)
readonly_fields = fields
list_display = [
"upstream_block",
"upstream_usage_key",
"downstream_usage_key",
"version_synced",
"updated",
]
search_fields = [
"upstream_usage_key",
"upstream_context_key",
"downstream_usage_key",
"downstream_context_key",
]
def has_add_permission(self, request):
return False
def has_change_permission(self, request, obj=None):
return False
@admin.register(LearningContextLinksStatus)
class LearningContextLinksStatusAdmin(admin.ModelAdmin):
"""
LearningContextLinksStatus admin.
"""
fields = (
"context_key",
"status",
"created",
"updated",
)
readonly_fields = ("created", "updated")
list_display = (
"context_key",
"status",
"created",
"updated",
)
def has_add_permission(self, request):
return False
def has_change_permission(self, request, obj=None):
return False
admin.site.register(BackfillCourseTabsConfig, ConfigurationModelAdmin)
admin.site.register(VideoUploadConfig, ConfigurationModelAdmin)
admin.site.register(CourseOutlineRegenerate, CourseOutlineRegenerateAdmin)

94
cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py Normal file
View File

@@ -0,0 +1,94 @@
"""
Management command to recreate upstream-dowstream links in PublishableEntityLink for course(s).
This command can be run for all the courses or for given list of courses.
"""
from __future__ import annotations
import logging
from datetime import datetime, timezone
from django.core.management.base import BaseCommand, CommandError
from django.utils.translation import gettext as _
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
from ...tasks import create_or_update_upstream_links
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Recreate links for course(s) in PublishableEntityLink table.
Examples:
# Recreate upstream links for two courses.
$ ./manage.py cms recreate_upstream_links --course course-v1:edX+DemoX.1+2014 \
--course course-v1:edX+DemoX.2+2015
# Force recreate upstream links for one or more courses including processed ones.
$ ./manage.py cms recreate_upstream_links --course course-v1:edX+DemoX.1+2014 \
--course course-v1:edX+DemoX.2+2015 --force
# Recreate upstream links for all courses.
$ ./manage.py cms recreate_upstream_links --all
# Force recreate links for all courses including completely processed ones.
$ ./manage.py cms recreate_upstream_links --all --force
# Delete all links and force recreate links for all courses
$ ./manage.py cms recreate_upstream_links --all --force --replace
"""
def add_arguments(self, parser):
parser.add_argument(
'--course',
metavar=_('COURSE_KEY'),
action='append',
help=_('Recreate links for xblocks under given course keys. For eg. course-v1:edX+DemoX.1+2014'),
default=[],
)
parser.add_argument(
'--all',
action='store_true',
help=_(
'Recreate links for xblocks under all courses. NOTE: this can take long time depending'
' on number of course and xblocks'
),
)
parser.add_argument(
'--force',
action='store_true',
help=_('Recreate links even for completely processed courses.'),
)
parser.add_argument(
'--replace',
action='store_true',
help=_('Delete all and create links for given course(s).'),
)
def handle(self, *args, **options):
"""
Handle command
"""
courses = options['course']
should_process_all = options['all']
force = options['force']
replace = options['replace']
time_now = datetime.now(tz=timezone.utc)
if not courses and not should_process_all:
raise CommandError('Either --course or --all argument should be provided.')
if should_process_all and courses:
raise CommandError('Only one of --course or --all argument should be provided.')
if should_process_all:
courses = CourseOverview.get_all_course_keys()
for course in courses:
log.info(f"Start processing upstream->dowstream links in course: {course}")
try:
CourseKey.from_string(str(course))
except InvalidKeyError:
log.error(f"Invalid course key: {course}, skipping..")
continue
create_or_update_upstream_links.delay(str(course), force=force, replace=replace, created=time_now)

93
cms/djangoapps/contentstore/migrations/0009_learningcontextlinksstatus_publishableentitylink.py Normal file
View File

@@ -0,0 +1,93 @@
# Generated by Django 4.2.18 on 2025-02-05 05:33
import uuid
import django.db.models.deletion
import opaque_keys.edx.django.models
import openedx_learning.lib.fields
import openedx_learning.lib.validators
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('oel_publishing', '0002_alter_learningpackage_key_and_more'),
('contentstore', '0008_cleanstalecertificateavailabilitydatesconfig'),
]
operations = [
migrations.CreateModel(
name='LearningContextLinksStatus',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
(
'context_key',
opaque_keys.edx.django.models.CourseKeyField(
help_text='Linking status for course context key', max_length=255, unique=True
),
),
(
'status',
models.CharField(
choices=[
('pending', 'Pending'),
('processing', 'Processing'),
('failed', 'Failed'),
('completed', 'Completed'),
],
help_text='Status of links in given learning context/course.',
max_length=20,
),
),
('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])),
('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])),
],
options={
'verbose_name': 'Learning Context Links status',
'verbose_name_plural': 'Learning Context Links status',
},
),
migrations.CreateModel(
name='PublishableEntityLink',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')),
(
'upstream_usage_key',
opaque_keys.edx.django.models.UsageKeyField(
help_text='Upstream block usage key, this value cannot be null and useful to track upstream library blocks that do not exist yet',
max_length=255,
),
),
(
'upstream_context_key',
openedx_learning.lib.fields.MultiCollationCharField(
db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'},
db_index=True,
help_text='Upstream context key i.e., learning_package/library key',
max_length=500,
),
),
('downstream_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255, unique=True)),
('downstream_context_key', opaque_keys.edx.django.models.CourseKeyField(db_index=True, max_length=255)),
('version_synced', models.IntegerField()),
('version_declined', models.IntegerField(blank=True, null=True)),
('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])),
('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])),
(
'upstream_block',
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name='links',
to='oel_publishing.publishableentity',
),
),
],
options={
'verbose_name': 'Publishable Entity Link',
'verbose_name_plural': 'Publishable Entity Links',
},
),
]

178
cms/djangoapps/contentstore/models.py
View File

@@ -3,8 +3,20 @@ Models for contentstore
"""
from datetime import datetime, timezone
from config_models.models import ConfigurationModel
from django.db import models
from django.db.models.fields import IntegerField, TextField
from django.utils.translation import gettext_lazy as _
from opaque_keys.edx.django.models import CourseKeyField, UsageKeyField
from opaque_keys.edx.keys import CourseKey, UsageKey
from openedx_learning.api.authoring_models import Component, PublishableEntity
from openedx_learning.lib.fields import (
immutable_uuid_field,
key_field,
manual_date_time_field,
)
class VideoUploadConfig(ConfigurationModel):
@@ -63,3 +75,169 @@ class CleanStaleCertificateAvailabilityDatesConfig(ConfigurationModel):
"`clean_stale_certificate_available_dates` management command.' See the management command for options."
)
)
class PublishableEntityLink(models.Model):
"""
This represents link between any two publishable entities or link between publishable entity and a course
xblock. It helps in tracking relationship between xblocks imported from libraries and used in different courses.
"""
uuid = immutable_uuid_field()
upstream_block = models.ForeignKey(
PublishableEntity,
on_delete=models.SET_NULL,
related_name="links",
null=True,
blank=True,
)
upstream_usage_key = UsageKeyField(
max_length=255,
help_text=_(
"Upstream block usage key, this value cannot be null"
" and useful to track upstream library blocks that do not exist yet"
)
)
# Search by library/upstream context key
upstream_context_key = key_field(
help_text=_("Upstream context key i.e., learning_package/library key"),
db_index=True,
)
# A downstream entity can only link to single upstream entity
# whereas an entity can be upstream for multiple downstream entities.
downstream_usage_key = UsageKeyField(max_length=255, unique=True)
# Search by course/downstream key
downstream_context_key = CourseKeyField(max_length=255, db_index=True)
version_synced = models.IntegerField()
version_declined = models.IntegerField(null=True, blank=True)
created = manual_date_time_field()
updated = manual_date_time_field()
def __str__(self):
return f"{self.upstream_usage_key}->{self.downstream_usage_key}"
class Meta:
verbose_name = _("Publishable Entity Link")
verbose_name_plural = _("Publishable Entity Links")
@classmethod
def update_or_create(
cls,
upstream_block: Component | None,
/,
upstream_usage_key: UsageKey,
upstream_context_key: str,
downstream_usage_key: UsageKey,
downstream_context_key: CourseKey,
version_synced: int,
version_declined: int | None = None,
created: datetime | None = None,
) -> "PublishableEntityLink":
"""
Update or create entity link. This will only update `updated` field if something has changed.
"""
if not created:
created = datetime.now(tz=timezone.utc)
new_values = {
'upstream_usage_key': upstream_usage_key,
'upstream_context_key': upstream_context_key,
'downstream_usage_key': downstream_usage_key,
'downstream_context_key': downstream_context_key,
'version_synced': version_synced,
'version_declined': version_declined,
}
if upstream_block:
new_values.update(
{
'upstream_block': upstream_block.publishable_entity,
}
)
try:
link = cls.objects.get(downstream_usage_key=downstream_usage_key)
has_changes = False
for key, value in new_values.items():
prev = getattr(link, key)
# None != None is True, so we need to check for it specially
if prev != value and ~(prev is None and value is None):
has_changes = True
setattr(link, key, value)
if has_changes:
link.updated = created
link.save()
except cls.DoesNotExist:
link = cls(**new_values)
link.created = created
link.updated = created
link.save()
return link
class LearningContextLinksStatusChoices(models.TextChoices):
"""
Enumerates the states that a LearningContextLinksStatus can be in.
"""
PENDING = "pending", _("Pending")
PROCESSING = "processing", _("Processing")
FAILED = "failed", _("Failed")
COMPLETED = "completed", _("Completed")
class LearningContextLinksStatus(models.Model):
"""
This table stores current processing status of upstream-downstream links in PublishableEntityLink table for a
course or a learning context.
"""
context_key = CourseKeyField(
max_length=255,
# Single entry for a learning context or course
unique=True,
help_text=_("Linking status for course context key"),
)
status = models.CharField(
max_length=20,
choices=LearningContextLinksStatusChoices.choices,
help_text=_("Status of links in given learning context/course."),
)
created = manual_date_time_field()
updated = manual_date_time_field()
class Meta:
verbose_name = _("Learning Context Links status")
verbose_name_plural = _("Learning Context Links status")
def __str__(self):
return f"{self.status}|{self.context_key}"
@classmethod
def get_or_create(cls, context_key: str, created: datetime | None = None) -> "LearningContextLinksStatus":
"""
Get or create course link status row from LearningContextLinksStatus table for given course key.
Args:
context_key: Learning context or Course key
Returns:
LearningContextLinksStatus object
"""
if not created:
created = datetime.now(tz=timezone.utc)
status, _ = cls.objects.get_or_create(
context_key=context_key,
defaults={
'status': LearningContextLinksStatusChoices.PENDING,
'created': created,
'updated': created,
},
)
return status
def update_status(
self,
status: LearningContextLinksStatusChoices,
updated: datetime | None = None
) -> None:
"""
Updates entity links processing status of given learning context.
"""
self.status = status
self.updated = updated or datetime.now(tz=timezone.utc)
self.save()

69
cms/djangoapps/contentstore/signals/handlers.py
View File

@@ -12,8 +12,14 @@ from django.db import transaction
from django.dispatch import receiver
from edx_toggles.toggles import SettingToggle
from opaque_keys.edx.keys import CourseKey
from openedx_events.content_authoring.data import CourseCatalogData, CourseScheduleData
from openedx_events.content_authoring.signals import COURSE_CATALOG_INFO_CHANGED
from openedx_events.content_authoring.data import CourseCatalogData, CourseData, CourseScheduleData, XBlockData
from openedx_events.content_authoring.signals import (
COURSE_CATALOG_INFO_CHANGED,
COURSE_IMPORT_COMPLETED,
XBLOCK_CREATED,
XBLOCK_DELETED,
XBLOCK_UPDATED,
)
from pytz import UTC
from cms.djangoapps.contentstore.courseware_index import (
@@ -29,6 +35,10 @@ from openedx.core.djangoapps.discussions.tasks import update_discussions_setting
from openedx.core.lib.gating import api as gating_api
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import SignalHandler, modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from ..models import PublishableEntityLink
from ..tasks import create_or_update_upstream_links, handle_create_or_update_xblock_upstream_link
from .signals import GRADING_POLICY_CHANGED
log = logging.getLogger(__name__)
@@ -197,12 +207,19 @@ def handle_item_deleted(**kwargs):
# Strip branch info
usage_key = usage_key.for_branch(None)
course_key = usage_key.course_key
deleted_block = modulestore().get_item(usage_key)
try:
deleted_block = modulestore().get_item(usage_key)
except ItemNotFoundError:
return
id_list = {deleted_block.location}
for block in yield_dynamic_block_descendants(deleted_block, kwargs.get('user_id')):
# Remove prerequisite milestone data
gating_api.remove_prerequisite(block.location)
# Remove any 'requires' course content milestone relationships
gating_api.set_required_content(course_key, block.location, None, None, None)
id_list.add(block.location)
PublishableEntityLink.objects.filter(downstream_usage_key__in=id_list).delete()
@receiver(GRADING_POLICY_CHANGED)
@@ -224,3 +241,49 @@ def handle_grading_policy_changed(sender, **kwargs):
task_id=result.task_id,
kwargs=kwargs,
))
@receiver(XBLOCK_CREATED)
@receiver(XBLOCK_UPDATED)
def create_or_update_upstream_downstream_link_handler(**kwargs):
"""
Automatically create or update upstream->downstream link in database.
"""
xblock_info = kwargs.get("xblock_info", None)
if not xblock_info or not isinstance(xblock_info, XBlockData):
log.error("Received null or incorrect data for event")
return
handle_create_or_update_xblock_upstream_link.delay(str(xblock_info.usage_key))
@receiver(XBLOCK_DELETED)
def delete_upstream_downstream_link_handler(**kwargs):
"""
Delete upstream->downstream link from database on xblock delete.
"""
xblock_info = kwargs.get("xblock_info", None)
if not xblock_info or not isinstance(xblock_info, XBlockData):
log.error("Received null or incorrect data for event")
return
PublishableEntityLink.objects.filter(
downstream_usage_key=xblock_info.usage_key
).delete()
@receiver(COURSE_IMPORT_COMPLETED)
def handle_new_course_import(**kwargs):
"""
Automatically create upstream->downstream links for course in database on new import.
"""
course_data = kwargs.get("course", None)
if not course_data or not isinstance(course_data, CourseData):
log.error("Received null or incorrect data for event")
return
create_or_update_upstream_links.delay(
str(course_data.course_key),
force=True,
replace=True
)

78
cms/djangoapps/contentstore/tasks.py
View File

@@ -2,17 +2,17 @@
This file contains celery tasks for contentstore views
"""
import asyncio
import base64
import json
import os
import re
import shutil
import tarfile
import re
import aiohttp
import asyncio
from datetime import datetime
from datetime import datetime, timezone
from tempfile import NamedTemporaryFile, mkdtemp
import aiohttp
import olxcleaner
import pkg_resources
from ccx_keys.locator import CCXLocator
@@ -28,11 +28,12 @@ from edx_django_utils.monitoring import (
set_code_owner_attribute,
set_code_owner_attribute_from_module,
set_custom_attribute,
set_custom_attributes_for_course_key
set_custom_attributes_for_course_key,
)
from olxcleaner.exceptions import ErrorLevel
from olxcleaner.reporting import report_error_summary, report_errors
from opaque_keys.edx.keys import CourseKey
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey, UsageKey
from opaque_keys.edx.locator import LibraryLocator
from organizations.api import add_organization_course, ensure_organization
from organizations.exceptions import InvalidOrganizationException
@@ -46,15 +47,16 @@ import cms.djangoapps.contentstore.errors as UserErrors
from cms.djangoapps.contentstore.courseware_index import (
CoursewareSearchIndexer,
LibrarySearchIndexer,
SearchIndexingError
SearchIndexingError,
)
from cms.djangoapps.contentstore.storage import course_import_export_storage
from cms.djangoapps.contentstore.utils import (
IMPORTABLE_FILE_TYPES,
create_or_update_xblock_upstream_link,
delete_course,
initialize_permissions,
reverse_usage_url,
translation_language,
delete_course
)
from cms.djangoapps.contentstore.xblock_storage_handlers.view_handlers import get_block_info
from cms.djangoapps.models.settings.course_metadata import CourseMetadata
@@ -70,6 +72,7 @@ from openedx.core.djangoapps.discussions.config.waffle import ENABLE_NEW_STRUCTU
from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, Provider
from openedx.core.djangoapps.discussions.tasks import update_unit_discussion_state_from_discussion_blocks
from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse
from openedx.core.lib import ensure_cms
from openedx.core.lib.extract_archive import safe_extractall
from xmodule.contentstore.django import contentstore
from xmodule.course_block import CourseFields
@@ -79,6 +82,8 @@ from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import DuplicateCourseError, InvalidProctoringProvider, ItemNotFoundError
from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml
from xmodule.modulestore.xml_importer import CourseImportException, import_course_from_xml, import_library_from_xml
from .models import LearningContextLinksStatus, LearningContextLinksStatusChoices, PublishableEntityLink
from .outlines import update_outline_from_modulestore
from .outlines_regenerate import CourseOutlineRegenerate
from .toggles import bypass_olx_failure_enabled
@@ -1404,3 +1409,60 @@ def _save_broken_links_file(artifact, file_to_save):
def _write_broken_links_to_file(broken_or_locked_urls, broken_links_file):
with open(broken_links_file.name, 'w') as file:
json.dump(broken_or_locked_urls, file, indent=4)
@shared_task
@set_code_owner_attribute
def handle_create_or_update_xblock_upstream_link(usage_key):
"""
Create or update upstream link for a single xblock.
"""
ensure_cms("handle_create_or_update_xblock_upstream_link may only be executed in a CMS context")
try:
xblock = modulestore().get_item(UsageKey.from_string(usage_key))
except (ItemNotFoundError, InvalidKeyError):
LOGGER.exception(f'Could not find item for given usage_key: {usage_key}')
return
if not xblock.upstream or not xblock.upstream_version:
return
create_or_update_xblock_upstream_link(xblock, xblock.course_id)
@shared_task
@set_code_owner_attribute
def create_or_update_upstream_links(
course_key_str: str,
force: bool = False,
replace: bool = False,
created: datetime | None = None,
):
"""
A Celery task to create or update upstream downstream links in database from course xblock content.
"""
ensure_cms("create_or_update_upstream_links may only be executed in a CMS context")
if not created:
created = datetime.now(timezone.utc)
course_status = LearningContextLinksStatus.get_or_create(course_key_str, created)
if course_status.status in [
LearningContextLinksStatusChoices.COMPLETED,
LearningContextLinksStatusChoices.PROCESSING
] and not force:
return
store = modulestore()
course_key = CourseKey.from_string(course_key_str)
course_status.update_status(
LearningContextLinksStatusChoices.PROCESSING,
updated=created,
)
if replace:
PublishableEntityLink.objects.filter(downstream_context_key=course_key).delete()
try:
xblocks = store.get_items(course_key, settings={"upstream": lambda x: x is not None})
except ItemNotFoundError:
LOGGER.exception(f'Could not find items for given course: {course_key}')
course_status.update_status(LearningContextLinksStatusChoices.FAILED)
return
for xblock in xblocks:
create_or_update_xblock_upstream_link(xblock, course_key_str, created)
course_status.update_status(LearningContextLinksStatusChoices.COMPLETED)

274
cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py Normal file
View File

@@ -0,0 +1,274 @@
"""
Tests for upstream downstream tracking links.
"""
from io import StringIO
from uuid import uuid4
from django.core.management import call_command
from django.core.management.base import CommandError
from django.test import TestCase
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryUsageLocatorV2
from openedx_events.tests.utils import OpenEdxEventsTestMixin
from common.djangoapps.student.tests.factories import UserFactory
from openedx.core.djangolib.testing.utils import skip_unless_cms
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory
from ..models import LearningContextLinksStatus, LearningContextLinksStatusChoices, PublishableEntityLink
class BaseUpstreamLinksHelpers(TestCase):
"""
Base class with helpers to create xblocks.
"""
def _set_course_data(self, course):
self.section = BlockFactory.create(parent=course, category="chapter", display_name="Section") # pylint: disable=attribute-defined-outside-init
self.sequence = BlockFactory.create(parent=self.section, category="sequential", display_name="Sequence") # pylint: disable=attribute-defined-outside-init
self.unit = BlockFactory.create(parent=self.sequence, category="vertical", display_name="Unit") # pylint: disable=attribute-defined-outside-init
def _create_block(self, num: int, category="html"):
"""
Create xblock with random upstream key and version number.
"""
random_upstream = LibraryUsageLocatorV2.from_string(
f"lb:OpenedX:CSPROB2:{category}:{uuid4()}"
)
return random_upstream, BlockFactory.create(
parent=self.unit, # pylint: disable=attribute-defined-outside-init
category=category,
display_name=f"An {category} Block - {num}",
upstream=str(random_upstream),
upstream_version=num,
)
def _create_block_and_expected_links_data(self, course_key: str | CourseKey, num_blocks: int = 3):
"""
Creates xblocks and its expected links data for given course_key
"""
data = []
for i in range(num_blocks):
upstream, block = self._create_block(i + 1)
data.append({
"upstream_block": None,
"downstream_context_key": course_key,
"downstream_usage_key": block.usage_key,
"upstream_usage_key": upstream,
"upstream_context_key": str(upstream.context_key),
"version_synced": i + 1,
"version_declined": None,
})
return data
def _compare_links(self, course_key, expected):
"""
Compares links for given course with passed expected list of dicts.
"""
links = list(PublishableEntityLink.objects.filter(downstream_context_key=course_key).values(
'upstream_block',
'upstream_usage_key',
'upstream_context_key',
'downstream_usage_key',
'downstream_context_key',
'version_synced',
'version_declined',
))
self.assertListEqual(links, expected)
@skip_unless_cms
class TestRecreateUpstreamLinks(ModuleStoreTestCase, OpenEdxEventsTestMixin, BaseUpstreamLinksHelpers):
"""
Test recreate_upstream_links management command.
"""
ENABLED_SIGNALS = ['course_deleted', 'course_published']
ENABLED_OPENEDX_EVENTS = []
def setUp(self):
super().setUp()
self.user = UserFactory()
self.course_1 = course_1 = CourseFactory.create(emit_signals=True)
self.course_key_1 = course_key_1 = self.course_1.id
with self.store.bulk_operations(course_key_1):
self._set_course_data(course_1)
self.expected_links_1 = self._create_block_and_expected_links_data(course_key_1)
self.course_2 = course_2 = CourseFactory.create(emit_signals=True)
self.course_key_2 = course_key_2 = self.course_2.id
with self.store.bulk_operations(course_key_2):
self._set_course_data(course_2)
self.expected_links_2 = self._create_block_and_expected_links_data(course_key_2)
self.course_3 = course_3 = CourseFactory.create(emit_signals=True)
self.course_key_3 = course_key_3 = self.course_3.id
with self.store.bulk_operations(course_key_3):
self._set_course_data(course_3)
self.expected_links_3 = self._create_block_and_expected_links_data(course_key_3)
def call_command(self, *args, **kwargs):
"""
call command with pass args.
"""
out = StringIO()
kwargs['stdout'] = out
err = StringIO()
kwargs['stderr'] = err
call_command('recreate_upstream_links', *args, **kwargs)
return out, err
def test_call_with_invalid_args(self):
"""
Test command with invalid args.
"""
with self.assertRaisesRegex(CommandError, 'Either --course or --all argument'):
self.call_command()
with self.assertRaisesRegex(CommandError, 'Only one of --course or --all argument'):
self.call_command('--all', '--course', str(self.course_key_1))
def test_call_for_single_course(self):
"""
Test command with single course argument
"""
# Pre-checks
assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_1)).exists()
assert not PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_1).exists()
# Run command
self.call_command('--course', str(self.course_key_1))
# Post verfication
assert LearningContextLinksStatus.objects.filter(
context_key=str(self.course_key_1)
).first().status == LearningContextLinksStatusChoices.COMPLETED
self._compare_links(self.course_key_1, self.expected_links_1)
def test_call_for_multiple_course(self):
"""
Test command with multiple course arguments
"""
# Pre-checks
assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_2)).exists()
assert not PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_2).exists()
assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_3)).exists()
assert not PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_3).exists()
# Run command
self.call_command('--course', str(self.course_key_2), '--course', str(self.course_key_3))
# Post verfication
assert LearningContextLinksStatus.objects.filter(
context_key=str(self.course_key_2)
).first().status == LearningContextLinksStatusChoices.COMPLETED
assert LearningContextLinksStatus.objects.filter(
context_key=str(self.course_key_3)
).first().status == LearningContextLinksStatusChoices.COMPLETED
self._compare_links(self.course_key_2, self.expected_links_2)
self._compare_links(self.course_key_3, self.expected_links_3)
def test_call_for_all_courses(self):
"""
Test command with multiple course arguments
"""
# Delete all links and status just to make sure --all option works
LearningContextLinksStatus.objects.all().delete()
PublishableEntityLink.objects.all().delete()
# Pre-checks
assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_1)).exists()
assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_2)).exists()
assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_3)).exists()
# Run command
self.call_command('--all')
# Post verfication
assert LearningContextLinksStatus.objects.filter(
context_key=str(self.course_key_1)
).first().status == LearningContextLinksStatusChoices.COMPLETED
assert LearningContextLinksStatus.objects.filter(
context_key=str(self.course_key_2)
).first().status == LearningContextLinksStatusChoices.COMPLETED
assert LearningContextLinksStatus.objects.filter(
context_key=str(self.course_key_3)
).first().status == LearningContextLinksStatusChoices.COMPLETED
self._compare_links(self.course_key_1, self.expected_links_1)
self._compare_links(self.course_key_2, self.expected_links_2)
self._compare_links(self.course_key_3, self.expected_links_3)
def test_call_for_invalid_course(self):
"""
Test recreate_upstream_links with nonexistent course
"""
course_key = "invalid-course"
with self.assertLogs(level="ERROR") as ctx:
self.call_command('--course', course_key)
self.assertEqual(
f'Invalid course key: {course_key}, skipping..',
ctx.records[0].getMessage()
)
def test_call_for_nonexistent_course(self):
"""
Test recreate_upstream_links with nonexistent course
"""
course_key = "course-v1:unix+ux1+2024_T2"
with self.assertLogs(level="ERROR") as ctx:
self.call_command('--course', course_key)
self.assertIn(
f'Could not find items for given course: {course_key}',
ctx.records[0].getMessage()
)
@skip_unless_cms
class TestUpstreamLinksEvents(ModuleStoreTestCase, OpenEdxEventsTestMixin, BaseUpstreamLinksHelpers):
"""
Test signals related to managing upstream->downstream links.
"""
ENABLED_SIGNALS = ['course_deleted', 'course_published']
ENABLED_OPENEDX_EVENTS = [
"org.openedx.content_authoring.xblock.created.v1",
"org.openedx.content_authoring.xblock.updated.v1",
"org.openedx.content_authoring.xblock.deleted.v1",
]
def setUp(self):
super().setUp()
self.user = UserFactory()
self.course_1 = course_1 = CourseFactory.create(emit_signals=True)
self.course_key_1 = course_key_1 = self.course_1.id
with self.store.bulk_operations(course_key_1):
self._set_course_data(course_1)
self.expected_links_1 = self._create_block_and_expected_links_data(course_key_1)
self.course_2 = course_2 = CourseFactory.create(emit_signals=True)
self.course_key_2 = course_key_2 = self.course_2.id
with self.store.bulk_operations(course_key_2):
self._set_course_data(course_2)
self.expected_links_2 = self._create_block_and_expected_links_data(course_key_2)
self.course_3 = course_3 = CourseFactory.create(emit_signals=True)
self.course_key_3 = course_key_3 = self.course_3.id
with self.store.bulk_operations(course_key_3):
self._set_course_data(course_3)
self.expected_links_3 = self._create_block_and_expected_links_data(course_key_3)
def test_create_or_update_events(self):
"""
Test task create_or_update_upstream_links for a course
"""
assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_1)).exists()
assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_2)).exists()
assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_3)).exists()
assert PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_1).count() == 3
assert PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_2).count() == 3
assert PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_3).count() == 3
self._compare_links(self.course_key_1, self.expected_links_1)
self._compare_links(self.course_key_2, self.expected_links_2)
self._compare_links(self.course_key_3, self.expected_links_3)
def test_delete_handler(self):
"""
Test whether links are deleted on deletion of xblock.
"""
usage_key = self.expected_links_1[0]["downstream_usage_key"]
assert PublishableEntityLink.objects.filter(downstream_usage_key=usage_key).exists()
self.store.delete_item(usage_key, self.user.id)
assert not PublishableEntityLink.objects.filter(downstream_usage_key=usage_key).exists()

15
cms/djangoapps/contentstore/tests/test_utils.py
View File

@@ -12,6 +12,7 @@ from django.test.utils import override_settings
from edx_toggles.toggles.testutils import override_waffle_flag
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import CourseLocator, LibraryLocator
from openedx_events.tests.utils import OpenEdxEventsTestMixin
from path import Path as path
from pytz import UTC
from rest_framework import status
@@ -31,10 +32,13 @@ from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disa
from xmodule.modulestore.tests.django_utils import ( # lint-amnesty, pylint: disable=wrong-import-order
TEST_DATA_SPLIT_MODULESTORE,
ModuleStoreTestCase,
SharedModuleStoreTestCase
SharedModuleStoreTestCase,
)
from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.partitions.partitions import Group, UserPartition # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.modulestore.tests.factories import (
BlockFactory,
CourseFactory,
)
from xmodule.partitions.partitions import Group, UserPartition
class LMSLinksTestCase(TestCase):
@@ -935,10 +939,13 @@ class UpdateCourseDetailsTests(ModuleStoreTestCase):
@override_waffle_flag(ENABLE_NOTIFICATIONS, active=True)
class CourseUpdateNotificationTests(ModuleStoreTestCase):
class CourseUpdateNotificationTests(OpenEdxEventsTestMixin, ModuleStoreTestCase):
"""
Unit tests for the course_update notification.
"""
ENABLED_OPENEDX_EVENTS = [
"org.openedx.learning.course.notification.requested.v1",
]
def setUp(self):
"""

65
cms/djangoapps/contentstore/utils.py
View File

@@ -2,6 +2,7 @@
Common utility functions useful throughout the contentstore
"""
from __future__ import annotations
import configparser
import html
import logging
@@ -9,12 +10,12 @@ import re
from collections import defaultdict
from contextlib import contextmanager
from datetime import datetime, timezone
from urllib.parse import quote_plus, urlencode, urlunparse, urlparse
from urllib.parse import quote_plus, urlencode, urlparse, urlunparse
from uuid import uuid4
from bs4 import BeautifulSoup
from django.conf import settings
from django.core.exceptions import ValidationError
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.urls import reverse
from django.utils import translation
from django.utils.text import Truncator
@@ -22,16 +23,13 @@ from django.utils.translation import gettext as _
from eventtracking import tracker
from help_tokens.core import HelpUrlExpert
from lti_consumer.models import CourseAllowPIISharingInLTIFlag
from opaque_keys.edx.keys import CourseKey, UsageKey
from milestones import api as milestones_api
from opaque_keys.edx.keys import CourseKey, UsageKey, UsageKeyV2
from opaque_keys.edx.locator import LibraryLocator
from openedx.core.lib.teams_config import CONTENT_GROUPS_FOR_TEAMS, TEAM_SCHEME
from openedx_events.content_authoring.data import DuplicatedXBlockData
from openedx_events.content_authoring.signals import XBLOCK_DUPLICATED
from openedx_events.learning.data import CourseNotificationData
from openedx_events.learning.signals import COURSE_NOTIFICATION_REQUESTED
from milestones import api as milestones_api
from pytz import UTC
from xblock.fields import Scope
@@ -42,13 +40,14 @@ from cms.djangoapps.contentstore.toggles import (
libraries_v2_enabled,
split_library_view_on_dashboard,
use_new_advanced_settings_page,
use_new_course_outline_page,
use_new_certificates_page,
use_new_course_outline_page,
use_new_course_team_page,
use_new_custom_pages,
use_new_export_page,
use_new_files_uploads_page,
use_new_grading_page,
use_new_group_configurations_page,
use_new_course_team_page,
use_new_home_page,
use_new_import_page,
use_new_schedule_details_page,
@@ -58,16 +57,15 @@ from cms.djangoapps.contentstore.toggles import (
use_new_updates_page,
use_new_video_editor,
use_new_video_uploads_page,
use_new_custom_pages,
)
from cms.djangoapps.models.settings.course_grading import CourseGradingModel
from cms.djangoapps.models.settings.course_metadata import CourseMetadata
from common.djangoapps.course_action_state.models import CourseRerunUIStateManager, CourseRerunState
from common.djangoapps.course_action_state.managers import CourseActionStateItemNotFoundError
from common.djangoapps.course_action_state.models import CourseRerunState, CourseRerunUIStateManager
from common.djangoapps.course_modes.models import CourseMode
from common.djangoapps.edxmako.services import MakoService
from common.djangoapps.student import auth
from common.djangoapps.student.auth import has_studio_read_access, has_studio_write_access, STUDIO_EDIT_ROLES
from common.djangoapps.student.auth import STUDIO_EDIT_ROLES, has_studio_read_access, has_studio_write_access
from common.djangoapps.student.models import CourseEnrollment
from common.djangoapps.student.roles import (
CourseInstructorRole,
@@ -76,15 +74,15 @@ from common.djangoapps.student.roles import (
)
from common.djangoapps.track import contexts
from common.djangoapps.util.course import get_link_for_about_page
from common.djangoapps.util.date_utils import get_default_time_display
from common.djangoapps.util.milestones_helpers import (
generate_milestone_namespace,
get_namespace_choices,
is_prerequisite_courses_enabled,
is_valid_course_key,
remove_prerequisite_course,
set_prerequisite_courses,
get_namespace_choices,
generate_milestone_namespace
)
from common.djangoapps.util.date_utils import get_default_time_display
from common.djangoapps.xblock_django.api import deprecated_xblocks
from common.djangoapps.xblock_django.user_service import DjangoXBlockUserService
from openedx.core import toggles as core_toggles
@@ -94,23 +92,28 @@ from openedx.core.djangoapps.discussions.config.waffle import ENABLE_PAGES_AND_R
from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration
from openedx.core.djangoapps.django_comment_common.models import assign_default_role
from openedx.core.djangoapps.django_comment_common.utils import seed_permissions_roles
from openedx.core.djangoapps.models.course_details import CourseDetails
from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
from openedx.core.djangoapps.site_configuration.models import SiteConfiguration
from openedx.core.djangoapps.models.course_details import CourseDetails
from openedx.core.djangoapps.xblock.api import get_component_from_usage_key
from openedx.core.lib.courses import course_image_url
from openedx.core.lib.html_to_text import html_to_text
from openedx.core.lib.teams_config import CONTENT_GROUPS_FOR_TEAMS, TEAM_SCHEME
from openedx.features.content_type_gating.models import ContentTypeGatingConfig
from openedx.features.content_type_gating.partitions import CONTENT_TYPE_GATING_SCHEME
from openedx.features.course_experience.waffle import ENABLE_COURSE_ABOUT_SIDEBAR_HTML
from xmodule.library_tools import LegacyLibraryToolsService
from xmodule.course_block import DEFAULT_START_DATE # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.data import CertificatesDisplayBehaviors
from xmodule.library_tools import LegacyLibraryToolsService
from xmodule.modulestore import ModuleStoreEnum # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.modulestore.exceptions import ItemNotFoundError # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.partitions.partitions_service import get_all_partitions_for_course # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.services import SettingsService, ConfigurationService, TeamsConfigurationService
from xmodule.partitions.partitions_service import (
get_all_partitions_for_course, # lint-amnesty, pylint: disable=wrong-import-order
)
from xmodule.services import ConfigurationService, SettingsService, TeamsConfigurationService
from .models import PublishableEntityLink
IMPORTABLE_FILE_TYPES = ('.tar.gz', '.zip')
log = logging.getLogger(__name__)
@@ -2354,3 +2357,27 @@ def get_xblock_render_error(request, xblock):
return str(exc)
return ""
def create_or_update_xblock_upstream_link(xblock, course_key: str | CourseKey, created: datetime | None = None):
"""
Create or update upstream->downstream link in database for given xblock.
"""
if not xblock.upstream:
return None
upstream_usage_key = UsageKeyV2.from_string(xblock.upstream)
try:
lib_component = get_component_from_usage_key(upstream_usage_key)
except ObjectDoesNotExist:
log.error(f"Library component not found for {upstream_usage_key}")
lib_component = None
PublishableEntityLink.update_or_create(
lib_component,
upstream_usage_key=xblock.upstream,
upstream_context_key=str(upstream_usage_key.context_key),
downstream_context_key=course_key,
downstream_usage_key=xblock.usage_key,
version_synced=xblock.upstream_version,
version_declined=xblock.upstream_version_declined,
created=created,
)

3
openedx/core/djangoapps/content_libraries/signal_handlers.py
View File

@@ -5,7 +5,7 @@ Content library signal handlers.
import logging
from django.conf import settings
from django.db.models.signals import post_save, post_delete, m2m_changed
from django.db.models.signals import m2m_changed, post_delete, post_save
from django.dispatch import receiver
from opaque_keys import InvalidKeyError
@@ -28,7 +28,6 @@ from lms.djangoapps.grades.api import signals as grades_signals
from .api import library_component_usage_key
from .models import ContentLibrary, LtiGradedResource
log = logging.getLogger(__name__)

2
openedx/core/djangoapps/content_libraries/tasks.py
View File

@@ -22,11 +22,11 @@ from celery import shared_task
from celery_utils.logged_task import LoggedTask
from celery.utils.log import get_task_logger
from edx_django_utils.monitoring import set_code_owner_attribute, set_code_owner_attribute_from_module
from opaque_keys.edx.keys import CourseKey
from user_tasks.tasks import UserTask, UserTaskStatus
from xblock.fields import Scope
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import BlockUsageLocator
from openedx.core.lib import ensure_cms
from xmodule.capa_block import ProblemBlock

2
requirements/edx/base.txt
View File

@@ -811,7 +811,7 @@ openedx-django-require==2.1.0
# via -r requirements/edx/kernel.in
openedx-django-wiki==2.1.0
# via -r requirements/edx/kernel.in
openedx-events==9.15.2
openedx-events==9.18.0
# via
# -r requirements/edx/kernel.in
# edx-enterprise

2
requirements/edx/development.txt
View File

@@ -1361,7 +1361,7 @@ openedx-django-wiki==2.1.0
# via
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
openedx-events==9.15.2
openedx-events==9.18.0
# via
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt

2
requirements/edx/doc.txt
View File

@@ -984,7 +984,7 @@ openedx-django-require==2.1.0
# via -r requirements/edx/base.txt
openedx-django-wiki==2.1.0
# via -r requirements/edx/base.txt
openedx-events==9.15.2
openedx-events==9.18.0
# via
# -r requirements/edx/base.txt
# edx-enterprise

2
requirements/edx/testing.txt
View File

@@ -1032,7 +1032,7 @@ openedx-django-require==2.1.0
# via -r requirements/edx/base.txt
openedx-django-wiki==2.1.0
# via -r requirements/edx/base.txt
openedx-events==9.15.2
openedx-events==9.18.0
# via
# -r requirements/edx/base.txt
# edx-enterprise

13
xmodule/modulestore/tests/test_mixed_modulestore.py
View File

@@ -164,6 +164,19 @@ class CommonMixedModuleStoreSetup(CourseComparisonTest, OpenEdxEventsTestMixin):
self.course_locations = {}
self.user_id = ModuleStoreEnum.UserID.test
# mock and ignore publishable link entity related tasks to avoid unnecessary
# errors as it is tested separately
if settings.ROOT_URLCONF == 'cms.urls':
create_or_update_xblock_upstream_link_patch = patch(
'cms.djangoapps.contentstore.signals.handlers.handle_create_or_update_xblock_upstream_link'
)
create_or_update_xblock_upstream_link_patch.start()
self.addCleanup(create_or_update_xblock_upstream_link_patch.stop)
publishableEntityLinkPatch = patch(
'cms.djangoapps.contentstore.signals.handlers.PublishableEntityLink'
)
publishableEntityLinkPatch.start()
self.addCleanup(publishableEntityLinkPatch.stop)
def _check_connection(self):
"""

23
xmodule/modulestore/xml_importer.py
View File

@@ -27,6 +27,7 @@ import mimetypes
import os
import re
from abc import abstractmethod
from datetime import datetime, timezone
import xblock
from django.core.exceptions import ObjectDoesNotExist
@@ -34,12 +35,15 @@ from django.utils.translation import gettext as _
from lxml import etree
from opaque_keys.edx.keys import UsageKey
from opaque_keys.edx.locator import LibraryLocator
from openedx_events.content_authoring.data import CourseData
from openedx_events.content_authoring.signals import COURSE_IMPORT_COMPLETED
from path import Path as path
from xblock.core import XBlockMixin
from xblock.fields import Reference, ReferenceList, ReferenceValueDict, Scope
from xblock.runtime import DictKeyValueStore, KvsFieldData
from common.djangoapps.util.monitoring import monitor_import_failure
from openedx.core.djangoapps.content_tagging.api import import_course_tags_from_csv
from xmodule.assetstore import AssetMetadata
from xmodule.contentstore.content import StaticContent
from xmodule.errortracker import make_error_tracker
@@ -52,7 +56,6 @@ from xmodule.modulestore.xml import ImportSystem, LibraryXMLModuleStore, XMLModu
from xmodule.tabs import CourseTabList
from xmodule.util.misc import escape_invalid_characters
from xmodule.x_module import XModuleMixin
from openedx.core.djangoapps.content_tagging.api import import_course_tags_from_csv
from .inheritance import own_metadata
from .store_utilities import rewrite_nonportable_content_links
@@ -548,6 +551,11 @@ class ImportManager:
# pylint: disable=raise-missing-from
raise BlockFailedToImport(leftover.display_name, leftover.location)
def post_course_import(self, dest_id):
"""
Tasks that need to triggered after a course is imported.
"""
def run_imports(self):
"""
Iterate over the given directories and yield courses.
@@ -589,6 +597,7 @@ class ImportManager:
logging.info(f'Course import {dest_id}: No tags.csv file present.')
except ValueError as e:
logging.info(f'Course import {dest_id}: {str(e)}')
self.post_course_import(dest_id)
yield courselike
@@ -717,6 +726,18 @@ class CourseImportManager(ImportManager):
csv_path = path(data_path) / 'tags.csv'
import_course_tags_from_csv(csv_path, dest_id)
def post_course_import(self, dest_id):
"""
Trigger celery task to create upstream links for newly imported blocks.
"""
# .. event_implemented_name: COURSE_IMPORT_COMPLETED
COURSE_IMPORT_COMPLETED.send_event(
time=datetime.now(timezone.utc),
course=CourseData(
course_key=dest_id
)
)
class LibraryImportManager(ImportManager):
"""