Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: handle tags when importing/exporting courses (#34356)

Browse Source
This commit is contained in:
Chris Chávez
2024-04-08 13:39:46 -05:00
committed by GitHub
parent 54eeedf3a4
commit ddb407a3f2
18 changed files with 589 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
cms/djangoapps/contentstore/management/commands/tests/test_export_olx.py
View File

@@ -17,6 +17,8 @@ from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from openedx.core.djangoapps.content_tagging.tests.test_objecttag_export_helpers import TaggedCourseMixin
class TestArgParsingCourseExportOlx(unittest.TestCase):
"""
@@ -31,7 +33,7 @@ class TestArgParsingCourseExportOlx(unittest.TestCase):
call_command('export_olx')
class TestCourseExportOlx(ModuleStoreTestCase):
class TestCourseExportOlx(TaggedCourseMixin, ModuleStoreTestCase):
"""
Test exporting OLX content from a course or library.
"""
@@ -61,7 +63,7 @@ class TestCourseExportOlx(ModuleStoreTestCase):
)
return course.id
def check_export_file(self, tar_file, course_key):
def check_export_file(self, tar_file, course_key, with_tags=False):
"""Check content of export file."""
names = tar_file.getnames()
dirname = "{0.org}-{0.course}-{0.run}".format(course_key)
@@ -71,6 +73,10 @@ class TestCourseExportOlx(ModuleStoreTestCase):
self.assertIn(f"{dirname}/about/overview.html", names)
self.assertIn(f"{dirname}/assets/assets.xml", names)
self.assertIn(f"{dirname}/policies", names)
if with_tags:
self.assertIn(f"{dirname}/tags.csv", names)
else:
self.assertNotIn(f"{dirname}/tags.csv", names)
def test_export_course(self):
test_course_key = self.create_dummy_course(ModuleStoreEnum.Type.split)
@@ -98,3 +104,11 @@ class TestCourseExportOlx(ModuleStoreTestCase):
output = output_wrapper.bytes_io.read()
with tarfile.open(fileobj=BytesIO(output), mode="r:gz") as tar_file:
self.check_export_file(tar_file, test_course_key)
def test_export_course_with_tags(self):
tmp_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, tmp_dir)
filename = tmp_dir / 'test.tar.gz'
call_command('export_olx', '--output', filename, str(self.course.id))
with tarfile.open(filename) as tar_file:
self.check_export_file(tar_file, self.course.id, with_tags=True)

8
openedx/core/djangoapps/content/search/documents.py
View File

@@ -170,10 +170,10 @@ def _tags_for_content_object(object_id: UsageKey | LearningContextKey) -> dict:
}
for obj_tag in all_tags:
# Add the taxonomy name:
if obj_tag.name not in result[Fields.tags_taxonomy]:
result[Fields.tags_taxonomy].append(obj_tag.name)
# Taxonomy name plus each level of tags, in a list:
parts = [obj_tag.name] + obj_tag.get_lineage() # e.g. ["Location", "North America", "Canada", "Vancouver"]
if obj_tag.taxonomy.name not in result[Fields.tags_taxonomy]:
result[Fields.tags_taxonomy].append(obj_tag.taxonomy.name)
# Taxonomy name plus each level of tags, in a list: # e.g. ["Location", "North America", "Canada", "Vancouver"]
parts = [obj_tag.taxonomy.name] + obj_tag.get_lineage()
parts = [part.replace(" > ", " _ ") for part in parts] # Escape our separator.
# Now we build each level (tags.level0, tags.level1, etc.) as applicable.
# We have a hard-coded limit of 4 levels of tags for now (see Fields.tags above).

132
openedx/core/djangoapps/content_tagging/api.py
View File

@@ -2,15 +2,21 @@
Content Tagging APIs
"""
from __future__ import annotations
import io
from itertools import groupby
import csv
from typing import Iterator
from opaque_keys.edx.keys import UsageKey
import openedx_tagging.core.tagging.api as oel_tagging
from django.db.models import Exists, OuterRef, Q, QuerySet
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryLocatorV2
from openedx_tagging.core.tagging.models import ObjectTag, Taxonomy
from openedx_tagging.core.tagging.models.utils import TAGS_CSV_SEPARATOR
from organizations.models import Organization
from .helpers.objecttag_export_helpers import build_object_tree_with_objecttags, iterate_with_level
from .models import TaxonomyOrg
from .types import ContentKey, TagValuesByObjectIdDict, TagValuesByTaxonomyIdDict, TaxonomyDict
@@ -164,7 +170,7 @@ def get_all_object_tags(
all_object_tags = ObjectTag.objects.filter(
Q(tag__isnull=False, tag__taxonomy__isnull=False),
object_id_clause,
).select_related("tag__taxonomy")
).select_related("tag__taxonomy").order_by("object_id")
if prefetch_orgs:
all_object_tags = all_object_tags.prefetch_related("tag__taxonomy__taxonomyorg_set")
@@ -174,7 +180,8 @@ def get_all_object_tags(
for object_id, block_tags in groupby(all_object_tags, lambda x: x.object_id):
grouped_object_tags[object_id] = {}
for taxonomy_id, taxonomy_tags in groupby(block_tags, lambda x: x.tag.taxonomy_id if x.tag else 0):
block_tags_sorted = sorted(block_tags, key=lambda x: x.tag.taxonomy_id if x.tag else 0) # type: ignore
for taxonomy_id, taxonomy_tags in groupby(block_tags_sorted, lambda x: x.tag.taxonomy_id if x.tag else 0):
object_tags_list = list(taxonomy_tags)
grouped_object_tags[object_id][taxonomy_id] = [
tag.value for tag in object_tags_list
@@ -185,7 +192,7 @@ def get_all_object_tags(
assert object_tags_list[0].tag.taxonomy
taxonomies[taxonomy_id] = object_tags_list[0].tag.taxonomy
return grouped_object_tags, taxonomies
return grouped_object_tags, dict(sorted(taxonomies.items()))
def set_all_object_tags(
@@ -211,6 +218,125 @@ def set_all_object_tags(
)
def generate_csv_rows(object_id, buffer) -> Iterator[str]:
"""
Returns a CSV string with tags and taxonomies of all blocks of `object_id`
"""
content_key = get_content_key_from_string(object_id)
if isinstance(content_key, UsageKey):
raise ValueError("The object_id must be a CourseKey or a LibraryLocatorV2.")
all_object_tags, taxonomies = get_all_object_tags(content_key)
tagged_content = build_object_tree_with_objecttags(content_key, all_object_tags)
header = {"name": "Name", "type": "Type", "id": "ID"}
# Prepare the header for the taxonomies
for taxonomy_id, taxonomy in taxonomies.items():
header[f"taxonomy_{taxonomy_id}"] = taxonomy.export_id
csv_writer = csv.DictWriter(buffer, fieldnames=header.keys(), quoting=csv.QUOTE_NONNUMERIC)
yield csv_writer.writerow(header)
# Iterate over the blocks and yield the rows
for item, level in iterate_with_level(tagged_content):
block_key = get_content_key_from_string(item.block_id)
block_data = {
"name": level * " " + item.display_name,
"type": item.category,
"id": getattr(block_key, 'block_id', item.block_id),
}
# Add the tags for each taxonomy
for taxonomy_id in taxonomies:
if taxonomy_id in item.object_tags:
block_data[f"taxonomy_{taxonomy_id}"] = f"{TAGS_CSV_SEPARATOR} ".join(
list(item.object_tags[taxonomy_id])
)
yield csv_writer.writerow(block_data)
def export_tags_in_csv_file(object_id, file_dir, file_name) -> None:
"""
Writes a CSV file with tags and taxonomies of all blocks of `object_id`
"""
buffer = io.StringIO()
for _ in generate_csv_rows(object_id, buffer):
# The generate_csv_rows function is a generator,
# we don't need to do anything with the result here
pass
with file_dir.open(file_name, 'w') as csv_file:
buffer.seek(0)
csv_file.write(buffer.read())
def set_exported_object_tags(
content_key: ContentKey,
exported_tags: TagValuesByTaxonomyIdDict,
) -> None:
"""
Sets the tags for the given exported content object.
"""
content_key_str = str(content_key)
# Clear all tags related with the content.
oel_tagging.delete_object_tags(content_key_str)
for taxonomy_export_id, tags_values in exported_tags.items():
if not tags_values:
continue
taxonomy = oel_tagging.get_taxonomy_by_export_id(str(taxonomy_export_id))
oel_tagging.tag_object(
object_id=content_key_str,
taxonomy=taxonomy,
tags=tags_values,
create_invalid=True,
taxonomy_export_id=str(taxonomy_export_id),
)
def import_course_tags_from_csv(csv_path, course_id) -> None:
"""
Import tags from a csv file generated on export.
"""
# Open csv file and extract the tags
with open(csv_path, 'r') as csv_file:
csv_reader = csv.DictReader(csv_file)
tags_in_blocks = list(csv_reader)
def get_exported_tags(block) -> TagValuesByTaxonomyIdDict:
"""
Returns a map with taxonomy export_id and tags for this block.
"""
result = {}
for key, value in block.items():
if key in ['Type', 'Name', 'ID'] or not value:
continue
result[key] = value.split(TAGS_CSV_SEPARATOR)
return result
course_key = CourseKey.from_string(str(course_id))
for block in tags_in_blocks:
exported_tags = get_exported_tags(block)
block_type = block.get('Type', '')
block_id = block.get('ID', '')
if not block_type or not block_id:
raise ValueError(f"Invalid format of csv in: '{block}'.")
if block_type == 'course':
set_exported_object_tags(course_key, exported_tags)
else:
block_key = course_key.make_usage_key(block_type, block_id)
set_exported_object_tags(block_key, exported_tags)
def copy_object_tags(
source_content_key: ContentKey,
dest_content_key: ContentKey,

14
openedx/core/djangoapps/content_tagging/auth.py Normal file
View File

@@ -0,0 +1,14 @@
"""
Functions to validate the access in content tagging actions
"""
from openedx_tagging.core.tagging import rules as oel_tagging_rules
def has_view_object_tags_access(user, object_id):
return user.has_perm(
"oel_tagging.view_objecttag",
# The obj arg expects a model, but we are passing an object
oel_tagging_rules.ObjectTagPermissionItem(taxonomy=None, object_id=object_id), # type: ignore[arg-type]
)

0
openedx/core/djangoapps/content_tagging/helpers/__init__.py Normal file
View File

11
openedx/core/djangoapps/content_tagging/rest_api/v1/objecttag_export_helpers.py → openedx/core/djangoapps/content_tagging/helpers/objecttag_export_helpers.py
View File

@@ -12,10 +12,9 @@ from opaque_keys.edx.locator import LibraryLocatorV2
from xblock.core import XBlock
import openedx.core.djangoapps.content_libraries.api as library_api
from openedx.core.djangoapps.content_libraries.api import LibraryXBlockMetadata
from xmodule.modulestore.django import modulestore
from ...types import TagValuesByObjectIdDict, TagValuesByTaxonomyIdDict
from ..types import TagValuesByObjectIdDict, TagValuesByTaxonomyIdDict
@define
@@ -69,7 +68,7 @@ def _get_course_tagged_object_and_children(
def _get_library_tagged_object_and_children(
library_key: LibraryLocatorV2, object_tag_cache: TagValuesByObjectIdDict
) -> tuple[TaggedContent, list[LibraryXBlockMetadata]]:
) -> tuple[TaggedContent, list[library_api.LibraryXBlockMetadata]]:
"""
Returns a TaggedContent with library metadata with its tags, and its children.
"""
@@ -89,7 +88,7 @@ def _get_library_tagged_object_and_children(
library_components = library_api.get_library_components(library_key)
children = [
LibraryXBlockMetadata.from_component(library_key, component)
library_api.LibraryXBlockMetadata.from_component(library_key, component)
for component in library_components
]
@@ -117,7 +116,7 @@ def _get_xblock_tagged_object_and_children(
def _get_library_block_tagged_object(
library_block: LibraryXBlockMetadata, object_tag_cache: TagValuesByObjectIdDict
library_block: library_api.LibraryXBlockMetadata, object_tag_cache: TagValuesByObjectIdDict
) -> tuple[TaggedContent, None]:
"""
Returns a TaggedContent with library content block metadata and its tags,
@@ -144,7 +143,7 @@ def build_object_tree_with_objecttags(
"""
get_tagged_children: Union[
# _get_course_tagged_object_and_children type
Callable[[LibraryXBlockMetadata, dict[str, dict[int, list[Any]]]], tuple[TaggedContent, None]],
Callable[[library_api.LibraryXBlockMetadata, dict[str, dict[int, list[Any]]]], tuple[TaggedContent, None]],
# _get_library_block_tagged_object type
Callable[[UsageKey, dict[str, dict[int, list[Any]]]], tuple[TaggedContent, list[Any]]]
]

24
openedx/core/djangoapps/content_tagging/rest_api/v1/tests/test_views.py
View File

@@ -37,8 +37,7 @@ from openedx.core.djangoapps.content_tagging.models import TaxonomyOrg
from openedx.core.djangoapps.content_tagging.utils import rules_cache
from openedx.core.djangolib.testing.utils import skip_unless_cms
from .test_objecttag_export_helpers import TaggedCourseMixin
from ....tests.test_objecttag_export_helpers import TaggedCourseMixin
User = get_user_model()
@@ -1759,6 +1758,7 @@ class TestObjectTagViewSet(TestObjectTagMixin, APITestCase):
# Fetch this object's tags for a single taxonomy
expected_tags = [{
'name': 'Multiple Taxonomy',
'export_id': '13-multiple-taxonomy',
'taxonomy_id': taxonomy.pk,
'can_tag_object': True,
'tags': [
@@ -1854,24 +1854,8 @@ class TestContentObjectChildrenExportView(TaggedCourseMixin, APITestCase): # ty
assert response.status_code == status.HTTP_200_OK
assert response.headers['Content-Type'] == 'text/csv'
expected_csv = (
'"Name","Type","ID","1-taxonomy-1","2-taxonomy-2"\r\n'
'"Test Course","course","course-v1:orgA+test_course+test_run","Tag 1.1",""\r\n'
'" test sequential","sequential","block-v1:orgA+test_course+test_run+type@sequential+block@test_'
'sequential","Tag 1.1, Tag 1.2","Tag 2.1"\r\n'
'" test vertical1","vertical","block-v1:orgA+test_course+test_run+type@vertical+block@test_'
'vertical1","","Tag 2.2"\r\n'
'" test vertical2","vertical","block-v1:orgA+test_course+test_run+type@vertical+block@test_'
'vertical2","",""\r\n'
'" test html","html","block-v1:orgA+test_course+test_run+type@html+block@test_html","","Tag 2.1"\r\n'
'" untagged sequential","sequential","block-v1:orgA+test_course+test_run+type@sequential+block@untagged_'
'sequential","",""\r\n'
'" untagged vertical","vertical","block-v1:orgA+test_course+test_run+type@vertical+block@untagged_'
'vertical","",""\r\n'
)
zip_content = BytesIO(b"".join(response.streaming_content)).getvalue() # type: ignore[attr-defined]
assert zip_content == expected_csv.encode()
assert zip_content == self.expected_csv.encode()
def test_export_course_anoymous_forbidden(self) -> None:
url = OBJECT_TAGS_EXPORT_URL.format(object_id=str(self.course.id))
@@ -1888,7 +1872,7 @@ class TestContentObjectChildrenExportView(TaggedCourseMixin, APITestCase): # ty
url = OBJECT_TAGS_EXPORT_URL.format(object_id="invalid")
self.client.force_authenticate(user=self.staff)
response = self.client.get(url)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert response.status_code == status.HTTP_403_FORBIDDEN
@skip_unless_cms

85
openedx/core/djangoapps/content_tagging/rest_api/v1/views.py
View File

@@ -3,24 +3,21 @@ Tagging Org API Views
"""
from __future__ import annotations
import csv
from typing import Iterator
from django.db.models import Count
from django.http import StreamingHttpResponse
from opaque_keys.edx.keys import UsageKey
from openedx_tagging.core.tagging import rules as oel_tagging_rules
from openedx_tagging.core.tagging.rest_api.v1.views import ObjectTagView, TaxonomyView
from openedx_tagging.core.tagging import rules as oel_tagging_rules
from rest_framework import status
from rest_framework.decorators import action
from rest_framework.exceptions import PermissionDenied, ValidationError
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.views import APIView
from ...auth import has_view_object_tags_access
from ...api import (
create_taxonomy,
get_all_object_tags,
generate_csv_rows,
get_taxonomies,
get_taxonomies_for_org,
get_taxonomy,
@@ -28,9 +25,7 @@ from ...api import (
set_taxonomy_orgs
)
from ...rules import get_admin_orgs
from ...utils import get_content_key_from_string
from .filters import ObjectTagTaxonomyOrgFilterBackend, UserOrgFilterBackend
from .objecttag_export_helpers import build_object_tree_with_objecttags, iterate_with_level
from .serializers import TaxonomyOrgListQueryParamsSerializer, TaxonomyOrgSerializer, TaxonomyUpdateOrgBodySerializer
@@ -167,64 +162,22 @@ class ObjectTagExportView(APIView):
def write(self, value):
return value
def _generate_csv_rows() -> Iterator[str]:
"""
Receives the blocks, tags and taxonomies and returns a CSV string
"""
header = {"name": "Name", "type": "Type", "id": "ID"}
# Prepare the header for the taxonomies
for taxonomy_id, taxonomy in taxonomies.items():
header[f"taxonomy_{taxonomy_id}"] = taxonomy.export_id
csv_writer = csv.DictWriter(pseudo_buffer, fieldnames=header.keys(), quoting=csv.QUOTE_NONNUMERIC)
yield csv_writer.writerow(header)
# Iterate over the blocks and yield the rows
for item, level in iterate_with_level(tagged_content):
block_data = {
"name": level * " " + item.display_name,
"type": item.category,
"id": item.block_id,
}
# Add the tags for each taxonomy
for taxonomy_id in taxonomies:
if taxonomy_id in item.object_tags:
tag_values = item.object_tags[taxonomy_id]
block_data[f"taxonomy_{taxonomy_id}"] = ", ".join(tag_values)
yield csv_writer.writerow(block_data)
object_id: str = kwargs.get('context_id', None)
try:
content_key = get_content_key_from_string(object_id)
if isinstance(content_key, UsageKey):
raise ValidationError("The object_id must be a CourseKey or a LibraryLocatorV2.")
# Check if the user has permission to view object tags for this object_id
if not self.request.user.has_perm(
"oel_tagging.view_objecttag",
# The obj arg expects a model, but we are passing an object
oel_tagging_rules.ObjectTagPermissionItem(taxonomy=None, object_id=object_id), # type: ignore[arg-type]
):
raise PermissionDenied(
"You do not have permission to view object tags for this object_id."
)
all_object_tags, taxonomies = get_all_object_tags(content_key)
tagged_content = build_object_tree_with_objecttags(content_key, all_object_tags)
except ValueError as e:
raise ValidationError from e
pseudo_buffer = Echo()
return StreamingHttpResponse(
streaming_content=_generate_csv_rows(),
content_type="text/csv",
headers={'Content-Disposition': f'attachment; filename="{object_id}_tags.csv"'},
)
if not has_view_object_tags_access(self.request.user, object_id):
raise PermissionDenied(
"You do not have permission to view object tags for this object_id."
)
try:
return StreamingHttpResponse(
streaming_content=generate_csv_rows(
object_id,
pseudo_buffer,
),
content_type="text/csv",
headers={'Content-Disposition': f'attachment; filename="{object_id}_tags.csv"'},
)
except ValueError as e:
raise ValidationError from e

336
openedx/core/djangoapps/content_tagging/tests/test_api.py
View File

@@ -1,12 +1,15 @@
"""Tests for the Tagging models"""
import time
import io
import os
import tempfile
import ddt
from django.test.testcases import TestCase
from fs.osfs import OSFS
from opaque_keys.edx.keys import CourseKey, UsageKey
from opaque_keys.edx.locator import LibraryLocatorV2
from openedx_tagging.core.tagging.models import ObjectTag
from organizations.models import Organization
from .test_objecttag_export_helpers import TestGetAllObjectTagsMixin, TaggedCourseMixin
from .. import api
from ..utils import rules_cache
@@ -243,185 +246,6 @@ class TestAPITaxonomy(TestTaxonomyMixin, TestCase):
assert result[0]["depth"] == 0
class TestGetAllObjectTagsMixin:
"""
Set up data to test get_all_object_tags functions
"""
def setUp(self):
super().setUp()
self.taxonomy_1 = api.create_taxonomy(name="Taxonomy 1")
api.set_taxonomy_orgs(self.taxonomy_1, all_orgs=True)
api.add_tag_to_taxonomy(
taxonomy=self.taxonomy_1,
tag="Tag 1.1",
)
api.add_tag_to_taxonomy(
taxonomy=self.taxonomy_1,
tag="Tag 1.2",
)
self.taxonomy_2 = api.create_taxonomy(name="Taxonomy 2")
api.set_taxonomy_orgs(self.taxonomy_2, all_orgs=True)
api.add_tag_to_taxonomy(
taxonomy=self.taxonomy_2,
tag="Tag 2.1",
)
api.add_tag_to_taxonomy(
taxonomy=self.taxonomy_2,
tag="Tag 2.2",
)
api.tag_object(
object_id="course-v1:orgA+test_course+test_run",
taxonomy=self.taxonomy_1,
tags=['Tag 1.1'],
)
self.course_tags = api.get_object_tags("course-v1:orgA+test_course+test_run")
self.orgA = Organization.objects.create(name="Organization A", short_name="orgA")
self.orgB = Organization.objects.create(name="Organization B", short_name="orgB")
self.taxonomy_3 = api.create_taxonomy(name="Taxonomy 3", orgs=[self.orgA])
api.add_tag_to_taxonomy(
taxonomy=self.taxonomy_3,
tag="Tag 3.1",
)
# Tag blocks
api.tag_object(
object_id="block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential",
taxonomy=self.taxonomy_1,
tags=['Tag 1.1', 'Tag 1.2'],
)
self.sequential_tags1 = api.get_object_tags(
"block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential",
taxonomy_id=self.taxonomy_1.id,
)
api.tag_object(
object_id="block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential",
taxonomy=self.taxonomy_2,
tags=['Tag 2.1'],
)
self.sequential_tags2 = api.get_object_tags(
"block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential",
taxonomy_id=self.taxonomy_2.id,
)
api.tag_object(
object_id="block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1",
taxonomy=self.taxonomy_2,
tags=['Tag 2.2'],
)
self.vertical1_tags = api.get_object_tags(
"block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1"
)
api.tag_object(
object_id="block-v1:orgA+test_course+test_run+type@html+block@test_html",
taxonomy=self.taxonomy_2,
tags=['Tag 2.1'],
)
self.html_tags = api.get_object_tags("block-v1:orgA+test_course+test_run+type@html+block@test_html")
# Create "deleted" object tags, which will be omitted from the results.
for object_id in (
"course-v1:orgA+test_course+test_run",
"block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential",
"block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1",
"block-v1:orgA+test_course+test_run+type@html+block@test_html",
):
ObjectTag.objects.create(
object_id=str(object_id),
taxonomy=None,
tag=None,
_value="deleted tag",
_name="deleted taxonomy",
)
self.expected_course_objecttags = {
"course-v1:orgA+test_course+test_run": {
self.taxonomy_1.id: [tag.value for tag in self.course_tags],
},
"block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential": {
self.taxonomy_1.id: [tag.value for tag in self.sequential_tags1],
self.taxonomy_2.id: [tag.value for tag in self.sequential_tags2],
},
"block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1": {
self.taxonomy_2.id: [tag.value for tag in self.vertical1_tags],
},
"block-v1:orgA+test_course+test_run+type@html+block@test_html": {
self.taxonomy_2.id: [tag.value for tag in self.html_tags],
},
}
# Library tags and library contents need a unique block_id that is persisted along test runs
self.block_suffix = str(round(time.time() * 1000))
api.tag_object(
object_id=f"lib:orgA:lib_{self.block_suffix}",
taxonomy=self.taxonomy_2,
tags=['Tag 2.1'],
)
self.library_tags = api.get_object_tags(f"lib:orgA:lib_{self.block_suffix}")
api.tag_object(
object_id=f"lb:orgA:lib_{self.block_suffix}:problem:problem1_{self.block_suffix}",
taxonomy=self.taxonomy_1,
tags=['Tag 1.1'],
)
self.problem1_tags = api.get_object_tags(
f"lb:orgA:lib_{self.block_suffix}:problem:problem1_{self.block_suffix}"
)
api.tag_object(
object_id=f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}",
taxonomy=self.taxonomy_1,
tags=['Tag 1.2'],
)
self.library_html_tags1 = api.get_object_tags(
object_id=f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}",
taxonomy_id=self.taxonomy_1.id,
)
api.tag_object(
object_id=f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}",
taxonomy=self.taxonomy_2,
tags=['Tag 2.2'],
)
self.library_html_tags2 = api.get_object_tags(
object_id=f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}",
taxonomy_id=self.taxonomy_2.id,
)
# Create "deleted" object tags, which will be omitted from the results.
for object_id in (
f"lib:orgA:lib_{self.block_suffix}",
f"lb:orgA:lib_{self.block_suffix}:problem:problem1_{self.block_suffix}",
f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}",
):
ObjectTag.objects.create(
object_id=object_id,
taxonomy=None,
tag=None,
_value="deleted tag",
_name="deleted taxonomy",
)
self.expected_library_objecttags = {
f"lib:orgA:lib_{self.block_suffix}": {
self.taxonomy_2.id: [tag.value for tag in self.library_tags],
},
f"lb:orgA:lib_{self.block_suffix}:problem:problem1_{self.block_suffix}": {
self.taxonomy_1.id: [tag.value for tag in self.problem1_tags],
},
f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}": {
self.taxonomy_1.id: [tag.value for tag in self.library_html_tags1],
self.taxonomy_2.id: [tag.value for tag in self.library_html_tags2],
},
}
class TestAPIObjectTags(TestGetAllObjectTagsMixin, TestCase):
"""
Tests object tag API functions.
@@ -442,6 +266,53 @@ class TestAPIObjectTags(TestGetAllObjectTagsMixin, TestCase):
self.taxonomy_2.id: self.taxonomy_2,
}
def test_get_course_object_tags_with_add_tags(self):
"""
This test checks for an issue in get_all_object_tags:
If new tags are added to those already added previously,
the previous tags are lost.
This happens because the new tags will overwrite the old ones
in the result.
"""
# Tag in a new taxonomy
ObjectTag.objects.create(
object_id="block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1",
taxonomy=self.taxonomy_1,
tag=self.tag_1_1,
)
# Tag in a already tagged taxonomy
ObjectTag.objects.create(
object_id="block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1",
taxonomy=self.taxonomy_2,
tag=self.tag_2_1,
)
with self.assertNumQueries(1):
object_tags, taxonomies = api.get_all_object_tags(
CourseKey.from_string("course-v1:orgA+test_course+test_run")
)
vertical1_tags = api.get_object_tags(
"block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1",
taxonomy_id=self.taxonomy_1.id,
)
vertical2_tags = api.get_object_tags(
"block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1",
taxonomy_id=self.taxonomy_2.id,
)
# Add new object tags to the expected result
self.expected_course_objecttags["block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1"] = {
self.taxonomy_1.id: [tag.value for tag in vertical1_tags],
self.taxonomy_2.id: [tag.value for tag in vertical2_tags],
}
assert object_tags == self.expected_course_objecttags
assert taxonomies == {
self.taxonomy_1.id: self.taxonomy_1,
self.taxonomy_2.id: self.taxonomy_2,
}
def test_get_library_object_tags(self):
"""
Test the get_all_object_tags function using a library
@@ -476,7 +347,7 @@ class TestAPIObjectTags(TestGetAllObjectTagsMixin, TestCase):
assert len(dst_tags) == len(expected_tags)
for idx, src_tag in enumerate(expected_tags):
dst_tag = dst_tags[idx]
assert src_tag.name == dst_tag.name
assert src_tag.export_id == dst_tag.export_id
assert src_tag.value == dst_tag.value
def test_copy_object_tags(self):
@@ -508,3 +379,104 @@ class TestAPIObjectTags(TestGetAllObjectTagsMixin, TestCase):
expected_tags = list(self.sequential_tags1) + list(self.sequential_tags2)
with self.assertNumQueries(31): # TODO why so high?
self._test_copy_object_tags(src_key, dst_key, expected_tags)
class TestExportImportTags(TaggedCourseMixin):
"""
Tests for export/import functions
"""
def _create_csv_file(self, content):
"""
Create a csv file and returns the path and name
"""
file_dir_name = tempfile.mkdtemp()
file_name = f'{file_dir_name}/tags.csv'
with open(file_name, 'w') as csv_file:
csv_file.write(content)
return file_name
def test_generate_csv_rows(self) -> None:
buffer = io.StringIO()
list(api.generate_csv_rows(str(self.course.id), buffer))
buffer.seek(0)
csv_content = buffer.getvalue()
assert csv_content == self.expected_csv
def test_export_tags_in_csv_file(self) -> None:
file_dir_name = tempfile.mkdtemp()
file_dir = OSFS(file_dir_name)
file_name = 'tags.csv'
api.export_tags_in_csv_file(str(self.course.id), file_dir, file_name)
file_path = os.path.join(file_dir_name, file_name)
self.assertTrue(os.path.exists(file_path))
with open(file_path, 'r') as f:
content = f.read()
cleaned_content = content.replace('\r\n', '\n')
cleaned_expected_csv = self.expected_csv.replace('\r\n', '\n')
self.assertEqual(cleaned_content, cleaned_expected_csv)
def test_import_tags_invalid_format(self) -> None:
csv_path = self._create_csv_file('invalid format, Invalid\r\ntest1, test2')
with self.assertRaises(ValueError) as exc:
api.import_course_tags_from_csv(csv_path, self.course.id)
assert "Invalid format of csv in" in str(exc.exception)
def test_import_tags_valid_taxonomy_and_tags(self) -> None:
csv_path = self._create_csv_file(
'"Name","Type","ID","1-taxonomy-1","2-taxonomy-2"\r\n'
'"Test Course","course","course-v1:orgA+test_course+test_run","Tag 1.1",""\r\n'
)
api.import_course_tags_from_csv(csv_path, self.course.id)
object_tags = list(api.get_object_tags(self.course.id))
assert len(object_tags) == 1
object_tag = object_tags[0]
assert object_tag.tag == self.tag_1_1
assert object_tag.taxonomy == self.taxonomy_1
def test_import_tags_invalid_tag(self) -> None:
csv_path = self._create_csv_file(
'"Name","Type","ID","1-taxonomy-1","2-taxonomy-2"\r\n'
'"Test Course","course","course-v1:orgA+test_course+test_run","Tag 1.11",""\r\n'
)
api.import_course_tags_from_csv(csv_path, self.course.id)
object_tags = list(api.get_object_tags(self.course.id))
assert len(object_tags) == 0
object_tags = list(api.get_object_tags(
self.course.id,
include_deleted=True,
))
assert len(object_tags) == 1
object_tag = object_tags[0]
assert object_tag.tag is None
assert object_tag.value == 'Tag 1.11'
assert object_tag.taxonomy == self.taxonomy_1
def test_import_tags_invalid_taxonomy(self) -> None:
csv_path = self._create_csv_file(
'"Name","Type","ID","1-taxonomy-1-1"\r\n'
'"Test Course","course","course-v1:orgA+test_course+test_run","Tag 1.11"\r\n'
)
api.import_course_tags_from_csv(csv_path, self.course.id)
object_tags = list(api.get_object_tags(self.course.id))
assert len(object_tags) == 0
object_tags = list(api.get_object_tags(
self.course.id,
include_deleted=True,
))
assert len(object_tags) == 1
object_tag = object_tags[0]
assert object_tag.tag is None
assert object_tag.value == 'Tag 1.11'
assert object_tag.taxonomy is None
assert object_tag.export_id == '1-taxonomy-1-1'

198
openedx/core/djangoapps/content_tagging/rest_api/v1/tests/test_objecttag_export_helpers.py → openedx/core/djangoapps/content_tagging/tests/test_objecttag_export_helpers.py
View File

@@ -1,15 +1,196 @@
"""
Test the objecttag_export_helpers module
"""
import time
from unittest.mock import patch
from openedx.core.djangoapps.content_libraries import api as library_api
from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase
from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory
from .... import api
from ....tests.test_api import TestGetAllObjectTagsMixin
from ..objecttag_export_helpers import TaggedContent, build_object_tree_with_objecttags, iterate_with_level
from .. import api
from ..helpers.objecttag_export_helpers import TaggedContent, build_object_tree_with_objecttags, iterate_with_level
from openedx_tagging.core.tagging.models import ObjectTag
from organizations.models import Organization
class TestGetAllObjectTagsMixin:
"""
Set up data to test get_all_object_tags functions
"""
def setUp(self):
super().setUp()
self.taxonomy_1 = api.create_taxonomy(name="Taxonomy 1")
api.set_taxonomy_orgs(self.taxonomy_1, all_orgs=True)
self.tag_1_1 = api.add_tag_to_taxonomy(
taxonomy=self.taxonomy_1,
tag="Tag 1.1",
)
self.tag_1_2 = api.add_tag_to_taxonomy(
taxonomy=self.taxonomy_1,
tag="Tag 1.2",
)
self.taxonomy_2 = api.create_taxonomy(name="Taxonomy 2")
api.set_taxonomy_orgs(self.taxonomy_2, all_orgs=True)
self.tag_2_1 = api.add_tag_to_taxonomy(
taxonomy=self.taxonomy_2,
tag="Tag 2.1",
)
self.tag_2_2 = api.add_tag_to_taxonomy(
taxonomy=self.taxonomy_2,
tag="Tag 2.2",
)
api.tag_object(
object_id="course-v1:orgA+test_course+test_run",
taxonomy=self.taxonomy_1,
tags=['Tag 1.1'],
)
self.course_tags = api.get_object_tags("course-v1:orgA+test_course+test_run")
self.orgA = Organization.objects.create(name="Organization A", short_name="orgA")
self.orgB = Organization.objects.create(name="Organization B", short_name="orgB")
self.taxonomy_3 = api.create_taxonomy(name="Taxonomy 3", orgs=[self.orgA])
api.add_tag_to_taxonomy(
taxonomy=self.taxonomy_3,
tag="Tag 3.1",
)
# Tag blocks
api.tag_object(
object_id="block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential",
taxonomy=self.taxonomy_1,
tags=['Tag 1.1', 'Tag 1.2'],
)
self.sequential_tags1 = api.get_object_tags(
"block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential",
taxonomy_id=self.taxonomy_1.id,
)
api.tag_object(
object_id="block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential",
taxonomy=self.taxonomy_2,
tags=['Tag 2.1'],
)
self.sequential_tags2 = api.get_object_tags(
"block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential",
taxonomy_id=self.taxonomy_2.id,
)
api.tag_object(
object_id="block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1",
taxonomy=self.taxonomy_2,
tags=['Tag 2.2'],
)
self.vertical1_tags = api.get_object_tags(
"block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1"
)
api.tag_object(
object_id="block-v1:orgA+test_course+test_run+type@html+block@test_html",
taxonomy=self.taxonomy_2,
tags=['Tag 2.1'],
)
self.html_tags = api.get_object_tags("block-v1:orgA+test_course+test_run+type@html+block@test_html")
# Create "deleted" object tags, which will be omitted from the results.
for object_id in (
"course-v1:orgA+test_course+test_run",
"block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential",
"block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1",
"block-v1:orgA+test_course+test_run+type@html+block@test_html",
):
ObjectTag.objects.create(
object_id=str(object_id),
taxonomy=None,
tag=None,
_value="deleted tag",
_export_id="deleted_taxonomy",
)
self.expected_course_objecttags = {
"course-v1:orgA+test_course+test_run": {
self.taxonomy_1.id: [tag.value for tag in self.course_tags],
},
"block-v1:orgA+test_course+test_run+type@sequential+block@test_sequential": {
self.taxonomy_1.id: [tag.value for tag in self.sequential_tags1],
self.taxonomy_2.id: [tag.value for tag in self.sequential_tags2],
},
"block-v1:orgA+test_course+test_run+type@vertical+block@test_vertical1": {
self.taxonomy_2.id: [tag.value for tag in self.vertical1_tags],
},
"block-v1:orgA+test_course+test_run+type@html+block@test_html": {
self.taxonomy_2.id: [tag.value for tag in self.html_tags],
},
}
# Library tags and library contents need a unique block_id that is persisted along test runs
self.block_suffix = str(round(time.time() * 1000))
api.tag_object(
object_id=f"lib:orgA:lib_{self.block_suffix}",
taxonomy=self.taxonomy_2,
tags=['Tag 2.1'],
)
self.library_tags = api.get_object_tags(f"lib:orgA:lib_{self.block_suffix}")
api.tag_object(
object_id=f"lb:orgA:lib_{self.block_suffix}:problem:problem1_{self.block_suffix}",
taxonomy=self.taxonomy_1,
tags=['Tag 1.1'],
)
self.problem1_tags = api.get_object_tags(
f"lb:orgA:lib_{self.block_suffix}:problem:problem1_{self.block_suffix}"
)
api.tag_object(
object_id=f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}",
taxonomy=self.taxonomy_1,
tags=['Tag 1.2'],
)
self.library_html_tags1 = api.get_object_tags(
object_id=f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}",
taxonomy_id=self.taxonomy_1.id,
)
api.tag_object(
object_id=f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}",
taxonomy=self.taxonomy_2,
tags=['Tag 2.2'],
)
self.library_html_tags2 = api.get_object_tags(
object_id=f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}",
taxonomy_id=self.taxonomy_2.id,
)
# Create "deleted" object tags, which will be omitted from the results.
for object_id in (
f"lib:orgA:lib_{self.block_suffix}",
f"lb:orgA:lib_{self.block_suffix}:problem:problem1_{self.block_suffix}",
f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}",
):
ObjectTag.objects.create(
object_id=object_id,
taxonomy=None,
tag=None,
_value="deleted tag",
_export_id="deleted_taxonomy",
)
self.expected_library_objecttags = {
f"lib:orgA:lib_{self.block_suffix}": {
self.taxonomy_2.id: [tag.value for tag in self.library_tags],
},
f"lb:orgA:lib_{self.block_suffix}:problem:problem1_{self.block_suffix}": {
self.taxonomy_1.id: [tag.value for tag in self.problem1_tags],
},
f"lb:orgA:lib_{self.block_suffix}:html:html_{self.block_suffix}": {
self.taxonomy_1.id: [tag.value for tag in self.library_html_tags1],
self.taxonomy_2.id: [tag.value for tag in self.library_html_tags2],
},
}
class TaggedCourseMixin(TestGetAllObjectTagsMixin, ModuleStoreTestCase): # type: ignore[misc]
@@ -231,6 +412,17 @@ class TaggedCourseMixin(TestGetAllObjectTagsMixin, ModuleStoreTestCase): # type
(tagged_library_html, 1),
]
self.expected_csv = (
'"Name","Type","ID","1-taxonomy-1","2-taxonomy-2"\r\n'
'"Test Course","course","course-v1:orgA+test_course+test_run","Tag 1.1",""\r\n'
'" test sequential","sequential","test_sequential","Tag 1.1; Tag 1.2","Tag 2.1"\r\n'
'" test vertical1","vertical","test_vertical1","","Tag 2.2"\r\n'
'" test vertical2","vertical","test_vertical2","",""\r\n'
'" test html","html","test_html","","Tag 2.1"\r\n'
'" untagged sequential","sequential","untagged_sequential","",""\r\n'
'" untagged vertical","vertical","untagged_vertical","",""\r\n'
)
class TestContentTagChildrenExport(TaggedCourseMixin): # type: ignore[misc]
"""

2
openedx/core/djangoapps/content_tagging/tests/test_tasks.py
View File

@@ -38,6 +38,7 @@ class LanguageTaxonomyTestMixin:
create the taxonomy, simulating the effect of the following migrations:
1. openedx_tagging.core.tagging.migrations.0012_language_taxonomy
2. content_tagging.migrations.0007_system_defined_org_2
3. openedx_tagging.core.tagging.migrations.0015_taxonomy_export_id
"""
super().setUp()
Taxonomy.objects.get_or_create(id=-1, defaults={
@@ -47,6 +48,7 @@ class LanguageTaxonomyTestMixin:
"allow_multiple": False,
"allow_free_text": False,
"visible_to_authors": True,
"export_id": "-1_languages",
"_taxonomy_class": "openedx_tagging.core.tagging.models.system_defined.LanguageTaxonomy",
})
TaxonomyOrg.objects.get_or_create(taxonomy_id=-1, defaults={"org": None})

2
requirements/constraints.txt
View File

@@ -108,7 +108,7 @@ libsass==0.10.0
click==8.1.6
# pinning this version to avoid updates while the library is being developed
openedx-learning==0.6.3
openedx-learning==0.8.0
# Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise.
openai<=0.28.1

2
requirements/edx/base.txt
View File

@@ -794,7 +794,7 @@ openedx-filters==1.6.0
# via
# -r requirements/edx/kernel.in
# lti-consumer-xblock
openedx-learning==0.6.3
openedx-learning==0.8.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/kernel.in

2
requirements/edx/development.txt
View File

@@ -1319,7 +1319,7 @@ openedx-filters==1.6.0
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
# lti-consumer-xblock
openedx-learning==0.6.3
openedx-learning==0.8.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/doc.txt

2
requirements/edx/doc.txt
View File

@@ -932,7 +932,7 @@ openedx-filters==1.6.0
# via
# -r requirements/edx/base.txt
# lti-consumer-xblock
openedx-learning==0.6.3
openedx-learning==0.8.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt

2
requirements/edx/testing.txt
View File

@@ -987,7 +987,7 @@ openedx-filters==1.6.0
# via
# -r requirements/edx/base.txt
# lti-consumer-xblock
openedx-learning==0.6.3
openedx-learning==0.8.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt

13
xmodule/modulestore/xml_exporter.py
View File

@@ -12,6 +12,10 @@ import lxml.etree
from fs.osfs import OSFS
from opaque_keys.edx.locator import CourseLocator, LibraryLocator
from xblock.fields import Reference, ReferenceList, ReferenceValueDict, Scope
from openedx.core.djangoapps.content_tagging.api import (
export_tags_in_csv_file,
get_object_tag_counts
)
from xmodule.assetstore import AssetMetadata
from xmodule.contentstore.content import StaticContent
@@ -282,6 +286,15 @@ class CourseExportManager(ExportManager):
_export_drafts(self.modulestore, self.courselike_key, export_fs, xml_centric_courselike_key)
courselike_key_str = str(self.courselike_key)
block_id_pattern = f"{courselike_key_str.replace('course-v1:', 'block-v1:', 1)}*"
tags_count = get_object_tag_counts(block_id_pattern)
course_tags_count = get_object_tag_counts(courselike_key_str)
if tags_count or course_tags_count:
export_tags_in_csv_file(courselike_key_str, export_fs, 'tags.csv')
class LibraryExportManager(ExportManager):
"""

29
xmodule/modulestore/xml_importer.py
View File

@@ -52,6 +52,7 @@ from xmodule.modulestore.xml import ImportSystem, LibraryXMLModuleStore, XMLModu
from xmodule.tabs import CourseTabList
from xmodule.util.misc import escape_invalid_characters
from xmodule.x_module import XModuleMixin
from openedx.core.djangoapps.content_tagging.api import import_course_tags_from_csv
from .inheritance import own_metadata
from .store_utilities import rewrite_nonportable_content_links
@@ -476,6 +477,13 @@ class ImportManager:
"""
raise NotImplementedError
@abstractmethod
def import_tags(self, data_path, dest_id):
"""
To be overloaded with a method that adds tags to already imported blocks
"""
raise NotImplementedError
def recursive_build(self, source_courselike, courselike, courselike_key, dest_id):
"""
Recursively imports all child blocks from the temporary modulestore into the
@@ -574,6 +582,13 @@ class ImportManager:
# Import all draft items into the courselike.
courselike = self.import_drafts(courselike, courselike_key, data_path, dest_id)
with self.store.bulk_operations(dest_id):
try:
self.import_tags(data_path, dest_id)
except FileNotFoundError:
logging.info(f'Course import {dest_id}: No tags.csv file present.')
except ValueError as e:
logging.info(f'Course import {dest_id}: {str(e)}')
yield courselike
@@ -695,6 +710,13 @@ class CourseImportManager(ImportManager):
# Fetch the course to return the most recent course version.
return self.store.get_course(courselike.id.replace(branch=None, version_guid=None))
def import_tags(self, data_path, dest_id):
"""
Imports tags into course blocks.
"""
csv_path = path(data_path) / 'tags.csv'
import_course_tags_from_csv(csv_path, dest_id)
class LibraryImportManager(ImportManager):
"""
@@ -766,6 +788,13 @@ class LibraryImportManager(ImportManager):
"""
return courselike
def import_tags(self, data_path, dest_id):
"""
Imports tags into library blocks
"""
# We don't support tags in v1 libraries, and v2 libraries don't have
# an import/export format defined yet. No action needed here for now.
def import_course_from_xml(*args, **kwargs):
"""