Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Remove unneeded filter from static asset copy/paste code. (#32660)

Browse Source
This commit is contained in:
Braden MacDonald
2023-07-07 09:47:06 -07:00
committed by GitHub
parent 6d6893ca56
commit e705820d0d
3 changed files with 12 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
cms/envs/common.py
View File

@@ -2728,11 +2728,3 @@ BRAZE_COURSE_ENROLLMENT_CANVAS_ID = ''
DISCUSSIONS_INCONTEXT_FEEDBACK_URL = ''
DISCUSSIONS_INCONTEXT_LEARNMORE_URL = ''
OPEN_EDX_FILTERS_CONFIG = {
"org.openedx.content_authoring.staged_content.static_filter_source.v1": {
"pipeline": [
"openedx.core.djangoapps.content_staging.filters.IgnoreLargeFiles",
]
}
}

55
openedx/core/djangoapps/content_staging/filters.py
View File

@@ -1,55 +0,0 @@
"""
Filters that affect the behavior of staged content (and the clipboard)
"""
# pylint: disable=unused-argument
from __future__ import annotations
from attrs import asdict
from openedx_filters import PipelineStep
from openedx_filters.tooling import OpenEdxPublicFilter
from .data import StagedContentFileData
from .models import StagedContent
class StagingStaticAssetFilter(OpenEdxPublicFilter):
"""
A filter used to determine which static assets associate with an XBlock(s)
should be staged in the StagedContent app (e.g. the clipboard).
This API is considered BETA. Once it is stable, this definition should be moved into openedx_filters.
"""
filter_type = "org.openedx.content_authoring.staged_content.static_filter_source.v1"
@classmethod
def run_filter(cls, staged_content: StagedContent, file_datas: list[StagedContentFileData]):
"""
Run this filter, which requires the following arguments:
staged_content (StagedContent): details of the content being staged, as saved to the DB.
file_datas (list[StagedContentFileData]): details of the files being staged
"""
data = super().run_pipeline(staged_content=staged_content, file_datas=file_datas)
return data.get("file_datas")
class IgnoreLargeFiles(PipelineStep):
"""
Don't copy files over 10MB into the clipboard
"""
# pylint: disable=arguments-differ
def run_filter(self, staged_content: StagedContent, file_datas: list[StagedContentFileData]):
"""
Filter the list of file_datas to remove any large files
"""
limit = 10 * 1024 * 1024
def remove_large_data(fd: StagedContentFileData):
""" Remove 'data' from the immutable StagedContentFileData object, if it's above the size limit """
if fd.data and len(fd.data) > limit:
# these data objects are immutable so make a copy with data=None:
return StagedContentFileData(**{**asdict(fd), "data": None})
return fd
return {"file_datas": [remove_large_data(fd) for fd in file_datas]}

32
openedx/core/djangoapps/content_staging/views.py
View File

@@ -27,8 +27,7 @@ from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from .data import CLIPBOARD_PURPOSE, StagedContentFileData, StagedContentStatus
from .filters import StagingStaticAssetFilter
from .data import CLIPBOARD_PURPOSE, StagedContentStatus
from .models import StagedContent, StagedContentFile, UserClipboard
from .serializers import UserClipboardSerializer, PostToClipboardSerializer
from .tasks import delete_expired_clipboards
@@ -181,7 +180,6 @@ class ClipboardEndpoint(APIView):
"""
Helper method for "post to clipboard" API endpoint. This deals with copying static files into the clipboard.
"""
files_to_save: list[StagedContentFileData] = []
for f in static_files:
source_key = (
StaticContent.get_asset_key_from_path(usage_key.context_key, f.url)
@@ -201,30 +199,24 @@ class ClipboardEndpoint(APIView):
else:
continue # Skip this file - we don't need a reference to a non-existent file.
# Load the data:
entry = StagedContentFileData(
filename=f.name,
data=content,
source_key=source_key,
md5_hash=md5_hash,
)
files_to_save.append(entry)
# Because we store clipboard files on S3, uploading really large files will be too slow. And it's wasted if
# the copy-paste is just happening within a single course. So for files > 10MB, users must copy the files
# manually. In the future we can consider removing this or making it configurable or filterable.
limit = 10 * 1024 * 1024
if content and len(content) > limit:
content = None
# run filters on files_to_save. e.g. remove large files
files_to_save = StagingStaticAssetFilter.run_filter(staged_content=staged_content, file_datas=files_to_save)
for f in files_to_save:
try:
StagedContentFile.objects.create(
for_content=staged_content,
filename=f.filename,
filename=f.name,
# In some cases (e.g. really large files), we don't store the data here but we still keep track of
# the metadata. You can still use the metadata to determine if the file is already present or not,
# and then either inform the user or find another way to import the file (e.g. if the file still
# exists in the "Files & Uploads" contentstore of the source course, based on source_key_str).
data_file=ContentFile(content=f.data, name=f.filename) if f.data else None,
source_key_str=str(f.source_key) if f.source_key else "",
md5_hash=f.md5_hash or "",
data_file=ContentFile(content=content, name=f.name) if content else None,
source_key_str=str(source_key) if source_key else "",
md5_hash=md5_hash,
)
except Exception: # pylint: disable=broad-except
log.exception(f"Unable to copy static file {f.filename} to clipboard for component {usage_key}")
log.exception(f"Unable to copy static file {f.name} to clipboard for component {usage_key}")