diff --git a/cms/envs/common.py b/cms/envs/common.py index 3d7683ae84..bfecbffcaa 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -111,6 +111,10 @@ from lms.envs.common import ( # Enterprise service settings ENTERPRISE_CATALOG_INTERNAL_ROOT_URL, + # Blockstore + BLOCKSTORE_USE_BLOCKSTORE_APP_API, + BUNDLE_ASSET_STORAGE_SETTINGS, + # Methods to derive settings _make_mako_template_dirs, _make_locale_paths, @@ -1746,6 +1750,9 @@ INSTALLED_APPS = [ # For edx ace template tags 'edx_ace', + + # Blockstore + 'blockstore.apps.bundles', ] @@ -2102,6 +2109,7 @@ ENABLE_COMPREHENSIVE_THEMING = False DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', + 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', ] ############################ Cache Configuration ############################### diff --git a/lms/envs/common.py b/lms/envs/common.py index fd5d0d0048..74bfb6e979 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1085,6 +1085,7 @@ STATUS_MESSAGE_PATH = ENV_ROOT / "status_message.json" DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', + 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', 'edx_django_utils.db.read_replica.ReadReplicaRouter', ] @@ -3243,6 +3244,9 @@ INSTALLED_APPS = [ # For save for later 'lms.djangoapps.save_for_later', + + # Blockstore + 'blockstore.apps.bundles', ] ######################### CSRF ######################################### @@ -4970,6 +4974,10 @@ MAILCHIMP_NEW_USER_LIST_ID = "" BLOCKSTORE_PUBLIC_URL_ROOT = 'http://localhost:18250' BLOCKSTORE_API_URL = 'http://localhost:18250/api/v1/' +# Disable the Blockstore app API by default. +# See openedx.core.lib.blockstore_api.config for details. +BLOCKSTORE_USE_BLOCKSTORE_APP_API = False + # .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE # .. setting_default: default # .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. @@ -4985,6 +4993,40 @@ XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' # configured to expire after one hour. BLOCKSTORE_BUNDLE_CACHE_TIMEOUT = 3000 +# .. setting_name: BUNDLE_ASSET_URL_STORAGE_KEY +# .. setting_default: None +# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_SECRET` is +# set, and `boto3` is installed, this is used as an AWS IAM access key for +# generating signed, read-only URLs for blockstore assets stored in S3. +# Otherwise, URLs are generated based on the default storage configuration. +# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. +BUNDLE_ASSET_URL_STORAGE_KEY = None + +# .. setting_name: BUNDLE_ASSET_URL_STORAGE_SECRET +# .. setting_default: None +# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is +# set, and `boto3` is installed, this is used as an AWS IAM secret key for +# generating signed, read-only URLs for blockstore assets stored in S3. +# Otherwise, URLs are generated based on the default storage configuration. +# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. +BUNDLE_ASSET_URL_STORAGE_SECRET = None + +# .. setting_name: BUNDLE_ASSET_STORAGE_SETTINGS +# .. setting_default: dict, appropriate for file system storage. +# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is +# set, and `boto3` is installed, this provides the bucket name and location for blockstore assets stored in S3. +# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. +BUNDLE_ASSET_STORAGE_SETTINGS = dict( + # Backend storage + # STORAGE_CLASS='storages.backends.s3boto.S3BotoStorage', + # STORAGE_KWARGS=dict(bucket='bundle-asset-bucket', location='/path-to-bundles/'), + STORAGE_CLASS='django.core.files.storage.FileSystemStorage', + STORAGE_KWARGS=dict( + location=MEDIA_ROOT, + base_url=MEDIA_URL, + ), +) + ######################### MICROSITE ############################### MICROSITE_ROOT_DIR = '/edx/app/edxapp/edx-microsite' MICROSITE_CONFIGURATION = {} diff --git a/openedx/core/djangoapps/content_libraries/library_bundle.py b/openedx/core/djangoapps/content_libraries/library_bundle.py index b2d3c34efa..ed03b18012 100644 --- a/openedx/core/djangoapps/content_libraries/library_bundle.py +++ b/openedx/core/djangoapps/content_libraries/library_bundle.py @@ -2,7 +2,6 @@ Helper code for working with Blockstore bundles that contain OLX """ -import dateutil.parser import logging # lint-amnesty, pylint: disable=wrong-import-order from functools import lru_cache # lint-amnesty, pylint: disable=wrong-import-order @@ -347,12 +346,17 @@ class LibraryBundle: problem/quiz1/definition.xml problem/quiz1/static/image1.png Then this will return - [BundleFile(path="image1.png", size, url, hash_digest)] + [BundleFileData(path="image1.png", size, url, hash_digest)] """ path_prefix = self.get_static_prefix_for_definition(definition_key) path_prefix_len = len(path_prefix) return [ - blockstore_api.BundleFile(path=f.path[path_prefix_len:], size=f.size, url=f.url, hash_digest=f.hash_digest) + blockstore_api.BundleFileData( + path=f.path[path_prefix_len:], + size=f.size, + url=f.url, + hash_digest=f.hash_digest, + ) for f in get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) if f.path.startswith(path_prefix) ] @@ -369,8 +373,7 @@ class LibraryBundle: version = get_bundle_version_number(self.bundle_uuid) if version == 0: return None - created_at_str = blockstore_api.get_bundle_version(self.bundle_uuid, version)['snapshot']['created_at'] - last_published_time = dateutil.parser.parse(created_at_str) + last_published_time = blockstore_api.get_bundle_version(self.bundle_uuid, version).created_at self.cache.set(cache_key, last_published_time) return last_published_time diff --git a/openedx/core/lib/blockstore_api/__init__.py b/openedx/core/lib/blockstore_api/__init__.py index 483e97c2eb..50b352578c 100644 --- a/openedx/core/lib/blockstore_api/__init__.py +++ b/openedx/core/lib/blockstore_api/__init__.py @@ -5,15 +5,16 @@ This API does not do any caching; consider using BundleCache or (in openedx.core.djangolib.blockstore_cache) together with these API methods for improved performance. """ -from .models import ( - Collection, - Bundle, - Draft, - BundleFile, - DraftFile, - LinkReference, - LinkDetails, - DraftLinkDetails, +from blockstore.apps.api.data import ( + BundleFileData, +) +from blockstore.apps.api.exceptions import ( + CollectionNotFound, + BundleNotFound, + DraftNotFound, + BundleVersionNotFound, + BundleFileNotFound, + BundleStorageError, ) from .methods import ( # Collections: @@ -47,11 +48,3 @@ from .methods import ( # Misc: force_browser_url, ) -from .exceptions import ( - BlockstoreException, - CollectionNotFound, - BundleNotFound, - DraftNotFound, - BundleFileNotFound, - BundleStorageError, -) diff --git a/openedx/core/lib/blockstore_api/config/__init__.py b/openedx/core/lib/blockstore_api/config/__init__.py new file mode 100644 index 0000000000..2f9de4c0fc --- /dev/null +++ b/openedx/core/lib/blockstore_api/config/__init__.py @@ -0,0 +1,13 @@ +""" +Helper method to indicate when the blockstore app API is enabled. +""" +from django.conf import settings +from .waffle import BLOCKSTORE_USE_BLOCKSTORE_APP_API + + +def use_blockstore_app(): + """ + Use the Blockstore app API if the settings say to (e.g. in test) + or if the waffle switch is enabled. + """ + return settings.BLOCKSTORE_USE_BLOCKSTORE_APP_API or BLOCKSTORE_USE_BLOCKSTORE_APP_API.is_enabled() diff --git a/openedx/core/lib/blockstore_api/config/waffle.py b/openedx/core/lib/blockstore_api/config/waffle.py new file mode 100644 index 0000000000..ebbacb7f59 --- /dev/null +++ b/openedx/core/lib/blockstore_api/config/waffle.py @@ -0,0 +1,20 @@ +""" +Toggles for blockstore. +""" + +from edx_toggles.toggles import WaffleSwitch + +# .. toggle_name: blockstore.use_blockstore_app_api +# .. toggle_implementation: WaffleSwitch +# .. toggle_default: False +# .. toggle_description: Enable to use the installed blockstore app's Python API directly instead of the +# external blockstore service REST API. +# The blockstore REST API is used by default. +# .. toggle_use_cases: temporary, open_edx +# .. toggle_creation_date: 2022-01-13 +# .. toggle_target_removal_date: None +# .. toggle_tickets: TNL-8705, BD-14 +# .. toggle_warnings: This temporary feature toggle does not have a target removal date. +BLOCKSTORE_USE_BLOCKSTORE_APP_API = WaffleSwitch( + 'blockstore.use_blockstore_app_api', __name__ +) diff --git a/openedx/core/lib/blockstore_api/db_routers.py b/openedx/core/lib/blockstore_api/db_routers.py new file mode 100644 index 0000000000..fd0ff50c95 --- /dev/null +++ b/openedx/core/lib/blockstore_api/db_routers.py @@ -0,0 +1,60 @@ +""" +Blockstore database router. + +Blockstore started life as an IDA, but is now a Django app plugin within edx-platform. +This router exists to smooth blockstore's transition into edxapp. +""" +from django.conf import settings + + +class BlockstoreRouter: + """ + A Database Router that uses the ``blockstore`` database, if it's configured in settings. + """ + ROUTE_APP_LABELS = {'bundles'} + DATABASE_NAME = 'blockstore' + + def _use_blockstore(self, model): + """ + Return True if the given model should use the blockstore database. + + Ensures that a ``blockstore`` database is configured, and checks the ``model``'s app label. + """ + return (self.DATABASE_NAME in settings.DATABASES) and (model._meta.app_label in self.ROUTE_APP_LABELS) + + def db_for_read(self, model, **hints): # pylint: disable=unused-argument + """ + Use the BlockstoreRouter.DATABASE_NAME when reading blockstore app tables. + """ + if self._use_blockstore(model): + return self.DATABASE_NAME + return None + + def db_for_write(self, model, **hints): # pylint: disable=unused-argument + """ + Use the BlockstoreRouter.DATABASE_NAME when writing to blockstore app tables. + """ + if self._use_blockstore(model): + return self.DATABASE_NAME + return None + + def allow_relation(self, obj1, obj2, **hints): # pylint: disable=unused-argument + """ + Allow relations if both objects are blockstore app models. + """ + if self._use_blockstore(obj1) and self._use_blockstore(obj2): + return True + return None + + def allow_migrate(self, db, app_label, model_name=None, **hints): # pylint: disable=unused-argument + """ + Ensure the blockstore tables only appear in the blockstore database. + """ + if model_name is not None: + model = hints.get('model') + if model is not None and self._use_blockstore(model): + return db == self.DATABASE_NAME + if db == self.DATABASE_NAME: + return False + + return None diff --git a/openedx/core/lib/blockstore_api/exceptions.py b/openedx/core/lib/blockstore_api/exceptions.py deleted file mode 100644 index b58251d31f..0000000000 --- a/openedx/core/lib/blockstore_api/exceptions.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Exceptions that may be raised by the Blockstore API -""" - - -class BlockstoreException(Exception): - pass - - -class NotFound(BlockstoreException): - pass - - -class CollectionNotFound(NotFound): - pass - - -class BundleNotFound(NotFound): - pass - - -class DraftNotFound(NotFound): - pass - - -class BundleFileNotFound(NotFound): - pass - - -class BundleStorageError(BlockstoreException): - pass diff --git a/openedx/core/lib/blockstore_api/methods.py b/openedx/core/lib/blockstore_api/methods.py index 0dacae8a04..7d7c65decd 100644 --- a/openedx/core/lib/blockstore_api/methods.py +++ b/openedx/core/lib/blockstore_api/methods.py @@ -3,6 +3,7 @@ API Client methods for working with Blockstore bundles and drafts """ import base64 +from functools import wraps from urllib.parse import urlencode from uuid import UUID @@ -11,23 +12,40 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured import requests -from .models import ( - Bundle, - Collection, - Draft, - BundleFile, - DraftFile, - LinkDetails, - LinkReference, - DraftLinkDetails, +from blockstore.apps.api.data import ( + BundleData, + CollectionData, + DraftData, + BundleVersionData, + BundleFileData, + DraftFileData, + BundleLinkData, + DraftLinkData, + Dependency, ) -from .exceptions import ( +from blockstore.apps.api.exceptions import ( NotFound, CollectionNotFound, BundleNotFound, DraftNotFound, BundleFileNotFound, ) +import blockstore.apps.api.methods as blockstore_api_methods + +from .config import use_blockstore_app + + +def toggle_blockstore_api(func): + """ + Decorator function to toggle usage of the Blockstore service + and the in-built Blockstore app dependency. + """ + @wraps(func) + def wrapper(*args, **kwargs): + if use_blockstore_app(): + return getattr(blockstore_api_methods, func.__name__)(*args, **kwargs) + return func(*args, **kwargs) + return wrapper def api_url(*path_parts): @@ -55,17 +73,17 @@ def api_request(method, url, **kwargs): def _collection_from_response(data): """ Given data about a Collection returned by any blockstore REST API, convert it to - a Collection instance. + a CollectionData instance. """ - return Collection(uuid=UUID(data['uuid']), title=data['title']) + return CollectionData(uuid=UUID(data['uuid']), title=data['title']) def _bundle_from_response(data): """ Given data about a Bundle returned by any blockstore REST API, convert it to - a Bundle instance. + a BundleData instance. """ - return Bundle( + return BundleData( uuid=UUID(data['uuid']), title=data['title'], description=data['description'], @@ -78,25 +96,51 @@ def _bundle_from_response(data): ) +def _bundle_version_from_response(data): + """ + Given data about a BundleVersion returned by any blockstore REST API, convert it to + a BundleVersionData instance. + """ + return BundleVersionData( + bundle_uuid=UUID(data['bundle_uuid']), + version=data.get('version', 0), + change_description=data['change_description'], + created_at=dateutil.parser.parse(data['snapshot']['created_at']), + files={ + path: BundleFileData(path=path, **filedata) + for path, filedata in data['snapshot']['files'].items() + }, + links={ + name: BundleLinkData( + name=name, + direct=Dependency(**link["direct"]), + indirect=[Dependency(**ind) for ind in link["indirect"]], + ) + for name, link in data['snapshot']['links'].items() + } + ) + + def _draft_from_response(data): """ Given data about a Draft returned by any blockstore REST API, convert it to - a Draft instance. + a DraftData instance. """ - return Draft( + return DraftData( uuid=UUID(data['uuid']), bundle_uuid=UUID(data['bundle_uuid']), name=data['name'], + created_at=dateutil.parser.parse(data['staged_draft']['created_at']), updated_at=dateutil.parser.parse(data['staged_draft']['updated_at']), files={ - path: DraftFile(path=path, **file) + path: DraftFileData(path=path, **file) for path, file in data['staged_draft']['files'].items() }, links={ - name: DraftLinkDetails( + name: DraftLinkData( name=name, - direct=LinkReference(**link["direct"]), - indirect=[LinkReference(**ind) for ind in link["indirect"]], + direct=Dependency(**link["direct"]), + indirect=[Dependency(**ind) for ind in link["indirect"]], modified=link["modified"], ) for name, link in data['staged_draft']['links'].items() @@ -104,6 +148,7 @@ def _draft_from_response(data): ) +@toggle_blockstore_api def get_collection(collection_uuid): """ Retrieve metadata about the specified collection @@ -118,6 +163,7 @@ def get_collection(collection_uuid): return _collection_from_response(data) +@toggle_blockstore_api def create_collection(title): """ Create a new collection. @@ -126,6 +172,7 @@ def create_collection(title): return _collection_from_response(result) +@toggle_blockstore_api def update_collection(collection_uuid, title): """ Update a collection's title @@ -136,6 +183,7 @@ def update_collection(collection_uuid, title): return _collection_from_response(result) +@toggle_blockstore_api def delete_collection(collection_uuid): """ Delete a collection @@ -144,6 +192,7 @@ def delete_collection(collection_uuid): api_request('delete', api_url('collections', str(collection_uuid))) +@toggle_blockstore_api def get_bundles(uuids=None, text_search=None): """ Get the details of all bundles @@ -159,6 +208,7 @@ def get_bundles(uuids=None, text_search=None): return [_bundle_from_response(item) for item in response] +@toggle_blockstore_api def get_bundle(bundle_uuid): """ Retrieve metadata about the specified bundle @@ -173,6 +223,7 @@ def get_bundle(bundle_uuid): return _bundle_from_response(data) +@toggle_blockstore_api def create_bundle(collection_uuid, slug, title="New Bundle", description=""): """ Create a new bundle. @@ -188,6 +239,7 @@ def create_bundle(collection_uuid, slug, title="New Bundle", description=""): return _bundle_from_response(result) +@toggle_blockstore_api def update_bundle(bundle_uuid, **fields): """ Update a bundle's title, description, slug, or collection. @@ -207,6 +259,7 @@ def update_bundle(bundle_uuid, **fields): return _bundle_from_response(result) +@toggle_blockstore_api def delete_bundle(bundle_uuid): """ Delete a bundle @@ -215,6 +268,7 @@ def delete_bundle(bundle_uuid): api_request('delete', api_url('bundles', str(bundle_uuid))) +@toggle_blockstore_api def get_draft(draft_uuid): """ Retrieve metadata about the specified draft. @@ -228,6 +282,7 @@ def get_draft(draft_uuid): return _draft_from_response(data) +@toggle_blockstore_api def get_or_create_bundle_draft(bundle_uuid, draft_name): """ Retrieve metadata about the specified draft. @@ -245,6 +300,7 @@ def get_or_create_bundle_draft(bundle_uuid, draft_name): return get_draft(UUID(response["uuid"])) +@toggle_blockstore_api def commit_draft(draft_uuid): """ Commit all of the pending changes in the draft, creating a new version of @@ -255,6 +311,7 @@ def commit_draft(draft_uuid): api_request('post', api_url('drafts', str(draft_uuid), 'commit')) +@toggle_blockstore_api def delete_draft(draft_uuid): """ Delete the specified draft, removing any staged changes/files/deletes. @@ -264,6 +321,7 @@ def delete_draft(draft_uuid): api_request('delete', api_url('drafts', str(draft_uuid))) +@toggle_blockstore_api def get_bundle_version(bundle_uuid, version_number): """ Get the details of the specified bundle version @@ -271,9 +329,10 @@ def get_bundle_version(bundle_uuid, version_number): if version_number == 0: return None version_url = api_url('bundle_versions', str(bundle_uuid) + ',' + str(version_number)) - return api_request('get', version_url) + return _bundle_version_from_response(api_request('get', version_url)) +@toggle_blockstore_api def get_bundle_version_files(bundle_uuid, version_number): """ Get a list of the files in the specified bundle version @@ -281,9 +340,10 @@ def get_bundle_version_files(bundle_uuid, version_number): if version_number == 0: return [] version_info = get_bundle_version(bundle_uuid, version_number) - return [BundleFile(path=path, **file_metadata) for path, file_metadata in version_info["snapshot"]["files"].items()] + return list(version_info.files.values()) +@toggle_blockstore_api def get_bundle_version_links(bundle_uuid, version_number): """ Get a dictionary of the links in the specified bundle version @@ -291,22 +351,16 @@ def get_bundle_version_links(bundle_uuid, version_number): if version_number == 0: return {} version_info = get_bundle_version(bundle_uuid, version_number) - return { - name: LinkDetails( - name=name, - direct=LinkReference(**link["direct"]), - indirect=[LinkReference(**ind) for ind in link["indirect"]], - ) - for name, link in version_info['snapshot']['links'].items() - } + return version_info.links +@toggle_blockstore_api def get_bundle_files_dict(bundle_uuid, use_draft=None): """ Get a dict of all the files in the specified bundle. Returns a dict where the keys are the paths (strings) and the values are - BundleFile or DraftFile tuples. + BundleFileData or DraftFileData tuples. """ bundle = get_bundle(bundle_uuid) if use_draft and use_draft in bundle.drafts: # pylint: disable=unsupported-membership-test @@ -319,6 +373,7 @@ def get_bundle_files_dict(bundle_uuid, use_draft=None): return {file_meta.path: file_meta for file_meta in get_bundle_version_files(bundle_uuid, bundle.latest_version)} +@toggle_blockstore_api def get_bundle_files(bundle_uuid, use_draft=None): """ Get an iterator over all the files in the specified bundle or draft. @@ -326,12 +381,13 @@ def get_bundle_files(bundle_uuid, use_draft=None): return get_bundle_files_dict(bundle_uuid, use_draft).values() +@toggle_blockstore_api def get_bundle_links(bundle_uuid, use_draft=None): """ Get a dict of all the links in the specified bundle. Returns a dict where the keys are the link names (strings) and the values - are LinkDetails or DraftLinkDetails tuples. + are BundleLinkData or DraftLinkData tuples. """ bundle = get_bundle(bundle_uuid) if use_draft and use_draft in bundle.drafts: # pylint: disable=unsupported-membership-test @@ -344,6 +400,7 @@ def get_bundle_links(bundle_uuid, use_draft=None): return get_bundle_version_links(bundle_uuid, bundle.latest_version) +@toggle_blockstore_api def get_bundle_file_metadata(bundle_uuid, path, use_draft=None): """ Get the metadata of the specified file. @@ -358,6 +415,7 @@ def get_bundle_file_metadata(bundle_uuid, path, use_draft=None): ) +@toggle_blockstore_api def get_bundle_file_data(bundle_uuid, path, use_draft=None): """ Read all the data in the given bundle file and return it as a @@ -370,6 +428,7 @@ def get_bundle_file_data(bundle_uuid, path, use_draft=None): return r.content +@toggle_blockstore_api def write_draft_file(draft_uuid, path, contents): """ Create or overwrite the file at 'path' in the specified draft with the given @@ -382,11 +441,12 @@ def write_draft_file(draft_uuid, path, contents): """ api_request('patch', api_url('drafts', str(draft_uuid)), json={ 'files': { - path: encode_str_for_draft(contents) if contents is not None else None, + path: _encode_str_for_draft(contents) if contents is not None else None, }, }) +@toggle_blockstore_api def set_draft_link(draft_uuid, link_name, bundle_uuid, version): """ Create or replace the link with the given name in the specified draft so @@ -405,7 +465,7 @@ def set_draft_link(draft_uuid, link_name, bundle_uuid, version): }) -def encode_str_for_draft(input_str): +def _encode_str_for_draft(input_str): """ Given a string, return UTF-8 representation that is then base64 encoded. """ @@ -416,10 +476,10 @@ def encode_str_for_draft(input_str): return base64.b64encode(binary) +@toggle_blockstore_api def force_browser_url(blockstore_file_url): """ - Ensure that the given URL Blockstore is a URL accessible from the end user's - browser. + Ensure that the given devstack URL is a URL accessible from the end user's browser. """ # Hack: on some devstacks, we must necessarily use different URLs for # accessing Blockstore file data from within and outside of docker diff --git a/openedx/core/lib/blockstore_api/models.py b/openedx/core/lib/blockstore_api/models.py deleted file mode 100644 index 8f2127ca90..0000000000 --- a/openedx/core/lib/blockstore_api/models.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Data models used for Blockstore API Client -""" - -from datetime import datetime -from uuid import UUID - -import attr - - -def _convert_to_uuid(value): - if not isinstance(value, UUID): - return UUID(value) - return value - - -@attr.s(frozen=True) -class Collection: - """ - Metadata about a blockstore collection - """ - uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - title = attr.ib(type=str) - - -@attr.s(frozen=True) -class Bundle: - """ - Metadata about a blockstore bundle - """ - uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - title = attr.ib(type=str) - description = attr.ib(type=str) - slug = attr.ib(type=str) - drafts = attr.ib(type=dict) # Dict of drafts, where keys are the draft names and values are draft UUIDs - # Note that if latest_version is 0, it means that no versions yet exist - latest_version = attr.ib(type=int, validator=attr.validators.instance_of(int)) - - -@attr.s(frozen=True) -class Draft: - """ - Metadata about a blockstore draft - """ - uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - name = attr.ib(type=str) - updated_at = attr.ib(type=datetime, validator=attr.validators.instance_of(datetime)) - files = attr.ib(type=dict) - links = attr.ib(type=dict) - - -@attr.s(frozen=True) -class BundleFile: - """ - Metadata about a file in a blockstore bundle or draft. - """ - path = attr.ib(type=str) - size = attr.ib(type=int) - url = attr.ib(type=str) - hash_digest = attr.ib(type=str) - - -@attr.s(frozen=True) -class DraftFile(BundleFile): - """ - Metadata about a file in a blockstore draft. - """ - modified = attr.ib(type=bool) # Was this file modified in the draft? - - -@attr.s(frozen=True) -class LinkReference: - """ - A pointer to a specific BundleVersion - """ - bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid) - version = attr.ib(type=int) - snapshot_digest = attr.ib(type=str) - - -@attr.s(frozen=True) -class LinkDetails: - """ - Details about a specific link in a BundleVersion or Draft - """ - name = attr.ib(type=str) - direct = attr.ib(type=LinkReference) - indirect = attr.ib(type=list) # List of LinkReference objects - - -@attr.s(frozen=True) -class DraftLinkDetails(LinkDetails): - """ - Details about a specific link in a Draft - """ - modified = attr.ib(type=bool) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 0903cf5c91..4a1e4f4aa9 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -4,6 +4,8 @@ # # make upgrade # +-e git+https://github.com/openedx/blockstore.git@1.2.1#egg=blockstore==1.2.1 + # via -r requirements/edx/github.in -e common/lib/capa # via # -r requirements/edx/local.in @@ -54,6 +56,7 @@ attrs==21.4.0 # via # -r requirements/edx/base.in # aiohttp + # blockstore # edx-ace # openedx-events babel==2.9.1 @@ -182,6 +185,7 @@ django==3.2.13 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.in + # blockstore # django-appconf # django-classy-tags # django-config-models @@ -272,6 +276,8 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv +django-environ==0.8.1 + # via blockstore django-fernet-fields==0.6 # via # -r requirements/edx/base.in @@ -280,6 +286,7 @@ django-fernet-fields==0.6 django-filter==21.1 # via # -r requirements/edx/base.in + # blockstore # edx-enterprise # lti-consumer-xblock django-ipware==4.0.2 @@ -359,6 +366,7 @@ django-user-tasks==3.0.0 django-waffle==2.4.1 # via # -r requirements/edx/base.in + # blockstore # edx-django-utils # edx-drf-extensions # edx-enterprise @@ -372,8 +380,10 @@ django-webpack-loader==0.7.0 djangorestframework==3.12.4 # via # -r requirements/edx/base.in + # blockstore # django-config-models # django-user-tasks + # djangorestframework-expander # drf-jwt # drf-yasg # edx-api-doc-tools @@ -386,6 +396,8 @@ djangorestframework==3.12.4 # edx-submissions # ora2 # super-csv +djangorestframework-expander==0.2.3 + # via blockstore djangorestframework-xml==2.0.0 # via edx-enterprise docopt==0.6.2 @@ -403,7 +415,9 @@ drf-yasg==1.20.0 edx-ace==1.5.0 # via -r requirements/edx/base.in edx-api-doc-tools==1.6.0 - # via -r requirements/edx/base.in + # via + # -r requirements/edx/base.in + # blockstore edx-auth-backends==4.1.0 # via -r requirements/edx/base.in edx-braze-client==0.1.3 @@ -422,12 +436,15 @@ edx-celeryutils==1.2.1 edx-completion==4.2.0 # via -r requirements/edx/base.in edx-django-release-util==1.2.0 - # via -r requirements/edx/base.in + # via + # -r requirements/edx/base.in + # blockstore edx-django-sites-extensions==4.0.0 # via -r requirements/edx/base.in edx-django-utils==4.6.0 # via # -r requirements/edx/base.in + # blockstore # django-config-models # edx-drf-extensions # edx-enterprise @@ -686,6 +703,7 @@ mysqlclient==2.1.0 newrelic==7.10.0.175 # via # -r requirements/edx/base.in + # blockstore # edx-django-utils nltk==3.7 # via @@ -757,6 +775,8 @@ py2neo==2021.2.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in +pyblake2==1.1.2 + # via blockstore pycountry==22.3.5 # via -r requirements/edx/base.in pycparser==2.21 @@ -848,6 +868,7 @@ pytz==2022.1 # via # -r requirements/edx/base.in # babel + # blockstore # celery # django # django-ses @@ -988,6 +1009,7 @@ soupsieve==2.3.2.post1 sqlparse==0.4.2 # via # -r requirements/edx/base.in + # blockstore # django staff-graded-xblock==2.0.1 # via -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 4c844c51dd..f3d2c5839a 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -4,6 +4,8 @@ # # make upgrade # +-e git+https://github.com/openedx/blockstore.git@1.2.1#egg=blockstore==1.2.1 + # via -r requirements/edx/testing.txt -e common/lib/capa # via # -r requirements/edx/testing.txt @@ -79,6 +81,7 @@ attrs==21.4.0 # via # -r requirements/edx/testing.txt # aiohttp + # blockstore # edx-ace # jsonschema # openedx-events @@ -267,6 +270,7 @@ django==3.2.13 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/testing.txt + # blockstore # django-appconf # django-classy-tags # django-config-models @@ -364,6 +368,8 @@ django-crum==0.7.9 # super-csv django-debug-toolbar==3.2.4 # via -r requirements/edx/development.in +django-environ==0.8.1 + # via blockstore django-fernet-fields==0.6 # via # -r requirements/edx/testing.txt @@ -372,6 +378,7 @@ django-fernet-fields==0.6 django-filter==21.1 # via # -r requirements/edx/testing.txt + # blockstore # edx-enterprise # lti-consumer-xblock django-ipware==4.0.2 @@ -457,6 +464,7 @@ django-user-tasks==3.0.0 django-waffle==2.4.1 # via # -r requirements/edx/testing.txt + # blockstore # edx-django-utils # edx-drf-extensions # edx-enterprise @@ -470,8 +478,10 @@ django-webpack-loader==0.7.0 djangorestframework==3.12.4 # via # -r requirements/edx/testing.txt + # blockstore # django-config-models # django-user-tasks + # djangorestframework-expander # drf-jwt # drf-yasg # edx-api-doc-tools @@ -484,6 +494,8 @@ djangorestframework==3.12.4 # edx-submissions # ora2 # super-csv +djangorestframework-expander==0.2.3 + # via blockstore djangorestframework-xml==2.0.0 # via # -r requirements/edx/testing.txt @@ -512,11 +524,15 @@ drf-yasg==1.20.0 edx-ace==1.5.0 # via -r requirements/edx/testing.txt edx-api-doc-tools==1.6.0 - # via -r requirements/edx/testing.txt + # via + # -r requirements/edx/testing.txt + # blockstore edx-auth-backends==4.1.0 # via -r requirements/edx/testing.txt edx-braze-client==0.1.3 - # via -r requirements/edx/testing.txt + # via + # -r requirements/edx/testing.txt + # blockstore edx-bulk-grades==1.0.0 # via # -r requirements/edx/testing.txt @@ -531,12 +547,15 @@ edx-celeryutils==1.2.1 edx-completion==4.2.0 # via -r requirements/edx/testing.txt edx-django-release-util==1.2.0 - # via -r requirements/edx/testing.txt + # via + # -r requirements/edx/testing.txt + # blockstore edx-django-sites-extensions==4.0.0 # via -r requirements/edx/testing.txt edx-django-utils==4.6.0 # via # -r requirements/edx/testing.txt + # blockstore # django-config-models # edx-drf-extensions # edx-enterprise @@ -918,6 +937,7 @@ mysqlclient==2.1.0 newrelic==7.10.0.175 # via # -r requirements/edx/testing.txt + # blockstore # edx-django-utils nltk==3.7 # via @@ -1028,6 +1048,8 @@ py2neo==2021.2.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt +pyblake2==1.1.2 + # via blockstore pycodestyle==2.8.0 # via -r requirements/edx/testing.txt pycountry==22.3.5 @@ -1196,6 +1218,7 @@ pytz==2022.1 # via # -r requirements/edx/testing.txt # babel + # blockstore # celery # django # django-ses @@ -1403,6 +1426,7 @@ sphinxcontrib-serializinghtml==1.1.5 sqlparse==0.4.2 # via # -r requirements/edx/testing.txt + # blockstore # django # django-debug-toolbar staff-graded-xblock==2.0.1 diff --git a/requirements/edx/github.in b/requirements/edx/github.in index 6189a16b01..36ba3a4e02 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -63,6 +63,7 @@ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752 git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc3211822a776#egg=django-require==1.0.12 # Our libraries: +-e git+https://github.com/openedx/blockstore.git@1.2.1#egg=blockstore==1.2.1 -e git+https://github.com/edx/codejail.git@3.1.3#egg=codejail==3.1.3 -e git+https://github.com/edx/RateXBlock.git@2.0.1#egg=rate-xblock -e git+https://github.com/edx-solutions/xblock-google-drive.git@2d176468e33c0713c911b563f8f65f7cf232f5b6#egg=xblock-google-drive diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 3b4c8a7af6..8b4320063e 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -4,6 +4,8 @@ # # make upgrade # +-e git+https://github.com/openedx/blockstore.git@1.2.1#egg=blockstore==1.2.1 + # via -r requirements/edx/base.txt -e common/lib/capa # via # -r requirements/edx/base.txt @@ -74,6 +76,7 @@ attrs==21.4.0 # via # -r requirements/edx/base.txt # aiohttp + # blockstore # edx-ace # openedx-events # outcome @@ -255,6 +258,7 @@ distlib==0.3.4 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.txt + # blockstore # django-appconf # django-classy-tags # django-config-models @@ -349,6 +353,10 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv +django-environ==0.8.1 + # via + # -r requirements/edx/base.txt + # blockstore django-fernet-fields==0.6 # via # -r requirements/edx/base.txt @@ -357,6 +365,7 @@ django-fernet-fields==0.6 django-filter==21.1 # via # -r requirements/edx/base.txt + # blockstore # edx-enterprise # lti-consumer-xblock django-ipware==4.0.2 @@ -442,6 +451,7 @@ django-user-tasks==3.0.0 django-waffle==2.4.1 # via # -r requirements/edx/base.txt + # blockstore # edx-django-utils # edx-drf-extensions # edx-enterprise @@ -455,8 +465,10 @@ django-webpack-loader==0.7.0 djangorestframework==3.12.4 # via # -r requirements/edx/base.txt + # blockstore # django-config-models # django-user-tasks + # djangorestframework-expander # drf-jwt # drf-yasg # edx-api-doc-tools @@ -469,6 +481,10 @@ djangorestframework==3.12.4 # edx-submissions # ora2 # super-csv +djangorestframework-expander==0.2.3 + # via + # -r requirements/edx/base.txt + # blockstore djangorestframework-xml==2.0.0 # via # -r requirements/edx/base.txt @@ -495,11 +511,15 @@ drf-yasg==1.20.0 edx-ace==1.5.0 # via -r requirements/edx/base.txt edx-api-doc-tools==1.6.0 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # blockstore edx-auth-backends==4.1.0 # via -r requirements/edx/base.txt edx-braze-client==0.1.3 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # blockstore edx-bulk-grades==1.0.0 # via # -r requirements/edx/base.txt @@ -514,12 +534,15 @@ edx-celeryutils==1.2.1 edx-completion==4.2.0 # via -r requirements/edx/base.txt edx-django-release-util==1.2.0 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # blockstore edx-django-sites-extensions==4.0.0 # via -r requirements/edx/base.txt edx-django-utils==4.6.0 # via # -r requirements/edx/base.txt + # blockstore # django-config-models # edx-drf-extensions # edx-enterprise @@ -865,6 +888,7 @@ mysqlclient==2.1.0 newrelic==7.10.0.175 # via # -r requirements/edx/base.txt + # blockstore # edx-django-utils nltk==3.7 # via @@ -967,6 +991,10 @@ py2neo==2021.2.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt +pyblake2==1.1.2 + # via + # -r requirements/edx/base.txt + # blockstore pycodestyle==2.8.0 # via -r requirements/edx/testing.in pycountry==22.3.5 @@ -1123,6 +1151,7 @@ pytz==2022.1 # via # -r requirements/edx/base.txt # babel + # blockstore # celery # django # django-ses @@ -1298,6 +1327,7 @@ soupsieve==2.3.2.post1 sqlparse==0.4.2 # via # -r requirements/edx/base.txt + # blockstore # django staff-graded-xblock==2.0.1 # via -r requirements/edx/base.txt