From ff4c2c17c2e372ef64d2f9e7156c854c7708032a Mon Sep 17 00:00:00 2001 From: swdanielli Date: Tue, 19 Aug 2014 10:23:47 -0400 Subject: [PATCH 1/5] add pyfs service --- cms/envs/dev.py | 6 +++++- common/lib/xmodule/xmodule/modulestore/django.py | 4 +++- common/lib/xmodule/xmodule/modulestore/mixed.py | 3 ++- common/lib/xmodule/xmodule/modulestore/mongo/base.py | 8 ++++++++ .../lib/xmodule/xmodule/modulestore/split_mongo/split.py | 7 +++++-- common/lib/xmodule/xmodule/modulestore/xml.py | 6 +++++- lms/djangoapps/courseware/module_render.py | 2 ++ lms/envs/dev.py | 7 ++++++- lms/lib/xblock/runtime.py | 2 ++ requirements/edx/base.txt | 2 ++ requirements/edx/edx-private.txt | 1 - 11 files changed, 40 insertions(+), 8 deletions(-) diff --git a/cms/envs/dev.py b/cms/envs/dev.py index d1208c0080..2c3a64e599 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -27,6 +27,10 @@ update_module_store_settings( } ) +DJFS = {'type': 'osfs', + 'directory_root': 'cms/static/djpyfs', + 'url_root': '/static/djpyfs'} + # cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store # This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc) CONTENTSTORE = { @@ -129,7 +133,7 @@ CELERY_ALWAYS_EAGER = True ################################ DEBUG TOOLBAR ################################# INSTALLED_APPS += ('debug_toolbar', 'debug_toolbar_mongo') -MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) +MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware', 'djpyfs') INTERNAL_IPS = ('127.0.0.1',) DEBUG_TOOLBAR_PANELS = ( diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py index 84f694fd4e..ac0ac72f00 100644 --- a/common/lib/xmodule/xmodule/modulestore/django.py +++ b/common/lib/xmodule/xmodule/modulestore/django.py @@ -19,6 +19,7 @@ import threading from xmodule.util.django import get_current_request_hostname import xmodule.modulestore # pylint: disable=unused-import from xmodule.contentstore.django import contentstore +import xblock.reference.plugins # We may not always have the request_cache module available try: @@ -41,7 +42,7 @@ def load_function(path): return getattr(import_module(module_path), name) -def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None): +def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None, pyfs_service=None): """ This will return a new instance of a modulestore given an engine and options """ @@ -73,6 +74,7 @@ def create_modulestore_instance(engine, content_store, doc_store_config, options xblock_select=getattr(settings, 'XBLOCK_SELECT_FUNCTION', None), doc_store_config=doc_store_config, i18n_service=i18n_service or ModuleI18nService(), + pyfs_service=pyfs_service or xblock.reference.plugins.FSService(), branch_setting_func=_get_modulestore_branch_setting, create_modulestore_instance=create_modulestore_instance, **_options diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py index 91b4f1076d..c233c6bcb3 100644 --- a/common/lib/xmodule/xmodule/modulestore/mixed.py +++ b/common/lib/xmodule/xmodule/modulestore/mixed.py @@ -91,7 +91,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): """ ModuleStore knows how to route requests to the right persistence ms """ - def __init__(self, contentstore, mappings, stores, i18n_service=None, create_modulestore_instance=None, **kwargs): + def __init__(self, contentstore, mappings, stores, i18n_service=None, pyfs_service=None, create_modulestore_instance=None, **kwargs): """ Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a collection of other modulestore configuration information @@ -130,6 +130,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): store_settings.get('DOC_STORE_CONFIG', {}), store_settings.get('OPTIONS', {}), i18n_service=i18n_service, + pyfs_service=pyfs_service, ) # replace all named pointers to the store into actual pointers for course_key, store_name in self.mappings.iteritems(): diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index e2592168d2..05fcff9d43 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -361,6 +361,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): default_class=None, error_tracker=null_error_tracker, i18n_service=None, + pyfs_service=None, **kwargs): """ :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware. @@ -404,6 +405,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): self.error_tracker = error_tracker self.render_template = render_template self.i18n_service = i18n_service + self.pyfs_service = pyfs_service # performance optimization to prevent updating the meta-data inheritance tree during # bulk write operations @@ -691,6 +693,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): if self.i18n_service: services["i18n"] = self.i18n_service + if self.pyfs_service: + services["fs"] = self.pyfs_service + system = CachingDescriptorSystem( modulestore=self, course_key=course_key, @@ -984,6 +989,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): if self.i18n_service: services["i18n"] = self.i18n_service + if self.pyfs_service: + services["fs"] = self.pyfs_service + runtime = CachingDescriptorSystem( modulestore=self, module_data={}, diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index abe9a693f0..aa2c9ef914 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -120,8 +120,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): def __init__(self, contentstore, doc_store_config, fs_root, render_template, default_class=None, error_tracker=null_error_tracker, - i18n_service=None, services=None, - **kwargs): + i18n_service=None, pyfs_service=None, + services=None, **kwargs): """ :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware. """ @@ -148,6 +148,9 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): if i18n_service is not None: self.services["i18n"] = i18n_service + if pyfs_service is not None: + self.services["fs"] = pyfs_service + def close_connections(self): """ Closes any open connections to the underlying databases diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index b329fdc32a..907e8e42ab 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -370,7 +370,7 @@ class XMLModuleStore(ModuleStoreReadBase): """ def __init__( self, data_dir, default_class=None, course_dirs=None, course_ids=None, - load_error_modules=True, i18n_service=None, **kwargs + load_error_modules=True, i18n_service=None, pyfs_service=None, **kwargs ): """ Initialize an XMLModuleStore from data_dir @@ -409,6 +409,7 @@ class XMLModuleStore(ModuleStoreReadBase): self.field_data = inheriting_field_data(kvs=DictKeyValueStore()) self.i18n_service = i18n_service + self.pyfs_service = pyfs_service # If we are specifically asked for missing courses, that should # be an error. If we are asked for "all" courses, find the ones @@ -554,6 +555,9 @@ class XMLModuleStore(ModuleStoreReadBase): if self.i18n_service: services['i18n'] = self.i18n_service + if self.pyfs_service: + services['fs'] = self.pyfs_service + system = ImportSystem( xmlstore=self, course_id=course_id, diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index c385facb6d..9d084f64ec 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -3,6 +3,7 @@ import logging import mimetypes import static_replace +import xblock.reference.plugins from functools import partial from requests.auth import HTTPBasicAuth @@ -496,6 +497,7 @@ def get_module_system_for_user(user, field_data_cache, get_real_user=user_by_anonymous_id, services={ 'i18n': ModuleI18nService(), + 'fs': xblock.reference.plugins.FSService(), }, get_user_role=lambda: get_user_role(user, course_id), descriptor_runtime=descriptor.runtime, diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 0f4e91d547..33094de06c 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -49,6 +49,10 @@ LOGGING = get_logger_config(ENV_ROOT / "log", dev_env=True, debug=True) +DJFS = {'type': 'osfs', + 'directory_root': 'lms/static/djpyfs', + 'url_root': '/static/djpyfs'} + # If there is a database called 'read_replica', you can use the use_read_replica_if_available # function in util/query.py, which is useful for very large database reads DATABASES = { @@ -216,7 +220,8 @@ CELERY_ALWAYS_EAGER = True INSTALLED_APPS += ('debug_toolbar',) MIDDLEWARE_CLASSES += ('django_comment_client.utils.QueryCountDebugMiddleware', - 'debug_toolbar.middleware.DebugToolbarMiddleware',) + 'debug_toolbar.middleware.DebugToolbarMiddleware', + 'djpyfs') INTERNAL_IPS = ('127.0.0.1',) DEBUG_TOOLBAR_PANELS = ( diff --git a/lms/lib/xblock/runtime.py b/lms/lib/xblock/runtime.py index 4c547be647..f02657c511 100644 --- a/lms/lib/xblock/runtime.py +++ b/lms/lib/xblock/runtime.py @@ -3,6 +3,7 @@ Module implementing `xblock.runtime.Runtime` functionality for the LMS """ import re +import xblock.reference.plugins from django.core.urlresolvers import reverse from django.conf import settings @@ -193,4 +194,5 @@ class LmsModuleSystem(LmsHandlerUrls, ModuleSystem): # pylint: disable=abstract course_id=kwargs.get('course_id', None), track_function=kwargs.get('track_function', None), ) + services['fs'] = xblock.reference.plugins.FSService() super(LmsModuleSystem, self).__init__(**kwargs) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index ca886da487..f5e65f53be 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -137,3 +137,5 @@ git+https://github.com/mitocw/django-cas.git # edX packages edx-submissions==0.0.6 + +-e git+https://github.com/pmitros/django-pyfs.git@514607d78535fd80bfd23184cd292ee5799b500d#egg=djpyfs diff --git a/requirements/edx/edx-private.txt b/requirements/edx/edx-private.txt index f2d081b5f4..086198b451 100644 --- a/requirements/edx/edx-private.txt +++ b/requirements/edx/edx-private.txt @@ -15,4 +15,3 @@ # It is an R&D prototype, intended for roll-out one location in one course. # It should not be used without learning sciences support in the current state. -e git+https://github.com/pmitros/RecommenderXBlock.git@fae9e5bc8a8297cb15001f0d674430e3d22ffa35#egg=recommender-xblock - From 9d55cf3637836017aaa6d879349b21ba13cbe83d Mon Sep 17 00:00:00 2001 From: swdanielli Date: Tue, 19 Aug 2014 15:29:34 -0400 Subject: [PATCH 2/5] code clean --- cms/envs/dev.py | 8 +++++--- common/lib/xmodule/xmodule/modulestore/django.py | 4 ++-- common/lib/xmodule/xmodule/modulestore/mixed.py | 4 ++-- .../xmodule/xmodule/modulestore/mongo/base.py | 12 ++++++------ .../xmodule/modulestore/split_mongo/split.py | 6 +++--- lms/envs/dev.py | 16 ++++++++++------ 6 files changed, 28 insertions(+), 22 deletions(-) diff --git a/cms/envs/dev.py b/cms/envs/dev.py index 2c3a64e599..3a6e06f4a0 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -27,9 +27,11 @@ update_module_store_settings( } ) -DJFS = {'type': 'osfs', - 'directory_root': 'cms/static/djpyfs', - 'url_root': '/static/djpyfs'} +DJFS = { + 'type': 'osfs', + 'directory_root': 'cms/static/djpyfs', + 'url_root': '/static/djpyfs' +} # cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store # This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc) diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py index ac0ac72f00..67aebc14ad 100644 --- a/common/lib/xmodule/xmodule/modulestore/django.py +++ b/common/lib/xmodule/xmodule/modulestore/django.py @@ -42,7 +42,7 @@ def load_function(path): return getattr(import_module(module_path), name) -def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None, pyfs_service=None): +def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None, fs_service=None): """ This will return a new instance of a modulestore given an engine and options """ @@ -74,7 +74,7 @@ def create_modulestore_instance(engine, content_store, doc_store_config, options xblock_select=getattr(settings, 'XBLOCK_SELECT_FUNCTION', None), doc_store_config=doc_store_config, i18n_service=i18n_service or ModuleI18nService(), - pyfs_service=pyfs_service or xblock.reference.plugins.FSService(), + fs_service=fs_service or xblock.reference.plugins.FSService(), branch_setting_func=_get_modulestore_branch_setting, create_modulestore_instance=create_modulestore_instance, **_options diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py index c233c6bcb3..ab7f3cdb93 100644 --- a/common/lib/xmodule/xmodule/modulestore/mixed.py +++ b/common/lib/xmodule/xmodule/modulestore/mixed.py @@ -91,7 +91,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): """ ModuleStore knows how to route requests to the right persistence ms """ - def __init__(self, contentstore, mappings, stores, i18n_service=None, pyfs_service=None, create_modulestore_instance=None, **kwargs): + def __init__(self, contentstore, mappings, stores, i18n_service=None, fs_service=None, create_modulestore_instance=None, **kwargs): """ Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a collection of other modulestore configuration information @@ -130,7 +130,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): store_settings.get('DOC_STORE_CONFIG', {}), store_settings.get('OPTIONS', {}), i18n_service=i18n_service, - pyfs_service=pyfs_service, + fs_service=fs_service, ) # replace all named pointers to the store into actual pointers for course_key, store_name in self.mappings.iteritems(): diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 05fcff9d43..cd936dbb9f 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -361,7 +361,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): default_class=None, error_tracker=null_error_tracker, i18n_service=None, - pyfs_service=None, + fs_service=None, **kwargs): """ :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware. @@ -405,7 +405,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): self.error_tracker = error_tracker self.render_template = render_template self.i18n_service = i18n_service - self.pyfs_service = pyfs_service + self.fs_service = fs_service # performance optimization to prevent updating the meta-data inheritance tree during # bulk write operations @@ -693,8 +693,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): if self.i18n_service: services["i18n"] = self.i18n_service - if self.pyfs_service: - services["fs"] = self.pyfs_service + if self.fs_service: + services["fs"] = self.fs_service system = CachingDescriptorSystem( modulestore=self, @@ -989,8 +989,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): if self.i18n_service: services["i18n"] = self.i18n_service - if self.pyfs_service: - services["fs"] = self.pyfs_service + if self.fs_service: + services["fs"] = self.fs_service runtime = CachingDescriptorSystem( modulestore=self, diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index aa2c9ef914..41c9c66f32 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -120,7 +120,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): def __init__(self, contentstore, doc_store_config, fs_root, render_template, default_class=None, error_tracker=null_error_tracker, - i18n_service=None, pyfs_service=None, + i18n_service=None, fs_service=None, services=None, **kwargs): """ :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware. @@ -148,8 +148,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): if i18n_service is not None: self.services["i18n"] = i18n_service - if pyfs_service is not None: - self.services["fs"] = pyfs_service + if fs_service is not None: + self.services["fs"] = fs_service def close_connections(self): """ diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 33094de06c..a2124eed11 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -49,9 +49,11 @@ LOGGING = get_logger_config(ENV_ROOT / "log", dev_env=True, debug=True) -DJFS = {'type': 'osfs', - 'directory_root': 'lms/static/djpyfs', - 'url_root': '/static/djpyfs'} +DJFS = { + 'type': 'osfs', + 'directory_root': 'lms/static/djpyfs', + 'url_root': '/static/djpyfs' +} # If there is a database called 'read_replica', you can use the use_read_replica_if_available # function in util/query.py, which is useful for very large database reads @@ -219,9 +221,11 @@ CELERY_ALWAYS_EAGER = True ################################ DEBUG TOOLBAR ################################ INSTALLED_APPS += ('debug_toolbar',) -MIDDLEWARE_CLASSES += ('django_comment_client.utils.QueryCountDebugMiddleware', - 'debug_toolbar.middleware.DebugToolbarMiddleware', - 'djpyfs') +MIDDLEWARE_CLASSES += ( + 'django_comment_client.utils.QueryCountDebugMiddleware', + 'debug_toolbar.middleware.DebugToolbarMiddleware', + 'djpyfs', +) INTERNAL_IPS = ('127.0.0.1',) DEBUG_TOOLBAR_PANELS = ( From 504a7a21ba3ba6b8d7c66c2b17a117ebe2aca504 Mon Sep 17 00:00:00 2001 From: swdanielli Date: Tue, 19 Aug 2014 10:23:47 -0400 Subject: [PATCH 3/5] add pyfs service --- cms/envs/dev.py | 6 +++++- common/lib/xmodule/xmodule/modulestore/django.py | 4 +++- common/lib/xmodule/xmodule/modulestore/mixed.py | 3 ++- common/lib/xmodule/xmodule/modulestore/mongo/base.py | 8 ++++++++ .../lib/xmodule/xmodule/modulestore/split_mongo/split.py | 7 +++++-- common/lib/xmodule/xmodule/modulestore/xml.py | 6 +++++- lms/djangoapps/courseware/module_render.py | 2 ++ lms/envs/dev.py | 7 ++++++- lms/lib/xblock/runtime.py | 2 ++ requirements/edx/base.txt | 2 ++ requirements/edx/edx-private.txt | 1 - 11 files changed, 40 insertions(+), 8 deletions(-) diff --git a/cms/envs/dev.py b/cms/envs/dev.py index fd0249d313..7a485d2356 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -27,6 +27,10 @@ update_module_store_settings( } ) +DJFS = {'type': 'osfs', + 'directory_root': 'cms/static/djpyfs', + 'url_root': '/static/djpyfs'} + # cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store # This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc) CONTENTSTORE = { @@ -129,7 +133,7 @@ CELERY_ALWAYS_EAGER = True ################################ DEBUG TOOLBAR ################################# INSTALLED_APPS += ('debug_toolbar', 'debug_toolbar_mongo') -MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) +MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware', 'djpyfs') INTERNAL_IPS = ('127.0.0.1',) DEBUG_TOOLBAR_PANELS = ( diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py index 84f694fd4e..ac0ac72f00 100644 --- a/common/lib/xmodule/xmodule/modulestore/django.py +++ b/common/lib/xmodule/xmodule/modulestore/django.py @@ -19,6 +19,7 @@ import threading from xmodule.util.django import get_current_request_hostname import xmodule.modulestore # pylint: disable=unused-import from xmodule.contentstore.django import contentstore +import xblock.reference.plugins # We may not always have the request_cache module available try: @@ -41,7 +42,7 @@ def load_function(path): return getattr(import_module(module_path), name) -def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None): +def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None, pyfs_service=None): """ This will return a new instance of a modulestore given an engine and options """ @@ -73,6 +74,7 @@ def create_modulestore_instance(engine, content_store, doc_store_config, options xblock_select=getattr(settings, 'XBLOCK_SELECT_FUNCTION', None), doc_store_config=doc_store_config, i18n_service=i18n_service or ModuleI18nService(), + pyfs_service=pyfs_service or xblock.reference.plugins.FSService(), branch_setting_func=_get_modulestore_branch_setting, create_modulestore_instance=create_modulestore_instance, **_options diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py index 5cb862c50b..0f0d0e4c96 100644 --- a/common/lib/xmodule/xmodule/modulestore/mixed.py +++ b/common/lib/xmodule/xmodule/modulestore/mixed.py @@ -91,7 +91,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): """ ModuleStore knows how to route requests to the right persistence ms """ - def __init__(self, contentstore, mappings, stores, i18n_service=None, create_modulestore_instance=None, **kwargs): + def __init__(self, contentstore, mappings, stores, i18n_service=None, pyfs_service=None, create_modulestore_instance=None, **kwargs): """ Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a collection of other modulestore configuration information @@ -130,6 +130,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): store_settings.get('DOC_STORE_CONFIG', {}), store_settings.get('OPTIONS', {}), i18n_service=i18n_service, + pyfs_service=pyfs_service, ) # replace all named pointers to the store into actual pointers for course_key, store_name in self.mappings.iteritems(): diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 7568d566ee..9e1adc4c11 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -366,6 +366,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): default_class=None, error_tracker=null_error_tracker, i18n_service=None, + pyfs_service=None, **kwargs): """ :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware. @@ -409,6 +410,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): self.error_tracker = error_tracker self.render_template = render_template self.i18n_service = i18n_service + self.pyfs_service = pyfs_service # performance optimization to prevent updating the meta-data inheritance tree during # bulk write operations @@ -696,6 +698,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): if self.i18n_service: services["i18n"] = self.i18n_service + if self.pyfs_service: + services["fs"] = self.pyfs_service + system = CachingDescriptorSystem( modulestore=self, course_key=course_key, @@ -989,6 +994,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): if self.i18n_service: services["i18n"] = self.i18n_service + if self.pyfs_service: + services["fs"] = self.pyfs_service + runtime = CachingDescriptorSystem( modulestore=self, module_data={}, diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index 25b8391818..9255dc8ebe 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -120,8 +120,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): def __init__(self, contentstore, doc_store_config, fs_root, render_template, default_class=None, error_tracker=null_error_tracker, - i18n_service=None, services=None, - **kwargs): + i18n_service=None, pyfs_service=None, + services=None, **kwargs): """ :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware. """ @@ -148,6 +148,9 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): if i18n_service is not None: self.services["i18n"] = i18n_service + if pyfs_service is not None: + self.services["fs"] = pyfs_service + def close_connections(self): """ Closes any open connections to the underlying databases diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index b329fdc32a..907e8e42ab 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -370,7 +370,7 @@ class XMLModuleStore(ModuleStoreReadBase): """ def __init__( self, data_dir, default_class=None, course_dirs=None, course_ids=None, - load_error_modules=True, i18n_service=None, **kwargs + load_error_modules=True, i18n_service=None, pyfs_service=None, **kwargs ): """ Initialize an XMLModuleStore from data_dir @@ -409,6 +409,7 @@ class XMLModuleStore(ModuleStoreReadBase): self.field_data = inheriting_field_data(kvs=DictKeyValueStore()) self.i18n_service = i18n_service + self.pyfs_service = pyfs_service # If we are specifically asked for missing courses, that should # be an error. If we are asked for "all" courses, find the ones @@ -554,6 +555,9 @@ class XMLModuleStore(ModuleStoreReadBase): if self.i18n_service: services['i18n'] = self.i18n_service + if self.pyfs_service: + services['fs'] = self.pyfs_service + system = ImportSystem( xmlstore=self, course_id=course_id, diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index d1ebdfba74..b8accd95a8 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -3,6 +3,7 @@ import logging import mimetypes import static_replace +import xblock.reference.plugins from functools import partial from requests.auth import HTTPBasicAuth @@ -535,6 +536,7 @@ def get_module_system_for_user(user, field_data_cache, get_real_user=user_by_anonymous_id, services={ 'i18n': ModuleI18nService(), + 'fs': xblock.reference.plugins.FSService(), }, get_user_role=lambda: get_user_role(user, course_id), descriptor_runtime=descriptor.runtime, diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 1193c84e34..2a8771a2a3 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -50,6 +50,10 @@ LOGGING = get_logger_config(ENV_ROOT / "log", dev_env=True, debug=True) +DJFS = {'type': 'osfs', + 'directory_root': 'lms/static/djpyfs', + 'url_root': '/static/djpyfs'} + # If there is a database called 'read_replica', you can use the use_read_replica_if_available # function in util/query.py, which is useful for very large database reads DATABASES = { @@ -217,7 +221,8 @@ CELERY_ALWAYS_EAGER = True INSTALLED_APPS += ('debug_toolbar',) MIDDLEWARE_CLASSES += ('django_comment_client.utils.QueryCountDebugMiddleware', - 'debug_toolbar.middleware.DebugToolbarMiddleware',) + 'debug_toolbar.middleware.DebugToolbarMiddleware', + 'djpyfs') INTERNAL_IPS = ('127.0.0.1',) DEBUG_TOOLBAR_PANELS = ( diff --git a/lms/lib/xblock/runtime.py b/lms/lib/xblock/runtime.py index 4c547be647..f02657c511 100644 --- a/lms/lib/xblock/runtime.py +++ b/lms/lib/xblock/runtime.py @@ -3,6 +3,7 @@ Module implementing `xblock.runtime.Runtime` functionality for the LMS """ import re +import xblock.reference.plugins from django.core.urlresolvers import reverse from django.conf import settings @@ -193,4 +194,5 @@ class LmsModuleSystem(LmsHandlerUrls, ModuleSystem): # pylint: disable=abstract course_id=kwargs.get('course_id', None), track_function=kwargs.get('track_function', None), ) + services['fs'] = xblock.reference.plugins.FSService() super(LmsModuleSystem, self).__init__(**kwargs) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index a8eacf20d1..471041a97b 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -138,3 +138,5 @@ git+https://github.com/mitocw/django-cas.git # edX packages edx-submissions==0.0.6 + +-e git+https://github.com/pmitros/django-pyfs.git@514607d78535fd80bfd23184cd292ee5799b500d#egg=djpyfs diff --git a/requirements/edx/edx-private.txt b/requirements/edx/edx-private.txt index 1f3d45aa6d..4a10aed7ad 100644 --- a/requirements/edx/edx-private.txt +++ b/requirements/edx/edx-private.txt @@ -15,4 +15,3 @@ # It is an R&D prototype, intended for roll-out one location in one course. # It should not be used without learning sciences support in the current state. -e git+https://github.com/pmitros/RecommenderXBlock.git@fae9e5bc8a8297cb15001f0d674430e3d22ffa35#egg=recommender-xblock - From 8378422f3de1b73c746a04dd44f6284303294ee7 Mon Sep 17 00:00:00 2001 From: swdanielli Date: Tue, 19 Aug 2014 15:29:34 -0400 Subject: [PATCH 4/5] code clean --- cms/envs/dev.py | 8 +++++--- common/lib/xmodule/xmodule/modulestore/django.py | 4 ++-- common/lib/xmodule/xmodule/modulestore/mixed.py | 4 ++-- .../xmodule/xmodule/modulestore/mongo/base.py | 12 ++++++------ .../xmodule/modulestore/split_mongo/split.py | 6 +++--- lms/envs/dev.py | 16 ++++++++++------ 6 files changed, 28 insertions(+), 22 deletions(-) diff --git a/cms/envs/dev.py b/cms/envs/dev.py index 7a485d2356..e4f0030fb0 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -27,9 +27,11 @@ update_module_store_settings( } ) -DJFS = {'type': 'osfs', - 'directory_root': 'cms/static/djpyfs', - 'url_root': '/static/djpyfs'} +DJFS = { + 'type': 'osfs', + 'directory_root': 'cms/static/djpyfs', + 'url_root': '/static/djpyfs' +} # cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store # This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc) diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py index ac0ac72f00..67aebc14ad 100644 --- a/common/lib/xmodule/xmodule/modulestore/django.py +++ b/common/lib/xmodule/xmodule/modulestore/django.py @@ -42,7 +42,7 @@ def load_function(path): return getattr(import_module(module_path), name) -def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None, pyfs_service=None): +def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None, fs_service=None): """ This will return a new instance of a modulestore given an engine and options """ @@ -74,7 +74,7 @@ def create_modulestore_instance(engine, content_store, doc_store_config, options xblock_select=getattr(settings, 'XBLOCK_SELECT_FUNCTION', None), doc_store_config=doc_store_config, i18n_service=i18n_service or ModuleI18nService(), - pyfs_service=pyfs_service or xblock.reference.plugins.FSService(), + fs_service=fs_service or xblock.reference.plugins.FSService(), branch_setting_func=_get_modulestore_branch_setting, create_modulestore_instance=create_modulestore_instance, **_options diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py index 0f0d0e4c96..6b40be78fb 100644 --- a/common/lib/xmodule/xmodule/modulestore/mixed.py +++ b/common/lib/xmodule/xmodule/modulestore/mixed.py @@ -91,7 +91,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): """ ModuleStore knows how to route requests to the right persistence ms """ - def __init__(self, contentstore, mappings, stores, i18n_service=None, pyfs_service=None, create_modulestore_instance=None, **kwargs): + def __init__(self, contentstore, mappings, stores, i18n_service=None, fs_service=None, create_modulestore_instance=None, **kwargs): """ Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a collection of other modulestore configuration information @@ -130,7 +130,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): store_settings.get('DOC_STORE_CONFIG', {}), store_settings.get('OPTIONS', {}), i18n_service=i18n_service, - pyfs_service=pyfs_service, + fs_service=fs_service, ) # replace all named pointers to the store into actual pointers for course_key, store_name in self.mappings.iteritems(): diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 9e1adc4c11..b51264e93b 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -366,7 +366,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): default_class=None, error_tracker=null_error_tracker, i18n_service=None, - pyfs_service=None, + fs_service=None, **kwargs): """ :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware. @@ -410,7 +410,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): self.error_tracker = error_tracker self.render_template = render_template self.i18n_service = i18n_service - self.pyfs_service = pyfs_service + self.fs_service = fs_service # performance optimization to prevent updating the meta-data inheritance tree during # bulk write operations @@ -698,8 +698,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): if self.i18n_service: services["i18n"] = self.i18n_service - if self.pyfs_service: - services["fs"] = self.pyfs_service + if self.fs_service: + services["fs"] = self.fs_service system = CachingDescriptorSystem( modulestore=self, @@ -994,8 +994,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): if self.i18n_service: services["i18n"] = self.i18n_service - if self.pyfs_service: - services["fs"] = self.pyfs_service + if self.fs_service: + services["fs"] = self.fs_service runtime = CachingDescriptorSystem( modulestore=self, diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index 9255dc8ebe..036e7c0a54 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -120,7 +120,7 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): def __init__(self, contentstore, doc_store_config, fs_root, render_template, default_class=None, error_tracker=null_error_tracker, - i18n_service=None, pyfs_service=None, + i18n_service=None, fs_service=None, services=None, **kwargs): """ :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware. @@ -148,8 +148,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase): if i18n_service is not None: self.services["i18n"] = i18n_service - if pyfs_service is not None: - self.services["fs"] = pyfs_service + if fs_service is not None: + self.services["fs"] = fs_service def close_connections(self): """ diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 2a8771a2a3..77b0323b22 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -50,9 +50,11 @@ LOGGING = get_logger_config(ENV_ROOT / "log", dev_env=True, debug=True) -DJFS = {'type': 'osfs', - 'directory_root': 'lms/static/djpyfs', - 'url_root': '/static/djpyfs'} +DJFS = { + 'type': 'osfs', + 'directory_root': 'lms/static/djpyfs', + 'url_root': '/static/djpyfs' +} # If there is a database called 'read_replica', you can use the use_read_replica_if_available # function in util/query.py, which is useful for very large database reads @@ -220,9 +222,11 @@ CELERY_ALWAYS_EAGER = True ################################ DEBUG TOOLBAR ################################ INSTALLED_APPS += ('debug_toolbar',) -MIDDLEWARE_CLASSES += ('django_comment_client.utils.QueryCountDebugMiddleware', - 'debug_toolbar.middleware.DebugToolbarMiddleware', - 'djpyfs') +MIDDLEWARE_CLASSES += ( + 'django_comment_client.utils.QueryCountDebugMiddleware', + 'debug_toolbar.middleware.DebugToolbarMiddleware', + 'djpyfs', +) INTERNAL_IPS = ('127.0.0.1',) DEBUG_TOOLBAR_PANELS = ( From a7d78e755dbabafdb26cc5f4a9da8f57f3984f07 Mon Sep 17 00:00:00 2001 From: swdanielli Date: Mon, 25 Aug 2014 15:38:35 -0400 Subject: [PATCH 5/5] fix test case for pyfs --- .../xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py index 598784b5df..aead5d7c37 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -1280,7 +1280,7 @@ def load_function(path): # pylint: disable=unused-argument -def create_modulestore_instance(engine, contentstore, doc_store_config, options, i18n_service=None): +def create_modulestore_instance(engine, contentstore, doc_store_config, options, i18n_service=None, fs_service=None): """ This will return a new instance of a modulestore given an engine and options """