From 8fd3ab07ba357fee58f85b7cb72435ba1523c953 Mon Sep 17 00:00:00 2001 From: Felipe Montoya Date: Mon, 28 Apr 2014 12:03:48 -0500 Subject: [PATCH 001/137] If microsites are enabled and it is a microsite request, then take the microsite path for theming --- common/djangoapps/microsite_configuration/microsite.py | 2 +- lms/templates/main.html | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/common/djangoapps/microsite_configuration/microsite.py b/common/djangoapps/microsite_configuration/microsite.py index 91f8eaffa1..27f513ebe1 100644 --- a/common/djangoapps/microsite_configuration/microsite.py +++ b/common/djangoapps/microsite_configuration/microsite.py @@ -36,7 +36,7 @@ def is_request_in_microsite(): """ This will return if current request is a request within a microsite """ - return get_configuration() + return bool(get_configuration()) def get_value(val_name, default=None): diff --git a/lms/templates/main.html b/lms/templates/main.html index 8b1603f83a..acbabc2d92 100644 --- a/lms/templates/main.html +++ b/lms/templates/main.html @@ -20,6 +20,9 @@ <%def name="theme_enabled()"> <% return settings.FEATURES.get("USE_CUSTOM_THEME", False) %> +<%def name="is_microsite()"> + <% return microsite.is_request_in_microsite() %> + <%def name="stanford_theme_enabled()"> <% @@ -61,7 +64,7 @@ <%block name="headextra"/> <% - if theme_enabled(): + if theme_enabled() and not is_microsite(): header_extra_file = 'theme-head-extra.html' header_file = 'theme-header.html' google_analytics_file = 'theme-google-analytics.html' From b55362b925637ae553289a7405e93f0cdfd7792d Mon Sep 17 00:00:00 2001 From: Jason Bau Date: Fri, 8 Aug 2014 00:50:15 -0700 Subject: [PATCH 002/137] make unit tests respect mongo port/host settings (with default) settings are read in from environment variable --- cms/envs/test.py | 13 +++++++++-- .../lib/xmodule/xmodule/contentstore/mongo.py | 2 +- .../xmodule/modulestore/tests/django_utils.py | 10 ++++++--- .../modulestore/tests/mongo_connection.py | 10 +++++++++ .../modulestore/tests/test_contentstore.py | 7 +++--- .../test_cross_modulestore_import_export.py | 6 +++-- .../tests/test_mixed_modulestore.py | 7 ++++-- .../xmodule/modulestore/tests/test_mongo.py | 11 +++++----- .../tests/test_split_modulestore.py | 4 +++- .../tests/test_split_w_old_mongo.py | 4 +++- .../modulestore/tests/test_xml_importer.py | 6 +++-- lms/djangoapps/dashboard/git_import.py | 7 +++--- .../commands/tests/test_git_add_course.py | 6 +++-- .../dashboard/tests/test_sysadmin.py | 5 +++-- lms/envs/test.py | 22 +++++++++++++++++-- pavelib/utils/test/utils.py | 11 +++++++++- 16 files changed, 99 insertions(+), 32 deletions(-) create mode 100644 common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py diff --git a/cms/envs/test.py b/cms/envs/test.py index 068bd39c43..8522bda6bb 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -21,6 +21,12 @@ from uuid import uuid4 # import settings from LMS for consistent behavior with CMS from lms.envs.test import (WIKI_ENABLED, PLATFORM_NAME, SITE_NAME) +# mongo connection settings +MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) +MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost') + +THIS_UUID = uuid4().hex[:5] + # Nose Test Runner TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' @@ -79,15 +85,18 @@ update_module_store_settings( }, doc_store_settings={ 'db': 'test_xmodule', - 'collection': 'test_modulestore{0}'.format(uuid4().hex[:5]), + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, + 'collection': 'test_modulestore{0}'.format(THIS_UUID), }, ) CONTENTSTORE = { 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', 'DOC_STORE_CONFIG': { - 'host': 'localhost', + 'host': MONGO_HOST, 'db': 'test_xcontent', + 'port': MONGO_PORT_NUM, 'collection': 'dont_trip', }, # allow for additional options that can be keyed on a name, e.g. 'trashcan' diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py index 287ceeb32a..5a04bf91ee 100644 --- a/common/lib/xmodule/xmodule/contentstore/mongo.py +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -25,7 +25,7 @@ class MongoContentStore(ContentStore): :param collection: ignores but provided for consistency w/ other doc_store_config patterns """ - logging.debug('Using MongoDB for static content serving at host={0} db={1}'.format(host, db)) + logging.debug('Using MongoDB for static content serving at host={0} port={1} db={2}'.format(host, port, db)) _db = pymongo.database.Database( pymongo.MongoClient( host=host, diff --git a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py index 6a6d8843d3..38ce305592 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/django_utils.py @@ -2,7 +2,6 @@ """ Modulestore configuration for test cases. """ - from uuid import uuid4 from django.test import TestCase from django.contrib.auth.models import User @@ -13,6 +12,7 @@ import datetime import pytz from xmodule.tabs import CoursewareTab, CourseInfoTab, StaticTab, DiscussionTab, ProgressTab, WikiTab from xmodule.modulestore.tests.sample_courses import default_block_info_tree, TOY_BLOCK_INFO_TREE +from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST def mixed_store_config(data_dir, mappings): @@ -67,7 +67,8 @@ def draft_mongo_store_config(data_dir): 'NAME': 'draft', 'ENGINE': 'xmodule.modulestore.mongo.draft.DraftModuleStore', 'DOC_STORE_CONFIG': { - 'host': 'localhost', + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, 'db': 'test_xmodule', 'collection': 'modulestore{0}'.format(uuid4().hex[:5]), }, @@ -93,7 +94,8 @@ def split_mongo_store_config(data_dir): 'NAME': 'draft', 'ENGINE': 'xmodule.modulestore.split_mongo.split_draft.DraftVersioningModuleStore', 'DOC_STORE_CONFIG': { - 'host': 'localhost', + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, 'db': 'test_xmodule', 'collection': 'modulestore{0}'.format(uuid4().hex[:5]), }, @@ -229,6 +231,8 @@ class ModuleStoreTestCase(TestCase): if hasattr(module_store, '_drop_database'): module_store._drop_database() # pylint: disable=protected-access _CONTENTSTORE.clear() + if hasattr(module_store, 'close_connections'): + module_store.close_connections() @classmethod def setUpClass(cls): diff --git a/common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py new file mode 100644 index 0000000000..1b18d485d6 --- /dev/null +++ b/common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py @@ -0,0 +1,10 @@ +""" +This file is intended to provide settings for the mongodb connection used for tests. +The settings can be provided by environment variables in the shell running the tests. This reads +in a variety of environment variables but provides sensible defaults in case those env var +overrides don't exist +""" +import os + +MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) +MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost') diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_contentstore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_contentstore.py index cda981e329..9856730643 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_contentstore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_contentstore.py @@ -1,6 +1,7 @@ """ Test contentstore.mongo functionality """ +import os import logging from uuid import uuid4 import unittest @@ -17,12 +18,12 @@ from xmodule.contentstore.content import StaticContent from xmodule.exceptions import NotFoundError import ddt from __builtin__ import delattr - +from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST log = logging.getLogger(__name__) -HOST = 'localhost' -PORT = 27017 +HOST = MONGO_HOST +PORT = MONGO_PORT_NUM DB = 'test_mongo_%s' % uuid4().hex[:5] diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py index 3c407eceef..2dd62b9217 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py @@ -11,7 +11,6 @@ and then for each combination of modulestores, performing the sequence: 4) Compare all modules in the source and destination modulestores to make sure that they line up """ - import ddt import itertools import random @@ -28,9 +27,12 @@ from xmodule.contentstore.mongo import MongoContentStore from xmodule.modulestore.xml_importer import import_from_xml from xmodule.modulestore.xml_exporter import export_to_xml from xmodule.modulestore.split_mongo.split_draft import DraftVersioningModuleStore +from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST + COMMON_DOCSTORE_CONFIG = { - 'host': 'localhost' + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, } diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py index 598784b5df..f1abe0426e 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -15,6 +15,7 @@ from xmodule.exceptions import InvalidVersionError from opaque_keys.edx.locations import SlashSeparatedCourseKey from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator + # Mixed modulestore depends on django, so we'll manually configure some django settings # before importing the module # TODO remove this import and the configuration -- xmodule should not depend on django! @@ -26,6 +27,7 @@ if not settings.configured: settings.configure() from xmodule.modulestore.mixed import MixedModuleStore from xmodule.modulestore.draft_and_published import UnsupportedRevisionError +from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST @ddt.ddt @@ -34,8 +36,8 @@ class TestMixedModuleStore(unittest.TestCase): Quasi-superclass which tests Location based apps against both split and mongo dbs (Locator and Location-based dbs) """ - HOST = 'localhost' - PORT = 27017 + HOST = MONGO_HOST + PORT = MONGO_PORT_NUM DB = 'test_mongo_%s' % uuid4().hex[:5] COLLECTION = 'modulestore' FS_ROOT = DATA_DIR @@ -54,6 +56,7 @@ class TestMixedModuleStore(unittest.TestCase): } DOC_STORE_CONFIG = { 'host': HOST, + 'port': PORT, 'db': DB, 'collection': COLLECTION, } diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py index 15ba65aa9e..3cefa70203 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py @@ -36,12 +36,12 @@ from xmodule.exceptions import NotFoundError from git.test.lib.asserts import assert_not_none from xmodule.x_module import XModuleMixin from xmodule.modulestore.mongo.base import as_draft - +from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST log = logging.getLogger(__name__) -HOST = 'localhost' -PORT = 27017 +HOST = MONGO_HOST +PORT = MONGO_PORT_NUM DB = 'test_mongo_%s' % uuid4().hex[:5] COLLECTION = 'modulestore' FS_ROOT = DATA_DIR # TODO (vshnayder): will need a real fs_root for testing load_item @@ -91,12 +91,13 @@ class TestMongoModuleStore(unittest.TestCase): # connect to the db doc_store_config = { 'host': HOST, + 'port': PORT, 'db': DB, 'collection': COLLECTION, } # since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class # as well - content_store = MongoContentStore(HOST, DB) + content_store = MongoContentStore(HOST, DB, port=PORT) # # Also test draft store imports # @@ -148,7 +149,7 @@ class TestMongoModuleStore(unittest.TestCase): def test_mongo_modulestore_type(self): store = DraftModuleStore( None, - {'host': HOST, 'db': DB, 'collection': COLLECTION}, + {'host': HOST, 'db': DB, 'port': PORT, 'collection': COLLECTION}, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS ) assert_equals(store.get_modulestore_type(''), ModuleStoreEnum.Type.mongo) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py index 9ede47a126..abe634ed88 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py @@ -22,6 +22,7 @@ from xmodule.x_module import XModuleMixin from xmodule.fields import Date, Timedelta from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore from xmodule.modulestore.tests.test_modulestore import check_has_course_method +from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST BRANCH_NAME_DRAFT = ModuleStoreEnum.BranchName.draft @@ -36,8 +37,9 @@ class SplitModuleTest(unittest.TestCase): ''' # Snippets of what would be in the django settings envs file DOC_STORE_CONFIG = { - 'host': 'localhost', + 'host': MONGO_HOST, 'db': 'test_xmodule', + 'port': MONGO_PORT_NUM, 'collection': 'modulestore{0}'.format(uuid.uuid4().hex[:5]), } modulestore_options = { diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py index a2e2c2dcb7..da3c270641 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_split_w_old_mongo.py @@ -9,6 +9,7 @@ from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore from xmodule.modulestore.mongo import DraftMongoModuleStore from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST class SplitWMongoCourseBoostrapper(unittest.TestCase): @@ -27,7 +28,8 @@ class SplitWMongoCourseBoostrapper(unittest.TestCase): """ # Snippet of what would be in the django settings envs file db_config = { - 'host': 'localhost', + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, 'db': 'test_xmodule', } diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py b/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py index cf4a2f971e..4882c42734 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py @@ -10,6 +10,7 @@ from opaque_keys.edx.locations import Location from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.inheritance import InheritanceMixin from xmodule.modulestore.xml_importer import _import_module_and_update_references +from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST from opaque_keys.edx.locations import SlashSeparatedCourseKey from xmodule.tests import DATA_DIR from uuid import uuid4 @@ -21,8 +22,8 @@ class ModuleStoreNoSettings(unittest.TestCase): """ A mixin to create a mongo modulestore that avoids settings """ - HOST = 'localhost' - PORT = 27017 + HOST = MONGO_HOST + PORT = MONGO_PORT_NUM DB = 'test_mongo_%s' % uuid4().hex[:5] COLLECTION = 'modulestore' FS_ROOT = DATA_DIR @@ -36,6 +37,7 @@ class ModuleStoreNoSettings(unittest.TestCase): } DOC_STORE_CONFIG = { 'host': HOST, + 'port': PORT, 'db': DB, 'collection': COLLECTION, } diff --git a/lms/djangoapps/dashboard/git_import.py b/lms/djangoapps/dashboard/git_import.py index 9f0788d011..1fcf322ec1 100644 --- a/lms/djangoapps/dashboard/git_import.py +++ b/lms/djangoapps/dashboard/git_import.py @@ -128,6 +128,7 @@ def add_repo(repo, rdir_in, branch=None): # Set defaults even if it isn't defined in settings mongo_db = { 'host': 'localhost', + 'port': 27017, 'user': '', 'password': '', 'db': 'xlog', @@ -135,7 +136,7 @@ def add_repo(repo, rdir_in, branch=None): # Allow overrides if hasattr(settings, 'MONGODB_LOG'): - for config_item in ['host', 'user', 'password', 'db', ]: + for config_item in ['host', 'user', 'password', 'db', 'port']: mongo_db[config_item] = settings.MONGODB_LOG.get( config_item, mongo_db[config_item]) @@ -258,13 +259,13 @@ def add_repo(repo, rdir_in, branch=None): cwd=os.path.abspath(cdir))) # store import-command-run output in mongo - mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db) + mongouri = 'mongodb://{user}:{password}@{host}:{port}/{db}'.format(**mongo_db) try: if mongo_db['user'] and mongo_db['password']: mdb = mongoengine.connect(mongo_db['db'], host=mongouri) else: - mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host']) + mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'], port=mongo_db['port']) except mongoengine.connection.ConnectionError: log.exception('Unable to connect to mongodb to save log, please ' 'check MONGODB_LOG settings') diff --git a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py index a64a9c89cb..06cf8984ff 100644 --- a/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py +++ b/lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py @@ -1,7 +1,6 @@ """ Provide tests for git_add_course management command. """ - import logging import os import shutil @@ -21,9 +20,12 @@ from opaque_keys.edx.locations import SlashSeparatedCourseKey from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase import dashboard.git_import as git_import from dashboard.git_import import GitImportError +from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST + TEST_MONGODB_LOG = { - 'host': 'localhost', + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, 'user': '', 'password': '', 'db': 'test_xlog', diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py index 6f75c7a961..091f452553 100644 --- a/lms/djangoapps/dashboard/tests/test_sysadmin.py +++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py @@ -1,7 +1,6 @@ """ Provide tests for sysadmin dashboard feature in sysadmin.py """ - import glob import os import re @@ -30,10 +29,12 @@ from xmodule.modulestore.django import modulestore from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.xml import XMLModuleStore from opaque_keys.edx.locations import SlashSeparatedCourseKey +from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST TEST_MONGODB_LOG = { - 'host': 'localhost', + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, 'user': '', 'password': '', 'db': 'test_xlog', diff --git a/lms/envs/test.py b/lms/envs/test.py index 62b5002239..62f540007c 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -18,8 +18,14 @@ from path import path from warnings import filterwarnings, simplefilter from uuid import uuid4 +# mongo connection settings +MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) +MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost') + os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = 'localhost:8000-9000' +THIS_UUID = uuid4().hex[:5] + # can't test start dates with this True, but on the other hand, # can test everything else :) FEATURES['DISABLE_START_DATES'] = True @@ -118,16 +124,19 @@ update_module_store_settings( 'data_dir': COMMON_TEST_DATA_ROOT, }, doc_store_settings={ + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, 'db': 'test_xmodule', - 'collection': 'test_modulestore{0}'.format(uuid4().hex[:5]), + 'collection': 'test_modulestore{0}'.format(THIS_UUID), }, ) CONTENTSTORE = { 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', 'DOC_STORE_CONFIG': { - 'host': 'localhost', + 'host': MONGO_HOST, 'db': 'xcontent', + 'port': MONGO_PORT_NUM, } } @@ -338,3 +347,12 @@ VERIFY_STUDENT["SOFTWARE_SECURE"] = { VIDEO_CDN_URL = { 'CN': 'http://api.xuetangx.com/edx/video?s3_url=' } + +######### dashboard git log settings ######### +MONGODB_LOG = { + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, + 'user': '', + 'password': '', + 'db': 'xlog', +} diff --git a/pavelib/utils/test/utils.py b/pavelib/utils/test/utils.py index 82ec70915a..233cd0c9ff 100644 --- a/pavelib/utils/test/utils.py +++ b/pavelib/utils/test/utils.py @@ -3,6 +3,11 @@ Helper functions for test tasks """ from paver.easy import sh, task from pavelib.utils.envs import Env +import os + +MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) +MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost') + __test__ = False # do not collect @@ -42,4 +47,8 @@ def clean_mongo(): """ Clean mongo test databases """ - sh("mongo {repo_root}/scripts/delete-mongo-test-dbs.js".format(repo_root=Env.REPO_ROOT)) + sh("mongo {host}:{port} {repo_root}/scripts/delete-mongo-test-dbs.js".format( + host=MONGO_HOST, + port=MONGO_PORT_NUM, + repo_root=Env.REPO_ROOT, + )) From 422a6e20dbe297e84170d9616bf36110c1789f08 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Fri, 8 Aug 2014 09:36:08 -0700 Subject: [PATCH 003/137] Update the reference to gsehub/xblock-mentoring#4 and gsehub/xblock-mentoring#5 Port the Studio editor and remove the dataexport functionality Use the enforce_type feature to parse the bools --- requirements/edx/edx-private.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/edx/edx-private.txt b/requirements/edx/edx-private.txt index f2d081b5f4..1f3d45aa6d 100644 --- a/requirements/edx/edx-private.txt +++ b/requirements/edx/edx-private.txt @@ -1,7 +1,7 @@ # Requirements for edx.org that aren't necessarily needed for Open edX. -e git+ssh://git@github.com/jazkarta/edX-jsdraw.git@9fcd333aaa2ac3df65dd247b601ce0b56bb10cad#egg=edx-jsdraw --e git+https://github.com/gsehub/xblock-mentoring.git@e292496295dbb6e6d692015171a7dbaf45385321#egg=xblock-mentoring +-e git+https://github.com/gsehub/xblock-mentoring.git@d4532e4f89aaf36b56715be2abc0f9402912794e#egg=xblock-mentoring # Prototype XBlocks from edX learning sciences limited roll-outs and user testing. # Concept XBlock, in particular, is nowhere near finished and an early prototype. From aa1b9fece9e0cf681f82a2a8f14f0e4c7289b945 Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Mon, 11 Aug 2014 14:42:33 -0400 Subject: [PATCH 004/137] Properly (de)serialize references in old mongo LMS-11204 --- .../contentstore/views/tests/test_item.py | 2 +- .../xmodule/xmodule/modulestore/__init__.py | 4 +- .../lib/xmodule/xmodule/modulestore/mixed.py | 2 - .../xmodule/xmodule/modulestore/mongo/base.py | 76 ++++++++++--------- .../test_cross_modulestore_import_export.py | 2 + common/lib/xmodule/xmodule/tests/__init__.py | 23 +++++- 6 files changed, 64 insertions(+), 45 deletions(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_item.py b/cms/djangoapps/contentstore/views/tests/test_item.py index 9645ccbbf0..35097429c2 100644 --- a/cms/djangoapps/contentstore/views/tests/test_item.py +++ b/cms/djangoapps/contentstore/views/tests/test_item.py @@ -932,7 +932,7 @@ class TestEditSplitModule(ItemTest): # group_id_to_child and children have not changed yet. split_test = self._assert_children(2) - group_id_to_child = split_test.group_id_to_child + group_id_to_child = split_test.group_id_to_child.copy() self.assertEqual(2, len(group_id_to_child)) # Test environment and Studio use different module systems diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 25de6e22dc..df20eafec3 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -707,8 +707,8 @@ class EdxJSONEncoder(json.JSONEncoder): ISO date strings """ def default(self, obj): - if isinstance(obj, Location): - return obj.to_deprecated_string() + if isinstance(obj, (CourseKey, UsageKey)): + return unicode(obj) elif isinstance(obj, datetime.datetime): if obj.tzinfo is not None: if obj.utcoffset() is None: diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py index 91b4f1076d..1ade0dcb2b 100644 --- a/common/lib/xmodule/xmodule/modulestore/mixed.py +++ b/common/lib/xmodule/xmodule/modulestore/mixed.py @@ -170,8 +170,6 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): # return the default store return self.default_modulestore - # return the first store, as the default - return self.default_modulestore @property def default_modulestore(self): diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index e2592168d2..63a761ec87 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -238,10 +238,11 @@ class CachingDescriptorSystem(MakoDescriptorSystem): if not edit_info: module.edited_by = module.edited_on = module.subtree_edited_on = \ module.subtree_edited_by = module.published_date = None + raw_metadata = json_data.get('metadata', {}) # published_date was previously stored as a list of time components instead of a datetime - if metadata.get('published_date'): - module.published_date = datetime(*metadata.get('published_date')[0:6]).replace(tzinfo=UTC) - module.published_by = metadata.get('published_by') + if raw_metadata.get('published_date'): + module.published_date = datetime(*raw_metadata.get('published_date')[0:6]).replace(tzinfo=UTC) + module.published_by = raw_metadata.get('published_by') # otherwise restore the stored editing information else: module.edited_by = edit_info.get('edited_by') @@ -280,22 +281,26 @@ class CachingDescriptorSystem(MakoDescriptorSystem): :param course_key: a CourseKey object for the given course :param jsonfields: a dict of the jsonified version of the fields """ + result = {} for field_name, value in jsonfields.iteritems(): - if value: - field = class_.fields.get(field_name) - if field is None: - continue - elif isinstance(field, Reference): - jsonfields[field_name] = self._convert_reference_to_key(value) - elif isinstance(field, ReferenceList): - jsonfields[field_name] = [ - self._convert_reference_to_key(ele) for ele in value - ] - elif isinstance(field, ReferenceValueDict): - for key, subvalue in value.iteritems(): - assert isinstance(subvalue, basestring) - value[key] = self._convert_reference_to_key(subvalue) - return jsonfields + field = class_.fields.get(field_name) + if field is None: + continue + elif value is None: + result[field_name] = value + elif isinstance(field, Reference): + result[field_name] = self._convert_reference_to_key(value) + elif isinstance(field, ReferenceList): + result[field_name] = [ + self._convert_reference_to_key(ele) for ele in value + ] + elif isinstance(field, ReferenceValueDict): + result[field_name] = { + key: self._convert_reference_to_key(subvalue) for key, subvalue in value.iteritems() + } + else: + result[field_name] = value + return result def lookup_item(self, location): """ @@ -1125,14 +1130,11 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): therefore propagate subtree edit info up the tree """ try: - definition_data = self._convert_reference_fields_to_strings( - xblock, - xblock.get_explicitly_set_fields_by_scope() - ) + definition_data = self._serialize_scope(xblock, Scope.content) now = datetime.now(UTC) payload = { 'definition.data': definition_data, - 'metadata': self._convert_reference_fields_to_strings(xblock, own_metadata(xblock)), + 'metadata': self._serialize_scope(xblock, Scope.settings), 'edit_info.edited_on': now, 'edit_info.edited_by': user_id, 'edit_info.subtree_edited_on': now, @@ -1144,7 +1146,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): payload['edit_info.published_by'] = user_id if xblock.has_children: - children = self._convert_reference_fields_to_strings(xblock, {'children': xblock.children}) + children = self._serialize_scope(xblock, Scope.children) payload.update({'definition.children': children['children']}) self._update_single_item(xblock.scope_ids.usage_id, payload) @@ -1185,25 +1187,27 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): return xblock - def _convert_reference_fields_to_strings(self, xblock, jsonfields): + def _serialize_scope(self, xblock, scope): """ Find all fields of type reference and convert the payload from UsageKeys to deprecated strings :param xblock: the XBlock class :param jsonfields: a dict of the jsonified version of the fields """ - assert isinstance(jsonfields, dict) - for field_name, value in jsonfields.iteritems(): - if value: - if isinstance(xblock.fields[field_name], Reference): - jsonfields[field_name] = value.to_deprecated_string() - elif isinstance(xblock.fields[field_name], ReferenceList): + jsonfields = {} + for field_name, field in xblock.fields.iteritems(): + if (field.scope == scope and field.is_set_on(xblock)): + if isinstance(field, Reference): + jsonfields[field_name] = field.read_from(xblock).to_deprecated_string() + elif isinstance(field, ReferenceList): jsonfields[field_name] = [ - ele.to_deprecated_string() for ele in value + ele.to_deprecated_string() for ele in field.read_from(xblock) ] - elif isinstance(xblock.fields[field_name], ReferenceValueDict): - for key, subvalue in value.iteritems(): - assert isinstance(subvalue, UsageKey) - value[key] = subvalue.to_deprecated_string() + elif isinstance(field, ReferenceValueDict): + jsonfields[field_name] = { + key: unicode(subvalue) for key, subvalue in field.read_from(xblock).iteritems() + } + else: + jsonfields[field_name] = field.read_json(xblock) return jsonfields def _get_raw_parent_location(self, location, revision=ModuleStoreEnum.RevisionOption.published_only): diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py index 6ef5262c17..e20c36c13b 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py @@ -229,6 +229,8 @@ CONTENTSTORE_SETUPS = (MongoContentstoreBuilder(),) COURSE_DATA_NAMES = ( 'toy', 'manual-testing-complete', + 'split_test_module', + 'split_test_module_draft', ) diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py index dfd1a95b7a..e1f07c0266 100644 --- a/common/lib/xmodule/xmodule/tests/__init__.py +++ b/common/lib/xmodule/xmodule/tests/__init__.py @@ -16,7 +16,7 @@ from mock import Mock from path import path from xblock.field_data import DictFieldData -from xblock.fields import ScopeIds, Scope +from xblock.fields import ScopeIds, Scope, Reference, ReferenceList, ReferenceValueDict from xmodule.x_module import ModuleSystem, XModuleDescriptor, XModuleMixin from xmodule.modulestore.inheritance import InheritanceMixin, own_metadata @@ -159,6 +159,21 @@ class LogicTest(unittest.TestCase): return json.loads(self.xmodule.handle_ajax(dispatch, data)) +def map_references(value, field, actual_course_key): + """ + Map the references in value to actual_course_key and return value + """ + if not value: # if falsey + return value + if isinstance(field, Reference): + return value.map_into_course(actual_course_key) + if isinstance(field, ReferenceList): + return [sub.map_into_course(actual_course_key) for sub in value] + if isinstance(field, ReferenceValueDict): + return {key: ele.map_into_course(actual_course_key) for key, ele in value.iteritems()} + return value + + class CourseComparisonTest(unittest.TestCase): """ Mixin that has methods for comparing courses for equality. @@ -239,7 +254,7 @@ class CourseComparisonTest(unittest.TestCase): # compare fields self.assertEqual(expected_item.fields, actual_item.fields) - for field_name in expected_item.fields: + for field_name, field in expected_item.fields.iteritems(): if (expected_item.scope_ids.usage_id, field_name) in self.field_exclusions: continue @@ -250,8 +265,8 @@ class CourseComparisonTest(unittest.TestCase): if field_name == 'children': continue - exp_value = getattr(expected_item, field_name) - actual_value = getattr(actual_item, field_name) + exp_value = map_references(field.read_from(expected_item), field, actual_course_key) + actual_value = field.read_from(actual_item) self.assertEqual( exp_value, actual_value, From bdc4fd4d8630ef63ff4a5f582240b75b586018cd Mon Sep 17 00:00:00 2001 From: Christine Lytwynec Date: Thu, 14 Aug 2014 14:18:51 -0400 Subject: [PATCH 005/137] set root logging level for acceptance tests to ERROR --- cms/envs/acceptance.py | 3 +++ lms/envs/acceptance.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/cms/envs/acceptance.py b/cms/envs/acceptance.py index 7264394877..d2593d16a3 100644 --- a/cms/envs/acceptance.py +++ b/cms/envs/acceptance.py @@ -18,6 +18,9 @@ DEBUG = True import logging logging.basicConfig(filename=TEST_ROOT / "log" / "cms_acceptance.log", level=logging.ERROR) +# set root logger level +logging.getLogger().setLevel(logging.ERROR) + import os diff --git a/lms/envs/acceptance.py b/lms/envs/acceptance.py index 7494861fb9..c5d28db898 100644 --- a/lms/envs/acceptance.py +++ b/lms/envs/acceptance.py @@ -19,6 +19,9 @@ SITE_NAME = 'localhost:{}'.format(LETTUCE_SERVER_PORT) import logging logging.basicConfig(filename=TEST_ROOT / "log" / "lms_acceptance.log", level=logging.ERROR) +# set root logger level +logging.getLogger().setLevel(logging.ERROR) + import os from random import choice import string From 88ebac4faf3961e0803b7434d7b133cf46cd6b9e Mon Sep 17 00:00:00 2001 From: Will Daly Date: Thu, 14 Aug 2014 15:47:52 -0400 Subject: [PATCH 006/137] Use English for the marketing site buttons in an edx-controlled domain --- lms/djangoapps/courseware/tests/test_views.py | 67 +++++++++++++++---- lms/djangoapps/courseware/views.py | 20 +++++- 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/lms/djangoapps/courseware/tests/test_views.py b/lms/djangoapps/courseware/tests/test_views.py index b1e066e08e..540e725801 100644 --- a/lms/djangoapps/courseware/tests/test_views.py +++ b/lms/djangoapps/courseware/tests/test_views.py @@ -258,29 +258,44 @@ class ViewsTestCase(TestCase): self.assertIn('Coming Soon', response.content) def test_course_mktg_register(self): - admin = AdminFactory() - self.client.login(username=admin.username, password='test') - url = reverse('mktg_about_course', kwargs={'course_id': self.course_key.to_deprecated_string()}) - response = self.client.get(url) + response = self._load_mktg_about() self.assertIn('Register for', response.content) self.assertNotIn('and choose your student track', response.content) def test_course_mktg_register_multiple_modes(self): - admin = AdminFactory() - CourseMode.objects.get_or_create(mode_slug='honor', - mode_display_name='Honor Code Certificate', - course_id=self.course_key) - CourseMode.objects.get_or_create(mode_slug='verified', - mode_display_name='Verified Certificate', - course_id=self.course_key) - self.client.login(username=admin.username, password='test') - url = reverse('mktg_about_course', kwargs={'course_id': self.course_key.to_deprecated_string()}) - response = self.client.get(url) + CourseMode.objects.get_or_create( + mode_slug='honor', + mode_display_name='Honor Code Certificate', + course_id=self.course_key + ) + CourseMode.objects.get_or_create( + mode_slug='verified', + mode_display_name='Verified Certificate', + course_id=self.course_key + ) + + response = self._load_mktg_about() self.assertIn('Register for', response.content) self.assertIn('and choose your student track', response.content) # clean up course modes CourseMode.objects.all().delete() + @patch.dict(settings.FEATURES, {'IS_EDX_DOMAIN': True}) + def test_mktg_about_language_edx_domain(self): + # Since we're in an edx-controlled domain, and our marketing site + # supports only English, override the language setting + # and use English. + response = self._load_mktg_about(language='eo') + self.assertContains(response, "Register for") + + @patch.dict(settings.FEATURES, {'IS_EDX_DOMAIN': False}) + def test_mktg_about_language_openedx(self): + # If we're in an OpenEdX installation, + # may want to support languages other than English, + # so respect the language code. + response = self._load_mktg_about(language='eo') + self.assertContains(response, u"Régïstér för".encode('utf-8')) + def test_submission_history_accepts_valid_ids(self): # log into a staff account admin = AdminFactory() @@ -320,6 +335,30 @@ class ViewsTestCase(TestCase): response = self.client.get(url) self.assertFalse('" diff --git a/common/test/acceptance/tests/test_studio_container.py b/common/test/acceptance/tests/test_studio_container.py index 0d5c48ae4a..1af162e419 100644 --- a/common/test/acceptance/tests/test_studio_container.py +++ b/common/test/acceptance/tests/test_studio_container.py @@ -5,8 +5,6 @@ displaying containers within units. """ from nose.plugins.attrib import attr -from ..pages.studio.overview import CourseOutlinePage - from ..fixtures.course import XBlockFixtureDesc from ..pages.studio.component_editor import ComponentEditorView from ..pages.studio.html_component_editor import HtmlComponentEditorView @@ -16,87 +14,10 @@ from ..pages.lms.staff_view import StaffPage import datetime from bok_choy.promise import Promise, EmptyPromise -from .base_studio_test import StudioCourseTest - - -@attr('shard_1') -class ContainerBase(StudioCourseTest): - """ - Base class for tests that do operations on the container page. - """ - __test__ = False - - def setUp(self): - """ - Create a unique identifier for the course used in this test. - """ - # Ensure that the superclass sets up - super(ContainerBase, self).setUp() - - self.outline = CourseOutlinePage( - self.browser, - self.course_info['org'], - self.course_info['number'], - self.course_info['run'] - ) - - def go_to_nested_container_page(self): - """ - Go to the nested container page. - """ - unit = self.go_to_unit_page() - # The 0th entry is the unit page itself. - container = unit.xblocks[1].go_to_container() - return container - - def go_to_unit_page(self, section_name='Test Section', subsection_name='Test Subsection', unit_name='Test Unit'): - """ - Go to the test unit page. - - If make_draft is true, the unit page will be put into draft mode. - """ - self.outline.visit() - subsection = self.outline.section(section_name).subsection(subsection_name) - return subsection.toggle_expand().unit(unit_name).go_to() - - def verify_ordering(self, container, expected_orderings): - """ - Verifies the expected ordering of xblocks on the page. - """ - xblocks = container.xblocks - blocks_checked = set() - for expected_ordering in expected_orderings: - for xblock in xblocks: - parent = expected_ordering.keys()[0] - if xblock.name == parent: - blocks_checked.add(parent) - children = xblock.children - expected_length = len(expected_ordering.get(parent)) - self.assertEqual( - expected_length, len(children), - "Number of children incorrect for group {0}. Expected {1} but got {2}.".format(parent, expected_length, len(children))) - for idx, expected in enumerate(expected_ordering.get(parent)): - self.assertEqual(expected, children[idx].name) - blocks_checked.add(expected) - break - self.assertEqual(len(blocks_checked), len(xblocks)) - - def do_action_and_verify(self, action, expected_ordering): - """ - Perform the supplied action and then verify the resulting ordering. - """ - container = self.go_to_nested_container_page() - action(container) - - self.verify_ordering(container, expected_ordering) - - # Reload the page to see that the change was persisted. - container = self.go_to_nested_container_page() - self.verify_ordering(container, expected_ordering) +from .base_studio_test import ContainerBase class NestedVerticalTest(ContainerBase): - __test__ = False def populate_course_fixture(self, course_fixture): """ @@ -151,7 +72,6 @@ class DragAndDropTest(NestedVerticalTest): """ Tests of reordering within the container page. """ - __test__ = True def drag_and_verify(self, source, target, expected_ordering): self.do_action_and_verify( @@ -232,7 +152,6 @@ class AddComponentTest(NestedVerticalTest): """ Tests of adding a component to the container page. """ - __test__ = True def add_and_verify(self, menu_index, expected_ordering): self.do_action_and_verify( @@ -273,7 +192,6 @@ class DuplicateComponentTest(NestedVerticalTest): """ Tests of duplicating a component on the container page. """ - __test__ = True def duplicate_and_verify(self, source_index, expected_ordering): self.do_action_and_verify( @@ -320,7 +238,6 @@ class DeleteComponentTest(NestedVerticalTest): """ Tests of deleting a component from the container page. """ - __test__ = True def delete_and_verify(self, source_index, expected_ordering): self.do_action_and_verify( @@ -344,7 +261,6 @@ class EditContainerTest(NestedVerticalTest): """ Tests of editing a container. """ - __test__ = True def modify_display_name_and_verify(self, component): """ @@ -377,7 +293,6 @@ class UnitPublishingTest(ContainerBase): """ Tests of the publishing control and related widgets on the Unit page. """ - __test__ = True PUBLISHED_STATUS = "Publishing Status\nPublished (not yet released)" PUBLISHED_LIVE_STATUS = "Publishing Status\nPublished and Live" From 77f7f269336ff90034a9c81916e32873c43b3233 Mon Sep 17 00:00:00 2001 From: MarCnu Date: Fri, 1 Aug 2014 18:10:47 +0200 Subject: [PATCH 027/137] Add HTTP_RANGE compatibility for ContentStore file streaming Currently, users can only download ContentStore files from first byte to last byte. With this change, when a request to the ContentStore includes the HTTP "Range" parameter, it is parsed and StaticContentStream will stream the requested bytes. This change makes possible to stream video files (.mp4 especially) from the ContentStore. Users can now seek a specific time in the video without loading all the file. This is useful for courses with a limited number of students that doesn't require a dedicated video server. --- common/djangoapps/contentserver/middleware.py | 53 ++++++++- common/djangoapps/contentserver/tests/test.py | 96 +++++++++++++++-- .../xmodule/xmodule/contentstore/content.py | 19 +++- .../lib/xmodule/xmodule/tests/test_content.py | 101 +++++++++++++++++- 4 files changed, 259 insertions(+), 10 deletions(-) diff --git a/common/djangoapps/contentserver/middleware.py b/common/djangoapps/contentserver/middleware.py index e711fd2b66..0bcfbe3819 100644 --- a/common/djangoapps/contentserver/middleware.py +++ b/common/djangoapps/contentserver/middleware.py @@ -75,7 +75,58 @@ class StaticContentServer(object): if if_modified_since == last_modified_at_str: return HttpResponseNotModified() - response = HttpResponse(content.stream_data(), content_type=content.content_type) + # *** File streaming within a byte range *** + # If a Range is provided, parse Range attribute of the request + # Add Content-Range in the response if Range is structurally correct + # Request -> Range attribute structure: "Range: bytes=first-[last]" + # Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength" + response = None + if request.META.get('HTTP_RANGE'): + # Data from cache (StaticContent) has no easy byte management, so we use the DB instead (StaticContentStream) + if type(content) == StaticContent: + content = contentstore().find(loc, as_stream=True) + + # Let's parse the Range header, bytes=first-[last] + range_header = request.META['HTTP_RANGE'] + if '=' in range_header: + unit, byte_range = range_header.split('=') + # "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed + if unit == 'bytes' and '-' in byte_range: + first, last = byte_range.split('-') + # "first" must be a valid integer + try: + first = int(first) + except ValueError: + pass + if type(first) is int: + # "last" default value is the last byte of the file + # Users can ask "bytes=0-" to request the whole file when they don't know the length + try: + last = int(last) + except ValueError: + last = content.length - 1 + + if 0 <= first <= last < content.length: + # Valid Range attribute + response = HttpResponse(content.stream_data_in_range(first, last)) + response['Content-Range'] = 'bytes {first}-{last}/{length}'.format( + first=first, last=last, length=content.length + ) + response['Content-Length'] = str(last - first + 1) + response.status_code = 206 # HTTP_206_PARTIAL_CONTENT + if not response: + # Malformed Range attribute + response = HttpResponse() + response.status_code = 400 # HTTP_400_BAD_REQUEST + return response + + else: + # No Range attribute + response = HttpResponse(content.stream_data()) + response['Content-Length'] = content.length + + response['Accept-Ranges'] = 'bytes' + response['Content-Type'] = content.content_type response['Last-Modified'] = last_modified_at_str return response diff --git a/common/djangoapps/contentserver/tests/test.py b/common/djangoapps/contentserver/tests/test.py index 465b49709a..066551ac18 100644 --- a/common/djangoapps/contentserver/tests/test.py +++ b/common/djangoapps/contentserver/tests/test.py @@ -48,12 +48,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): # A locked asset self.locked_asset = self.course_key.make_asset_key('asset', 'sample_static.txt') self.url_locked = self.locked_asset.to_deprecated_string() + self.contentstore.set_attr(self.locked_asset, 'locked', True) # An unlocked asset self.unlocked_asset = self.course_key.make_asset_key('asset', 'another_static.txt') self.url_unlocked = self.unlocked_asset.to_deprecated_string() - - self.contentstore.set_attr(self.locked_asset, 'locked', True) + self.length_unlocked = self.contentstore.get_attr(self.unlocked_asset, 'length') def test_unlocked_asset(self): """ @@ -61,7 +61,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): """ self.client.logout() resp = self.client.get(self.url_unlocked) - self.assertEqual(resp.status_code, 200) # pylint: disable=E1103 + self.assertEqual(resp.status_code, 200) # pylint: disable=E1103 def test_locked_asset_not_logged_in(self): """ @@ -70,7 +70,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): """ self.client.logout() resp = self.client.get(self.url_locked) - self.assertEqual(resp.status_code, 403) # pylint: disable=E1103 + self.assertEqual(resp.status_code, 403) # pylint: disable=E1103 def test_locked_asset_not_registered(self): """ @@ -79,7 +79,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): """ self.client.login(username=self.non_staff_usr, password=self.non_staff_pwd) resp = self.client.get(self.url_locked) - self.assertEqual(resp.status_code, 403) # pylint: disable=E1103 + self.assertEqual(resp.status_code, 403) # pylint: disable=E1103 def test_locked_asset_registered(self): """ @@ -91,7 +91,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.client.login(username=self.non_staff_usr, password=self.non_staff_pwd) resp = self.client.get(self.url_locked) - self.assertEqual(resp.status_code, 200) # pylint: disable=E1103 + self.assertEqual(resp.status_code, 200) # pylint: disable=E1103 def test_locked_asset_staff(self): """ @@ -99,5 +99,87 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): """ self.client.login(username=self.staff_usr, password=self.staff_pwd) resp = self.client.get(self.url_locked) - self.assertEqual(resp.status_code, 200) # pylint: disable=E1103 + self.assertEqual(resp.status_code, 200) # pylint: disable=E1103 + def test_range_request_full_file(self): + """ + Test that a range request from byte 0 to last, + outputs partial content status code and valid Content-Range and Content-Length. + """ + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes=0-') + + self.assertEqual(resp.status_code, 206) # HTTP_206_PARTIAL_CONTENT + self.assertEqual(resp['Content-Range'], 'bytes {first}-{last}/{length}'.format( + first=0, last=self.length_unlocked-1, length=self.length_unlocked) + ) + self.assertEqual(resp['Content-Length'], str(self.length_unlocked)) + + def test_range_request_partial_file(self): + """ + Test that a range request for a partial file, + outputs partial content status code and valid Content-Range and Content-Length. + first_byte and last_byte are chosen to be simple but non trivial values. + """ + first_byte = self.length_unlocked / 4 + last_byte = self.length_unlocked / 2 + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}'.format( + first=first_byte, last=last_byte) + ) + + self.assertEqual(resp.status_code, 206) # HTTP_206_PARTIAL_CONTENT + self.assertEqual(resp['Content-Range'], 'bytes {first}-{last}/{length}'.format( + first=first_byte, last=last_byte, length=self.length_unlocked)) + self.assertEqual(resp['Content-Length'], str(last_byte - first_byte + 1)) + + def test_range_request_malformed_missing_equal(self): + """ + Test that a range request with malformed Range (missing '=') outputs status 400. + """ + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes 0-') + self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST + + def test_range_request_malformed_not_bytes(self): + """ + Test that a range request with malformed Range (not "bytes") outputs status 400. + "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed + """ + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bits=0-') + self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST + + def test_range_request_malformed_missing_minus(self): + """ + Test that a range request with malformed Range (missing '-') outputs status 400. + """ + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes=0') + self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST + + def test_range_request_malformed_first_not_integer(self): + """ + Test that a range request with malformed Range (first is not an integer) outputs status 400. + """ + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes=one-') + self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST + + def test_range_request_malformed_invalid_range(self): + """ + Test that a range request with malformed Range (first_byte > last_byte) outputs status 400. + """ + first_byte = self.length_unlocked / 2 + last_byte = self.length_unlocked / 4 + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}'.format( + first=first_byte, last=last_byte) + ) + + self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST + + def test_range_request_malformed_out_of_bounds(self): + """ + Test that a range request with malformed Range (last_byte == totalLength, offset by 1 error) + outputs status 400. + """ + last_byte = self.length_unlocked + resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes=0-{last}'.format( + last=last_byte) + ) + + self.assertEqual(resp.status_code, 400) # HTTP_400_BAD_REQUEST diff --git a/common/lib/xmodule/xmodule/contentstore/content.py b/common/lib/xmodule/xmodule/contentstore/content.py index 1d8a2cd23f..c45a56f391 100644 --- a/common/lib/xmodule/xmodule/contentstore/content.py +++ b/common/lib/xmodule/xmodule/contentstore/content.py @@ -5,6 +5,8 @@ XASSET_SRCREF_PREFIX = 'xasset:' XASSET_THUMBNAIL_TAIL_NAME = '.jpg' +STREAM_DATA_CHUNK_SIZE = 1024 + import os import logging import StringIO @@ -164,11 +166,26 @@ class StaticContentStream(StaticContent): def stream_data(self): while True: - chunk = self._stream.read(1024) + chunk = self._stream.read(STREAM_DATA_CHUNK_SIZE) if len(chunk) == 0: break yield chunk + def stream_data_in_range(self, first_byte, last_byte): + """ + Stream the data between first_byte and last_byte (included) + """ + self._stream.seek(first_byte) + position = first_byte + while True: + if last_byte < position + STREAM_DATA_CHUNK_SIZE - 1: + chunk = self._stream.read(last_byte - position + 1) + yield chunk + break + chunk = self._stream.read(STREAM_DATA_CHUNK_SIZE) + position += STREAM_DATA_CHUNK_SIZE + yield chunk + def close(self): self._stream.close() diff --git a/common/lib/xmodule/xmodule/tests/test_content.py b/common/lib/xmodule/xmodule/tests/test_content.py index 271d211422..f4e97f092f 100644 --- a/common/lib/xmodule/xmodule/tests/test_content.py +++ b/common/lib/xmodule/xmodule/tests/test_content.py @@ -1,8 +1,47 @@ import unittest -from xmodule.contentstore.content import StaticContent +from xmodule.contentstore.content import StaticContent, StaticContentStream from xmodule.contentstore.content import ContentStore from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation +SAMPLE_STRING = """ +This is a sample string with more than 1024 bytes, the default STREAM_DATA_CHUNK_SIZE + +Lorem Ipsum is simply dummy text of the printing and typesetting industry. +Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, +when an unknown printer took a galley of type and scrambled it to make a type +specimen book. It has survived not only five centuries, but also the leap into +electronic typesetting, remaining essentially unchanged. It was popularised in +the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, +nd more recently with desktop publishing software like Aldus PageMaker including +versions of Lorem Ipsum. + +It is a long established fact that a reader will be distracted by the readable +content of a page when looking at its layout. The point of using Lorem Ipsum is +that it has a more-or-less normal distribution of letters, as opposed to using +'Content here, content here', making it look like readable English. Many desktop +ublishing packages and web page editors now use Lorem Ipsum as their default model +text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. +Various versions have evolved over the years, sometimes by accident, sometimes on purpose +injected humour and the like). + +Lorem Ipsum is simply dummy text of the printing and typesetting industry. +Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, +when an unknown printer took a galley of type and scrambled it to make a type +specimen book. It has survived not only five centuries, but also the leap into +electronic typesetting, remaining essentially unchanged. It was popularised in +the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, +nd more recently with desktop publishing software like Aldus PageMaker including +versions of Lorem Ipsum. + +It is a long established fact that a reader will be distracted by the readable +content of a page when looking at its layout. The point of using Lorem Ipsum is +that it has a more-or-less normal distribution of letters, as opposed to using +'Content here, content here', making it look like readable English. Many desktop +ublishing packages and web page editors now use Lorem Ipsum as their default model +text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. +Various versions have evolved over the years, sometimes by accident, sometimes on purpose +injected humour and the like). +""" class Content: def __init__(self, location, content_type): @@ -10,6 +49,30 @@ class Content: self.content_type = content_type +class FakeGridFsItem: + """ + This class provides the basic methods to get data from a GridFS item + """ + def __init__(self, string_data): + self.cursor = 0 + self.data = string_data + self.length = len(string_data) + + def seek(self, position): + """ + Set the cursor at "position" + """ + self.cursor = position + + def read(self, chunk_size): + """ + Read "chunk_size" bytes of data at position cursor and move the cursor + """ + chunk = self.data[self.cursor:(self.cursor + chunk_size)] + self.cursor += chunk_size + return chunk + + class ContentTest(unittest.TestCase): def test_thumbnail_none(self): # We had a bug where a thumbnail location of None was getting transformed into a Location tuple, with @@ -46,3 +109,39 @@ class ContentTest(unittest.TestCase): AssetLocation(u'foo', u'bar', None, u'asset', u'images_course_image.jpg', None), asset_location ) + + def test_static_content_stream_stream_data(self): + """ + Test StaticContentStream stream_data function, asserts that we get all the bytes + """ + data = SAMPLE_STRING + item = FakeGridFsItem(data) + static_content_stream = StaticContentStream('loc', 'name', 'type', item, length=item.length) + + total_length = 0 + stream = static_content_stream.stream_data() + for chunck in stream: + total_length += len(chunck) + + self.assertEqual(total_length, static_content_stream.length) + + def test_static_content_stream_stream_data_in_range(self): + """ + Test StaticContentStream stream_data_in_range function, + asserts that we get the requested number of bytes + first_byte and last_byte are chosen to be simple but non trivial values + and to have total_length > STREAM_DATA_CHUNK_SIZE (1024) + """ + data = SAMPLE_STRING + item = FakeGridFsItem(data) + static_content_stream = StaticContentStream('loc', 'name', 'type', item, length=item.length) + + first_byte = 100 + last_byte = 1500 + + total_length = 0 + stream = static_content_stream.stream_data_in_range(first_byte, last_byte) + for chunck in stream: + total_length += len(chunck) + + self.assertEqual(total_length, last_byte - first_byte + 1) From fb0ee690f226dc4be608ac20de28efff7a83637a Mon Sep 17 00:00:00 2001 From: David Baumgold Date: Tue, 19 Aug 2014 11:49:43 -0400 Subject: [PATCH 028/137] Cleaning up a few quality violations --- common/djangoapps/contentserver/middleware.py | 10 ++++++---- common/djangoapps/contentserver/tests/test.py | 14 ++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/common/djangoapps/contentserver/middleware.py b/common/djangoapps/contentserver/middleware.py index 0bcfbe3819..d32470ea15 100644 --- a/common/djangoapps/contentserver/middleware.py +++ b/common/djangoapps/contentserver/middleware.py @@ -1,5 +1,6 @@ -from django.http import (HttpResponse, HttpResponseNotModified, - HttpResponseForbidden) +from django.http import ( + HttpResponse, HttpResponseNotModified, HttpResponseForbidden +) from student.models import CourseEnrollment from xmodule.contentstore.django import contentstore @@ -13,6 +14,7 @@ from xmodule.exceptions import NotFoundError # TODO: Soon as we have a reasonable way to serialize/deserialize AssetKeys, we need # to change this file so instead of using course_id_partial, we're just using asset keys + class StaticContentServer(object): def process_request(self, request): # look to see if the request is prefixed with an asset prefix tag @@ -113,11 +115,11 @@ class StaticContentServer(object): first=first, last=last, length=content.length ) response['Content-Length'] = str(last - first + 1) - response.status_code = 206 # HTTP_206_PARTIAL_CONTENT + response.status_code = 206 # HTTP_206_PARTIAL_CONTENT if not response: # Malformed Range attribute response = HttpResponse() - response.status_code = 400 # HTTP_400_BAD_REQUEST + response.status_code = 400 # HTTP_400_BAD_REQUEST return response else: diff --git a/common/djangoapps/contentserver/tests/test.py b/common/djangoapps/contentserver/tests/test.py index 066551ac18..94d95d4d4a 100644 --- a/common/djangoapps/contentserver/tests/test.py +++ b/common/djangoapps/contentserver/tests/test.py @@ -42,8 +42,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall') - import_from_xml(modulestore(), self.user.id, 'common/test/data/', ['toy'], - static_content_store=self.contentstore, verbose=True) + import_from_xml( + modulestore(), self.user.id, 'common/test/data/', ['toy'], + static_content_store=self.contentstore, verbose=True + ) # A locked asset self.locked_asset = self.course_key.make_asset_key('asset', 'sample_static.txt') @@ -109,8 +111,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes=0-') self.assertEqual(resp.status_code, 206) # HTTP_206_PARTIAL_CONTENT - self.assertEqual(resp['Content-Range'], 'bytes {first}-{last}/{length}'.format( - first=0, last=self.length_unlocked-1, length=self.length_unlocked) + self.assertEqual( + resp['Content-Range'], + 'bytes {first}-{last}/{length}'.format( + first=0, last=self.length_unlocked - 1, + length=self.length_unlocked + ) ) self.assertEqual(resp['Content-Length'], str(self.length_unlocked)) From 0d63d0c1c10b8a5dcf02b414ad8c3ed3e51597b2 Mon Sep 17 00:00:00 2001 From: Daniel Friedman Date: Mon, 18 Aug 2014 13:25:24 -0400 Subject: [PATCH 029/137] Provide course outline template with start date --- cms/djangoapps/contentstore/views/course.py | 6 ++++ .../views/tests/test_course_index.py | 35 +++++++++++++++++++ cms/templates/course_outline.html | 1 + 3 files changed, 42 insertions(+) diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index 60d88027d3..5d36db91fc 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -16,8 +16,10 @@ from django.core.exceptions import PermissionDenied from django.core.urlresolvers import reverse from django.http import HttpResponseBadRequest, HttpResponseNotFound, HttpResponse from util.json_request import JsonResponse +from util.date_utils import get_default_time_display from edxmako.shortcuts import render_to_response +from xmodule.course_module import DEFAULT_START_DATE from xmodule.error_module import ErrorDescriptor from xmodule.modulestore.django import modulestore from xmodule.contentstore.content import StaticContent @@ -376,6 +378,8 @@ def course_index(request, course_key): sections = course_module.get_children() course_structure = _course_outline_json(request, course_module) locator_to_show = request.REQUEST.get('show', None) + course_release_date = get_default_time_display(course_module.start) if course_module.start != DEFAULT_START_DATE else _("Unscheduled") + settings_url = reverse_course_url('settings_handler', course_key) try: current_action = CourseRerunState.objects.find_first(course_key=course_key, should_display=True) @@ -392,6 +396,8 @@ def course_index(request, course_key): CourseGradingModel.fetch(course_key).graders ), 'rerun_notification_id': current_action.id if current_action else None, + 'course_release_date': course_release_date, + 'settings_url': settings_url, }) diff --git a/cms/djangoapps/contentstore/views/tests/test_course_index.py b/cms/djangoapps/contentstore/views/tests/test_course_index.py index e92ef9053d..cf9d99da39 100644 --- a/cms/djangoapps/contentstore/views/tests/test_course_index.py +++ b/cms/djangoapps/contentstore/views/tests/test_course_index.py @@ -3,6 +3,7 @@ Unit tests for getting the list of courses and the course outline. """ import json import lxml +import datetime from contentstore.tests.utils import CourseTestCase from contentstore.utils import reverse_course_url, add_instructor @@ -10,6 +11,8 @@ from contentstore.views.access import has_course_access from contentstore.views.course import course_outline_initial_state from contentstore.views.item import create_xblock_info, VisibilityState from course_action_state.models import CourseRerunState +from util.date_utils import get_default_time_display +from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory from opaque_keys.edx.locator import CourseLocator @@ -273,3 +276,35 @@ class TestCourseOutline(CourseTestCase): expanded_locators = initial_state['expanded_locators'] self.assertIn(unicode(self.sequential.location), expanded_locators) self.assertIn(unicode(self.vertical.location), expanded_locators) + + def test_start_date_on_page(self): + """ + Verify that the course start date is included on the course outline page. + """ + def _get_release_date(response): + """Return the release date from the course page""" + parsed_html = lxml.html.fromstring(response.content) + return parsed_html.find_class('course-status')[0].find_class('status-release-value')[0].text_content() + + def _assert_settings_link_present(response): + """ + Asserts there's a course settings link on the course page by the course release date. + """ + parsed_html = lxml.html.fromstring(response.content) + settings_link = parsed_html.find_class('course-status')[0].find_class('action-edit')[0].find('a') + self.assertIsNotNone(settings_link) + self.assertEqual(settings_link.get('href'), reverse_course_url('settings_handler', self.course.id)) + + outline_url = reverse_course_url('course_handler', self.course.id) + response = self.client.get(outline_url, {}, HTTP_ACCEPT='text/html') + + # A course with the default release date should display as "Unscheduled" + self.assertEqual(_get_release_date(response), 'Unscheduled') + _assert_settings_link_present(response) + + self.course.start = datetime.datetime(2014, 1, 1) + modulestore().update_item(self.course, ModuleStoreEnum.UserID.test) + response = self.client.get(outline_url, {}, HTTP_ACCEPT='text/html') + + self.assertEqual(_get_release_date(response), get_default_time_display(self.course.start)) + _assert_settings_link_present(response) diff --git a/cms/templates/course_outline.html b/cms/templates/course_outline.html index 8940d18bcf..d08ae0fa3b 100644 --- a/cms/templates/course_outline.html +++ b/cms/templates/course_outline.html @@ -42,6 +42,7 @@ from contentstore.utils import reverse_usage_url

${_("Content")} > ${_("Course Outline")} + ${course_release_date}