From f1406953e06f22a6957a6d82edc81552997802f7 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 19 Jul 2012 12:42:10 -0400 Subject: [PATCH 001/124] adding prod-requirements to match puppet configuration --- prod-requirements.txt | 51 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 prod-requirements.txt diff --git a/prod-requirements.txt b/prod-requirements.txt new file mode 100644 index 0000000000..ff4ca151f9 --- /dev/null +++ b/prod-requirements.txt @@ -0,0 +1,51 @@ +Django==1.3.1 +flup==1.0.3.dev-20110405 +lxml==2.3.4 +Mako==0.7.0 +Markdown==2.1.1 +markdown2==1.4.2 +python-memcached==1.48 +numpy==1.6.1 +Pygments==1.5 +boto==2.3.0 +django-storages==1.1.4 +django-masquerade==0.1.5 +fs==0.4.0 +django-jasmine==0.3.2 +path.py==2.2.2 +requests==0.12.1 +BeautifulSoup==3.2.1 +newrelic==1.2.1.265 +ipython==0.12.1 +django-pipeline==1.2.12 +django-staticfiles==1.2.1 +glob2==0.3 +sympy==0.7.1 +pymongo==2.2.1 +rednose==0.3.3 +mock==0.8.0 +GitPython==0.3.2.RC1 +PyYAML==3.10 +feedparser==5.1.2 +MySQL-python==1.2.3 +matplotlib==1.1.0 +scipy==0.10.1 +akismet==0.2.0 +Coffin==0.3.6 +django-celery==2.2.7 +django-countries==1.0.5 +django-followit==0.0.3 +django-keyedcache==1.4-6 +django-kombu==0.9.2 +django-mako==0.1.5pre +django-recaptcha-works==0.3.4 +django-robots==0.8.1 +django-ses==0.4.1 +django-threaded-multihost==1.4-1 +html5lib==0.90 +Jinja2==2.6 +oauth2==1.5.211 +pystache==0.3.1 +python-openid==2.2.5 +South==0.7.5 +Unidecode==0.04.9 From 68132b851715d89fd6682b6978a4d0ba8dc706f8 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 26 Jul 2012 17:13:01 -0400 Subject: [PATCH 002/124] removing mercurial dependency from linux/OSx --- create-dev-env.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index 91f9b81931..4d911dde96 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -110,7 +110,7 @@ NUMPY_VER="1.6.2" SCIPY_VER="0.10.1" BREW_FILE="$BASE/mitx/brew-formulas.txt" LOG="/var/tmp/install.log" -APT_PKGS="curl git mercurial python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript" +APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript" if [[ $EUID -eq 0 ]]; then error "This script should not be run using sudo or as the root user" @@ -209,10 +209,6 @@ case `uname -s` in output "Installing git" brew install git >> $LOG } - command -v hg &>/dev/null || { - output "Installaing mercurial" - brew install mercurial >> $LOG - } clone_repos From c8b2b4d464ff1fba507862a9e0fe2bf5499ef298 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 27 Jul 2012 10:14:07 -0400 Subject: [PATCH 003/124] Miniture staff dashboard. Not really tested, but should be safe to merge --- lms/djangoapps/dashboard/views.py | 29 +++++++++++++++++++++++++++++ lms/urls.py | 1 + 2 files changed, 30 insertions(+) create mode 100644 lms/djangoapps/dashboard/views.py diff --git a/lms/djangoapps/dashboard/views.py b/lms/djangoapps/dashboard/views.py new file mode 100644 index 0000000000..ba6c8ca152 --- /dev/null +++ b/lms/djangoapps/dashboard/views.py @@ -0,0 +1,29 @@ +# Create your views here. +import json +from datetime import datetime +from django.http import HttpResponse, Http404 + +def dictfetchall(cursor): + '''Returns all rows from a cursor as a dict. + Borrowed from Django documentation''' + desc = cursor.description + return [ + dict(zip([col[0] for col in desc], row)) + for row in cursor.fetchall() + ] + +def dashboard(request): + """ + Simple view that a loadbalancer can check to verify that the app is up + """ + if not request.user.is_staff: + raise Http404 + + query = "select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc" + + from django.db import connection + cursor = connection.cursor() + results = dictfetchall(cursor.execute(query)) + + + return HttpResponse(json.dumps(results, indent=4)) diff --git a/lms/urls.py b/lms/urls.py index 0785cd96d0..3667397c8c 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -13,6 +13,7 @@ if settings.DEBUG: urlpatterns = ('', url(r'^$', 'student.views.index', name="root"), # Main marketing page, or redirect to courseware url(r'^dashboard$', 'student.views.dashboard', name="dashboard"), + url(r'^admin_dashboard$', 'dashboard.views.dashboard'), url(r'^change_email$', 'student.views.change_email_request'), url(r'^email_confirm/(?P[^/]*)$', 'student.views.confirm_email_change'), From 6db146f6b49a1a5310cf1be04bace51090ec5fba Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 27 Jul 2012 10:19:51 -0400 Subject: [PATCH 004/124] Missing files --- lms/djangoapps/dashboard/__init__.py | 0 lms/djangoapps/dashboard/models.py | 3 +++ lms/djangoapps/dashboard/tests.py | 16 ++++++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 lms/djangoapps/dashboard/__init__.py create mode 100644 lms/djangoapps/dashboard/models.py create mode 100644 lms/djangoapps/dashboard/tests.py diff --git a/lms/djangoapps/dashboard/__init__.py b/lms/djangoapps/dashboard/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/dashboard/models.py b/lms/djangoapps/dashboard/models.py new file mode 100644 index 0000000000..71a8362390 --- /dev/null +++ b/lms/djangoapps/dashboard/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/lms/djangoapps/dashboard/tests.py b/lms/djangoapps/dashboard/tests.py new file mode 100644 index 0000000000..501deb776c --- /dev/null +++ b/lms/djangoapps/dashboard/tests.py @@ -0,0 +1,16 @@ +""" +This file demonstrates writing tests using the unittest module. These will pass +when you run "manage.py test". + +Replace this with more appropriate tests for your application. +""" + +from django.test import TestCase + + +class SimpleTest(TestCase): + def test_basic_addition(self): + """ + Tests that 1 + 1 always equals 2. + """ + self.assertEqual(1 + 1, 2) From 2df7e02e310516bbf2649988edb58e79b829ccf4 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 27 Jul 2012 10:23:44 -0400 Subject: [PATCH 005/124] Fixed comment --- lms/djangoapps/dashboard/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/dashboard/views.py b/lms/djangoapps/dashboard/views.py index ba6c8ca152..5cf3fa72c1 100644 --- a/lms/djangoapps/dashboard/views.py +++ b/lms/djangoapps/dashboard/views.py @@ -14,7 +14,9 @@ def dictfetchall(cursor): def dashboard(request): """ - Simple view that a loadbalancer can check to verify that the app is up + Quick hack to show staff enrollment numbers. This should be + replaced with a real dashboard later. This version is a short-term + bandaid for the next couple weeks. """ if not request.user.is_staff: raise Http404 From c51f78bdfad4900e9e112f9d1abf831e5a226a79 Mon Sep 17 00:00:00 2001 From: Kyle Fiedler Date: Tue, 24 Jul 2012 16:54:31 -0400 Subject: [PATCH 006/124] Added styles for course images to always the right dimentions --- lms/static/sass/multicourse/_course_about.scss | 13 +++++++------ lms/templates/portal/course_about.html | 2 ++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lms/static/sass/multicourse/_course_about.scss b/lms/static/sass/multicourse/_course_about.scss index f96d12f7a3..de09505252 100644 --- a/lms/static/sass/multicourse/_course_about.scss +++ b/lms/static/sass/multicourse/_course_about.scss @@ -27,14 +27,17 @@ max-width: 1200px; min-width: 760px; position: relative; - width: 100%; z-index: 2; - + > div.table { + display: table; + width: 100%; + } .intro { @include box-sizing(border-box); @include clearfix; - float: left; + display: table-cell; + vertical-align: middle; padding: 20px 20px; position: relative; width: flex-grid(8) + flex-gutter(); @@ -130,9 +133,7 @@ .media { background: transparent; @include box-sizing(border-box); - display: block; - float: right; - height: 180px; + display: table-cell; padding: 20px; position: relative; width: flex-grid(4); diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html index bb00a6abf0..7780d939de 100644 --- a/lms/templates/portal/course_about.html +++ b/lms/templates/portal/course_about.html @@ -51,6 +51,7 @@
+

${course.number}: ${get_course_about_section(course, "title")}${get_course_about_section(course, "university")}

@@ -84,6 +85,7 @@
% endif
+
From b402cde65014f1d078e85505622bd4389c7fdced Mon Sep 17 00:00:00 2001 From: Kyle Fiedler Date: Tue, 24 Jul 2012 13:48:20 -0400 Subject: [PATCH 007/124] Added some style for the dashboard message --- lms/static/sass/multicourse/_dashboard.scss | 21 +++++++++++++++++++++ lms/templates/dashboard.html | 1 - 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lms/static/sass/multicourse/_dashboard.scss b/lms/static/sass/multicourse/_dashboard.scss index a3d21cb1b3..9c2b71f5c0 100644 --- a/lms/static/sass/multicourse/_dashboard.scss +++ b/lms/static/sass/multicourse/_dashboard.scss @@ -2,6 +2,27 @@ @include clearfix; padding: 60px 0px 120px; + .dashboard-banner { + background: $yellow; + border: 1px solid rgb(200,200,200); + @include box-shadow(0 1px 0 0 rgba(255,255,255, 0.6)); + padding: 10px; + margin-bottom: 30px; + + &:empty { + display: none; + background-color: #FFF; + } + + h2 { + margin-bottom: 0; + } + + p { + margin-bottom: 0; + } + } + .profile-sidebar { background: transparent; float: left; diff --git a/lms/templates/dashboard.html b/lms/templates/dashboard.html index b0d4308253..f1e19b6e5f 100644 --- a/lms/templates/dashboard.html +++ b/lms/templates/dashboard.html @@ -36,7 +36,6 @@
${message} -
From a2271d4e48992682d14b8c436a22d18fae6a2b09 Mon Sep 17 00:00:00 2001 From: Bridger Maxwell Date: Tue, 24 Jul 2012 13:52:13 -0400 Subject: [PATCH 008/124] The message div will only show up when there is a message to display. --- lms/templates/dashboard.html | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lms/templates/dashboard.html b/lms/templates/dashboard.html index f1e19b6e5f..160b453853 100644 --- a/lms/templates/dashboard.html +++ b/lms/templates/dashboard.html @@ -34,9 +34,11 @@
-
- ${message} -
+ %if message: +
+ ${message} +
+ %endif
From d30ae5389ee47d78a202d4cc9363f316e4f1e6d4 Mon Sep 17 00:00:00 2001 From: Kyle Fiedler Date: Tue, 31 Jul 2012 14:18:42 -0400 Subject: [PATCH 009/124] Added fix for videos start to zero out colors and mit styles --- .../xmodule/xmodule/css/sequence/display.scss | 31 +++++++++---------- .../xmodule/xmodule/css/video/display.scss | 9 ++++-- lms/envs/common.py | 2 +- .../sass/course/courseware/_courseware.scss | 3 ++ lms/templates/main.html | 2 +- 5 files changed, 27 insertions(+), 20 deletions(-) diff --git a/common/lib/xmodule/xmodule/css/sequence/display.scss b/common/lib/xmodule/xmodule/css/sequence/display.scss index d2ea986a4c..7658797725 100644 --- a/common/lib/xmodule/xmodule/css/sequence/display.scss +++ b/common/lib/xmodule/xmodule/css/sequence/display.scss @@ -3,8 +3,9 @@ nav.sequence-nav { // import from external sources. @extend .topbar; - border-bottom: 1px solid darken($cream, 20%); - margin-bottom: $body-line-height; + border-bottom: 1px solid #ddd; + margin: (-(lh())) (-(lh())) lh() (-(lh())); + background: #eee; position: relative; @include border-top-right-radius(4px); @@ -12,6 +13,8 @@ nav.sequence-nav { @include box-sizing(border-box); display: table; height: 100%; + margin: 0; + padding-left: 0; padding-right: flex-grid(1, 9); width: 100%; @@ -20,7 +23,7 @@ nav.sequence-nav { } li { - border-left: 1px solid darken($cream, 20%); + border-left: 1px solid #eee; display: table-cell; min-width: 20px; @@ -32,17 +35,15 @@ nav.sequence-nav { background-repeat: no-repeat; &:hover { - background-color: lighten($cream, 3%); + background-color: #eee; } } .visited { - background-color: #DCCDA2; + background-color: #ddd; background-repeat: no-repeat; - @include box-shadow(inset 0 0 3px darken(#dccda2, 10%)); &:hover { - background-color: $cream; background-position: center center; } } @@ -214,7 +215,7 @@ nav.sequence-nav { &.prev, &.next { a { - background-color: darken($cream, 5%); + // background-color: darken($cream, 5%); background-position: center center; background-repeat: no-repeat; border-left: 1px solid darken(#f6efd4, 20%); @@ -241,7 +242,7 @@ nav.sequence-nav { background-image: url('../images/sequence-nav/previous-icon.png'); &:hover { - background-color: $cream; + // background-color: $cream; } } } @@ -251,7 +252,7 @@ nav.sequence-nav { background-image: url('../images/sequence-nav/next-icon.png'); &:hover { - background-color: $cream; + // background-color: $cream; } } } @@ -273,9 +274,8 @@ nav.sequence-bottom { ul { @extend .clearfix; - background-color: darken(#F6EFD4, 5%); - background-color: darken($cream, 5%); - border: 1px solid darken(#f6efd4, 20%); + background-color: #eee; + border: 1px solid #ddd; @include border-radius(3px); @include box-shadow(inset 0 0 0 1px lighten(#f6efd4, 5%)); @include inline-block(); @@ -297,14 +297,13 @@ nav.sequence-bottom { width: 45px; &:hover { - background-color: $cream; - color: darken($cream, 60%); + background-color: #ddd; + color: #000; opacity: .5; text-decoration: none; } &.disabled { - background-color: lighten($cream, 10%); opacity: .4; } } diff --git a/common/lib/xmodule/xmodule/css/video/display.scss b/common/lib/xmodule/xmodule/css/video/display.scss index b7187cff26..789a267755 100644 --- a/common/lib/xmodule/xmodule/css/video/display.scss +++ b/common/lib/xmodule/xmodule/css/video/display.scss @@ -114,14 +114,13 @@ div.video { @extend .dullify; float: left; list-style: none; - margin-right: lh(); + margin: 0 lh() 0 0; padding: 0; li { float: left; margin-bottom: 0; - a { border-bottom: none; border-right: 1px solid #000; @@ -183,6 +182,8 @@ div.video { ol.video_speeds { display: block; opacity: 1; + padding: 0; + margin: 0; } } @@ -210,6 +211,7 @@ div.video { font-weight: normal; letter-spacing: 1px; padding: 0 lh(.25) 0 lh(.5); + line-height: 46px; text-transform: uppercase; } @@ -218,6 +220,7 @@ div.video { font-weight: bold; margin-bottom: 0; padding: 0 lh(.5) 0 0; + line-height: 46px; } &:hover, &:active, &:focus { @@ -422,10 +425,12 @@ div.video { } ol.subtitles { + padding-left: 0; float: left; max-height: 460px; overflow: auto; width: flex-grid(3, 9); + margin: 0; li { border: 0; diff --git a/lms/envs/common.py b/lms/envs/common.py index 032a088195..95daa913e8 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -304,7 +304,7 @@ PIPELINE_CSS = { 'output_filename': 'css/lms-application.css', }, 'course': { - 'source_filenames': ['sass/course.scss', 'js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css', 'css/vendor/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css'], + 'source_filenames': ['sass/course.scss', 'js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css'], 'output_filename': 'css/lms-course.css', }, 'ie-fixes': { diff --git a/lms/static/sass/course/courseware/_courseware.scss b/lms/static/sass/course/courseware/_courseware.scss index 8a0d880ceb..7fc3fb0fa1 100644 --- a/lms/static/sass/course/courseware/_courseware.scss +++ b/lms/static/sass/course/courseware/_courseware.scss @@ -59,6 +59,9 @@ div.course-wrapper { } ol.vert-mod { + padding: 0; + margin: 0; + > li { @extend .clearfix; @extend .problem-set; diff --git a/lms/templates/main.html b/lms/templates/main.html index 7ef8960970..fb502bfe22 100644 --- a/lms/templates/main.html +++ b/lms/templates/main.html @@ -3,7 +3,7 @@ <%block name="title">edX - + <%static:css group='application'/> From 1279c86097527e433b17f56e3e6b3dea3dc18982 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 14:56:39 -0400 Subject: [PATCH 010/124] by default now stdout and stdin will appear on console, both are logged --- create-dev-env.sh | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index 4d911dde96..808cccbcb3 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -17,8 +17,10 @@ ouch() { !! ERROR !! The last command did not complete successfully, - see $LOG for more details or trying running the + For more details or trying running the script again with the -v flag. + + Output of the script is recorded in $LOG EOL printf '\E[0m' @@ -61,26 +63,26 @@ clone_repos() { if [[ -d "$BASE/mitx/.git" ]]; then output "Pulling mitx" cd "$BASE/mitx" - git pull >>$LOG + git pull else output "Cloning mitx" if [[ -d "$BASE/mitx" ]]; then mv "$BASE/mitx" "${BASE}/mitx.bak.$$" fi - git clone git@github.com:MITx/mitx.git >>$LOG + git clone git@github.com:MITx/mitx.git fi cd "$BASE" if [[ -d "$BASE/askbot-devel/.git" ]]; then output "Pulling askbot-devel" cd "$BASE/askbot-devel" - git pull >>$LOG + git pull else output "Cloning askbot-devel" if [[ -d "$BASE/askbot-devel" ]]; then mv "$BASE/askbot-devel" "${BASE}/askbot-devel.bak.$$" fi - git clone git@github.com:MITx/askbot-devel >>$LOG + git clone git@github.com:MITx/askbot-devel fi # By default, dev environments start with a copy of 6.002x @@ -90,14 +92,14 @@ clone_repos() { if [[ -d "$BASE/data/$REPO/.git" ]]; then output "Pulling $REPO" cd "$BASE/data/$REPO" - git pull >>$LOG + git pull else output "Cloning $REPO" if [[ -d "$BASE/data/$REPO" ]]; then mv "$BASE/data/$REPO" "${BASE}/data/$REPO.bak.$$" fi cd "$BASE/data" - git clone git@github.com:MITx/$REPO >>$LOG + git clone git@github.com:MITx/$REPO fi } @@ -109,7 +111,7 @@ RUBY_VER="1.9.3" NUMPY_VER="1.6.2" SCIPY_VER="0.10.1" BREW_FILE="$BASE/mitx/brew-formulas.txt" -LOG="/var/tmp/install.log" +LOG="/var/tmp/install-$(date +%Y%m%d-%H%M%S).log" APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript" if [[ $EUID -eq 0 ]]; then @@ -163,15 +165,15 @@ cat< >(tee $LOG) +exec 2>&1 + if [[ -f $HOME/.rvmrc ]]; then output "$HOME/.rvmrc alredy exists, not adding $RUBY_DIR" else @@ -179,7 +181,6 @@ else echo "export rvm_path=$RUBY_DIR" > $HOME/.rvmrc fi mkdir -p $BASE -rm -f $LOG case `uname -s` in [Ll]inux) command -v lsb_release &>/dev/null || { @@ -207,7 +208,7 @@ case `uname -s` in } command -v git &>/dev/null || { output "Installing git" - brew install git >> $LOG + brew install git } clone_repos @@ -221,16 +222,16 @@ case `uname -s` in for pkg in $(cat $BREW_FILE); do grep $pkg <(brew list) &>/dev/null || { output "Installing $pkg" - brew install $pkg >>$LOG + brew install $pkg } done command -v pip &>/dev/null || { output "Installing pip" - sudo easy_install pip >>$LOG + sudo easy_install pip } command -v virtualenv &>/dev/null || { output "Installing virtualenv" - sudo pip install virtualenv virtualenvwrapper >> $LOG + sudo pip install virtualenv virtualenvwrapper } command -v coffee &>/dev/null || { output "Installing coffee script" @@ -273,24 +274,24 @@ if [[ -n $compile ]]; then rm -f numpy.tar.gz scipy.tar.gz output "Compiling numpy" cd "$BASE/numpy-${NUMPY_VER}" - python setup.py install >>$LOG 2>&1 + python setup.py install output "Compiling scipy" cd "$BASE/scipy-${SCIPY_VER}" - python setup.py install >>$LOG 2>&1 + python setup.py install cd "$BASE" rm -rf numpy-${NUMPY_VER} scipy-${SCIPY_VER} fi output "Installing askbot requirements" -pip install -r askbot-devel/askbot_requirements.txt >>$LOG +pip install -r askbot-devel/askbot_requirements.txt output "Installing askbot-dev requirements" -pip install -r askbot-devel/askbot_requirements_dev.txt >>$LOG +pip install -r askbot-devel/askbot_requirements_dev.txt output "Installing MITx pre-requirements" -pip install -r mitx/pre-requirements.txt >> $LOG +pip install -r mitx/pre-requirements.txt # Need to be in the mitx dir to get the paths to local modules right output "Installing MITx requirements" cd mitx -pip install -r requirements.txt >>$LOG +pip install -r requirements.txt mkdir "$BASE/log" || true mkdir "$BASE/db" || true From aaeb0446d125a51d2d3c2d01bac49b000aad681f Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 14:58:07 -0400 Subject: [PATCH 011/124] version is required http://gembundler.com/man/gemfile.5.html, this was causing rvm to return non-zero and thus causing the create-dev-env.sh script to fail --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index c6a19caca2..9ad08c7adb 100644 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,5 @@ source :rubygems - +ruby "1.9.3" gem 'rake' gem 'sass', '3.1.15' gem 'bourbon', '~> 1.3.6' From 5e8d132aadeb249e07efeac9a83eb3f43c61b400 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 15:15:49 -0400 Subject: [PATCH 012/124] removing askbot-devel --- create-dev-env.sh | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index 808cccbcb3..0f54cc0d7a 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -73,18 +73,6 @@ clone_repos() { fi cd "$BASE" - if [[ -d "$BASE/askbot-devel/.git" ]]; then - output "Pulling askbot-devel" - cd "$BASE/askbot-devel" - git pull - else - output "Cloning askbot-devel" - if [[ -d "$BASE/askbot-devel" ]]; then - mv "$BASE/askbot-devel" "${BASE}/askbot-devel.bak.$$" - fi - git clone git@github.com:MITx/askbot-devel - fi - # By default, dev environments start with a copy of 6.002x cd "$BASE" mkdir -p "$BASE/data" @@ -282,10 +270,6 @@ if [[ -n $compile ]]; then rm -rf numpy-${NUMPY_VER} scipy-${SCIPY_VER} fi -output "Installing askbot requirements" -pip install -r askbot-devel/askbot_requirements.txt -output "Installing askbot-dev requirements" -pip install -r askbot-devel/askbot_requirements_dev.txt output "Installing MITx pre-requirements" pip install -r mitx/pre-requirements.txt # Need to be in the mitx dir to get the paths to local modules right From 50dc4e80a371ab59b28a4e7a9960547084d3338e Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 15:49:11 -0400 Subject: [PATCH 013/124] adding askbot as a submodule of mitx --- create-dev-env.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/create-dev-env.sh b/create-dev-env.sh index 0f54cc0d7a..a5da139789 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -71,6 +71,16 @@ clone_repos() { fi git clone git@github.com:MITx/mitx.git fi + + if [[ -d "$BASE/mitx/askbot/.git" ]]; then + output "Pulling askbot" + cd "$BASE/mitx/askbot" + git pull + else + output "Cloning askbot as a submodule of mitx" + cd "$BASE/mitx" + git submodule update --init + fi cd "$BASE" # By default, dev environments start with a copy of 6.002x @@ -276,6 +286,10 @@ pip install -r mitx/pre-requirements.txt output "Installing MITx requirements" cd mitx pip install -r requirements.txt +output "Installing askbot requirements" +pip install -r askbot/askbot_requirements.txt +pip install -r askbot/askbot_requirements_dev.txt + mkdir "$BASE/log" || true mkdir "$BASE/db" || true From 09199631197bff05d91d77ccb02e17b79d44ecbb Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 16:07:00 -0400 Subject: [PATCH 014/124] adding option back for --system-site-packages, will ensure virtualenv > 1.7 --- create-dev-env.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index a5da139789..e3f67653e6 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -38,7 +38,7 @@ usage() { Usage: $PROG [-c] [-v] [-h] -c compile scipy and numpy - -s do _not_ set --no-site-packages for virtualenv + -s give access to global site-packages for virtualenv -v set -x + spew -h this @@ -229,7 +229,7 @@ case `uname -s` in } command -v virtualenv &>/dev/null || { output "Installing virtualenv" - sudo pip install virtualenv virtualenvwrapper + sudo pip install 'virtualenv>1.7' virtualenvwrapper } command -v coffee &>/dev/null || { output "Installing coffee script" @@ -248,10 +248,12 @@ curl -sL get.rvm.io | bash -s stable source $RUBY_DIR/scripts/rvm # skip the intro LESS="-E" rvm install $RUBY_VER -if [[ -n $systempkgs ]]; then - virtualenv "$PYTHON_DIR" +if [[ $systempkgs ]]; then + virtualenv --system-site-packages "$PYTHON_DIR" else - virtualenv --no-site-packages "$PYTHON_DIR" + # default behavior for virtualenv>1.7 is + # --no-site-packages + virtualenv "$PYTHON_DIR" fi source $PYTHON_DIR/bin/activate output "Installing gem bundler" From e08cb5067ed746b1c1a02b3ef68ed0539e6bc191 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 16:09:11 -0400 Subject: [PATCH 015/124] always pip install virtualenv --- create-dev-env.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index e3f67653e6..74a96a5f4b 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -227,10 +227,10 @@ case `uname -s` in output "Installing pip" sudo easy_install pip } - command -v virtualenv &>/dev/null || { - output "Installing virtualenv" - sudo pip install 'virtualenv>1.7' virtualenvwrapper - } + + output "Installing virtualenv" + sudo pip install 'virtualenv>1.7' virtualenvwrapper + command -v coffee &>/dev/null || { output "Installing coffee script" curl http://npmjs.org/install.sh | sh From 71b9dbbb2e61b2eb756681c39056dd885e3672de Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 14:18:04 -0400 Subject: [PATCH 016/124] Remove obsolete js references --- lms/templates/courseware.html | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lms/templates/courseware.html b/lms/templates/courseware.html index c1658b3dee..3d163c1f78 100644 --- a/lms/templates/courseware.html +++ b/lms/templates/courseware.html @@ -18,14 +18,6 @@ ## ## - ## image input: for clicking on images (see imageinput.html) - - - ## TODO (cpennington): Remove this when we have a good way for modules to specify js to load on the page - ## and in the wiki - - - <%static:js group='courseware'/> <%include file="mathjax_include.html" /> From f3567ddd6637170e71317e57f267f2c7a7824881 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 14:03:03 -0400 Subject: [PATCH 017/124] Change absolute paths to relative in scss * makes collectstatic happy --- cms/static/sass/_content-types.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cms/static/sass/_content-types.scss b/cms/static/sass/_content-types.scss index 00af06d5ad..e85d2a5c24 100644 --- a/cms/static/sass/_content-types.scss +++ b/cms/static/sass/_content-types.scss @@ -56,10 +56,10 @@ .module a:first-child { @extend .content-type; - background-image: url('/static/img/content-types/module.png'); + background-image: url('../img/content-types/module.png'); } .module a:first-child { @extend .content-type; - background-image: url('/static/img/content-types/module.png'); + background-image: url('../img/content-types/module.png'); } From 9d14033f6b6f5cc8951e0fd772b378ee10d73e98 Mon Sep 17 00:00:00 2001 From: Kyle Fiedler Date: Tue, 31 Jul 2012 15:52:27 -0400 Subject: [PATCH 018/124] Added some styles for the top navigation and for the sidebar navigation --- lms/envs/common.py | 2 +- lms/static/sass/course/base/_base.scss | 8 +++++++ lms/static/sass/course/base/_extends.scss | 4 ++-- .../sass/course/courseware/_courseware.scss | 16 ------------- .../sass/course/courseware/_sidebar.scss | 24 +++++++++++-------- .../course/layout/_courseware_subnav.scss | 15 ++++++------ lms/templates/courseware.html | 1 - 7 files changed, 33 insertions(+), 37 deletions(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index 95daa913e8..d89e6760a7 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -304,7 +304,7 @@ PIPELINE_CSS = { 'output_filename': 'css/lms-application.css', }, 'course': { - 'source_filenames': ['sass/course.scss', 'js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css'], + 'source_filenames': ['js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css', 'sass/course.scss'], 'output_filename': 'css/lms-course.css', }, 'ie-fixes': { diff --git a/lms/static/sass/course/base/_base.scss b/lms/static/sass/course/base/_base.scss index 02aba1866e..8849c78cfc 100644 --- a/lms/static/sass/course/base/_base.scss +++ b/lms/static/sass/course/base/_base.scss @@ -5,3 +5,11 @@ body { h1, h2, h3, h4, h5, h6 { font-family: $sans-serif; } + +table { + table-layout: fixed; +} + +.container { + padding: lh(2); +} diff --git a/lms/static/sass/course/base/_extends.scss b/lms/static/sass/course/base/_extends.scss index b71b8161f6..e1fc953126 100644 --- a/lms/static/sass/course/base/_extends.scss +++ b/lms/static/sass/course/base/_extends.scss @@ -84,7 +84,8 @@ h1.top-header { } .sidebar { - border-right: 1px solid #d3d3d3; + border-right: 1px solid #C8C8C8; + @include box-shadow(inset -1px 0 0 #e6e6e6); @include box-sizing(border-box); display: table-cell; font-family: $sans-serif; @@ -181,7 +182,6 @@ h1.top-header { .topbar { @extend .clearfix; - background: $cream; border-bottom: 1px solid darken($cream, 10%); border-top: 1px solid #fff; font-size: 12px; diff --git a/lms/static/sass/course/courseware/_courseware.scss b/lms/static/sass/course/courseware/_courseware.scss index 7fc3fb0fa1..77f53e0241 100644 --- a/lms/static/sass/course/courseware/_courseware.scss +++ b/lms/static/sass/course/courseware/_courseware.scss @@ -3,22 +3,6 @@ html { max-height: 100%; } -body.courseware { - height: 100%; - max-height: 100%; - - .container { - padding-bottom: 40px; - margin-top: 20px; - } - - footer { - &.fixed-bottom { - Position: static; - } - } -} - div.course-wrapper { @extend .table-wrapper; diff --git a/lms/static/sass/course/courseware/_sidebar.scss b/lms/static/sass/course/courseware/_sidebar.scss index fc0291f48a..860d588a84 100644 --- a/lms/static/sass/course/courseware/_sidebar.scss +++ b/lms/static/sass/course/courseware/_sidebar.scss @@ -13,44 +13,50 @@ section.course-index { div#accordion { h3 { - @include box-shadow(inset 0 1px 0 0 #eee); - border-top: 1px solid #d3d3d3; - overflow: hidden; + @include border-radius(0); + border-top: 1px solid #e3e3e3; margin: 0; + overflow: hidden; &:first-child { border: none; } &:hover { - @include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(225,225,225))); + background: #eee; } &.ui-accordion-header { color: #000; a { - font-size: $body-font-size; + @include border-radius(0); + @include box-shadow(none); color: lighten($text-color, 10%); + font-size: $body-font-size; } &.ui-state-active { - @include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(225,225,225))); @extend .active; - border-bottom: 1px solid #d3d3d3; + border-bottom: none; + + &:hover { + background: none; + } } } } ul.ui-accordion-content { @include border-radius(0); - @include box-shadow(inset -1px 0 0 #e6e6e6); + background: #FFF; border: none; font-size: 12px; margin: 0; padding: 1em 1.5em; li { + @include border-radius(0); margin-bottom: lh(.5); a { @@ -120,8 +126,6 @@ section.course-index { font-weight: bold; > a { - background: rgb(240,240,240); - @include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(230,230,230))); border-color: rgb(200,200,200); &:after { diff --git a/lms/static/sass/course/layout/_courseware_subnav.scss b/lms/static/sass/course/layout/_courseware_subnav.scss index 7d2433138e..21f6187a83 100644 --- a/lms/static/sass/course/layout/_courseware_subnav.scss +++ b/lms/static/sass/course/layout/_courseware_subnav.scss @@ -1,9 +1,8 @@ nav.course-material { - background: rgb(210,210,210); @include clearfix; @include box-sizing(border-box); - @include box-shadow(inset 0 1px 5px 0 rgba(0,0,0, 0.05)); - border-bottom: 1px solid rgb(190,190,190); + background: #f6f6f6; + border-bottom: 1px solid rgb(200,200,200); margin: 0px auto 0px; padding: 0px; width: 100%; @@ -24,12 +23,14 @@ nav.course-material { list-style: none; a { - color: $lighter-base-font-color; + color: darken($lighter-base-font-color, 20%); display: block; text-align: center; - padding: 5px 13px; + padding: 8px 13px 12px; + font-size: 14px; + font-weight: 400; text-decoration: none; - text-shadow: 0 1px rgba(255,255,255, 0.4); + text-shadow: 0 1px rgb(255,255,255); &:hover { color: $base-font-color; @@ -41,7 +42,7 @@ nav.course-material { border-bottom: 0px; @include border-top-radius(4px); @include box-shadow(0 2px 0 0 rgba(255,255,255, 1)); - color: $base-font-color; + color: $blue; } } } diff --git a/lms/templates/courseware.html b/lms/templates/courseware.html index 3d163c1f78..9c145ba8c0 100644 --- a/lms/templates/courseware.html +++ b/lms/templates/courseware.html @@ -35,7 +35,6 @@
-

Courseware Index

close
From b5b0d0862159f4bb98538e66968e9afab4de8a00 Mon Sep 17 00:00:00 2001 From: Kyle Fiedler Date: Tue, 31 Jul 2012 16:28:48 -0400 Subject: [PATCH 019/124] Sorta fix askbot --- lms/askbot/skins/mitx/templates/base.html | 2 +- .../templates/meta/html_head_stylesheets.html | 1 + .../mitx/templates/navigation.jinja.html | 55 ++++++------------ .../skins/mitx/templates/widgets/footer.html | 56 +++++++++++-------- lms/static/sass/course/_help.scss | 54 ------------------ .../sass/course/discussion/_discussion.scss | 4 +- 6 files changed, 55 insertions(+), 117 deletions(-) delete mode 100644 lms/static/sass/course/_help.scss diff --git a/lms/askbot/skins/mitx/templates/base.html b/lms/askbot/skins/mitx/templates/base.html index ced2376a99..18ca213cb7 100644 --- a/lms/askbot/skins/mitx/templates/base.html +++ b/lms/askbot/skins/mitx/templates/base.html @@ -20,7 +20,7 @@ {% include "widgets/header.html" %} {# Logo, user tool navigation and meta navitation #} {# include "widgets/secondary_header.html" #} {# Scope selector, search input and ask button #} -
+
{% block body %} {% endblock %}
diff --git a/lms/askbot/skins/mitx/templates/meta/html_head_stylesheets.html b/lms/askbot/skins/mitx/templates/meta/html_head_stylesheets.html index 99223edac4..3ec11b59fd 100644 --- a/lms/askbot/skins/mitx/templates/meta/html_head_stylesheets.html +++ b/lms/askbot/skins/mitx/templates/meta/html_head_stylesheets.html @@ -1,3 +1,4 @@ {% load extra_filters_jinja %} {{ 'application' | compressed_css }} +{{ 'course' | compressed_css }} diff --git a/lms/askbot/skins/mitx/templates/navigation.jinja.html b/lms/askbot/skins/mitx/templates/navigation.jinja.html index d69afbebc6..59c7148184 100644 --- a/lms/askbot/skins/mitx/templates/navigation.jinja.html +++ b/lms/askbot/skins/mitx/templates/navigation.jinja.html @@ -1,46 +1,27 @@ -
+
diff --git a/lms/askbot/skins/mitx/templates/widgets/footer.html b/lms/askbot/skins/mitx/templates/widgets/footer.html index 4675a07047..d152e2a38e 100644 --- a/lms/askbot/skins/mitx/templates/widgets/footer.html +++ b/lms/askbot/skins/mitx/templates/widgets/footer.html @@ -1,26 +1,38 @@ - + +
+ +
+
+

© 2012 edX, some rights reserved.

+
+ +
+ Terms of Service + Privacy Policy + Honor Code + Help +
+
+ + + diff --git a/lms/static/sass/course/_help.scss b/lms/static/sass/course/_help.scss deleted file mode 100644 index cb505814e9..0000000000 --- a/lms/static/sass/course/_help.scss +++ /dev/null @@ -1,54 +0,0 @@ -section.help.main-content { - padding: lh(); - - h1 { - border-bottom: 1px solid #ddd; - margin-bottom: lh(); - margin-top: 0; - padding-bottom: lh(); - } - - p { - max-width: 700px; - } - - h2 { - margin-top: 0; - } - - section.self-help { - float: left; - margin-bottom: lh(); - margin-right: flex-gutter(); - width: flex-grid(6); - - ul { - margin-left: flex-gutter(6); - - li { - margin-bottom: lh(.5); - } - } - } - - section.help-email { - float: left; - width: flex-grid(6); - - dl { - display: block; - margin-bottom: lh(); - - dd { - margin-bottom: lh(); - } - - dt { - clear: left; - float: left; - font-weight: bold; - width: flex-grid(2, 6); - } - } - } -} diff --git a/lms/static/sass/course/discussion/_discussion.scss b/lms/static/sass/course/discussion/_discussion.scss index b9022a43d8..7b0aa601d9 100644 --- a/lms/static/sass/course/discussion/_discussion.scss +++ b/lms/static/sass/course/discussion/_discussion.scss @@ -1,8 +1,6 @@ // Generic layout styles for the discussion forums - body.askbot { - - section.main-content { + section.container { div.discussion-wrapper { @extend .table-wrapper; From 37ba0395e00c9a499c7e1c5a17e642da7c76ac3e Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 16:49:18 -0400 Subject: [PATCH 020/124] do not install virtualenv unless it needs to be upgraded (<1.7) --- create-dev-env.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/create-dev-env.sh b/create-dev-env.sh index 74a96a5f4b..25f6c8d401 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -228,6 +228,11 @@ case `uname -s` in sudo easy_install pip } + if ! grep -Eq ^1.7 <(virtualenv --version 2>/dev/null); then + output "Installing virtualenv >1.7" + sudo pip install 'virtualenv>1.7' virtualenvwrapper + fi + output "Installing virtualenv" sudo pip install 'virtualenv>1.7' virtualenvwrapper From 9c31715da59dbee5658f732ae58c1e4fcb287470 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 16:55:07 -0400 Subject: [PATCH 021/124] removing virtualenv install lines that arent needed --- create-dev-env.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index 25f6c8d401..7fabb26f96 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -233,9 +233,6 @@ case `uname -s` in sudo pip install 'virtualenv>1.7' virtualenvwrapper fi - output "Installing virtualenv" - sudo pip install 'virtualenv>1.7' virtualenvwrapper - command -v coffee &>/dev/null || { output "Installing coffee script" curl http://npmjs.org/install.sh | sh From 5c73a6f2ca741a16981e9e24a1fdea4dddd3b7cb Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 17:04:58 -0400 Subject: [PATCH 022/124] betting rvmrc handling --- create-dev-env.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index 7fabb26f96..a369df6b3b 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -163,6 +163,11 @@ cat< >(tee $LOG) exec 2>&1 -if [[ -f $HOME/.rvmrc ]]; then - output "$HOME/.rvmrc alredy exists, not adding $RUBY_DIR" -else - output "Creating $HOME/.rmrc so rvm uses $RUBY_DIR" +if ! grep -q "export rvm_path=$RUBY_DIR" ~/.rvmrc; then + if [[ -f $HOME/.rvmrc ]]; then + output "Copying existing .rvmrc to .rvmrc.bak" + cp $HOME/.rvmrc $HOME/.rvmrc.bak + fi + output "Creating $HOME/.rvmrc so rvm uses $RUBY_DIR" echo "export rvm_path=$RUBY_DIR" > $HOME/.rvmrc fi + mkdir -p $BASE case `uname -s` in [Ll]inux) From ecab194e47f14c1fa89f1d1e5615df7f3f15f948 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 17:14:22 -0400 Subject: [PATCH 023/124] removing unnecessary chdir --- create-dev-env.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index a369df6b3b..d0f23ca6e2 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -82,7 +82,6 @@ clone_repos() { git submodule update --init fi - cd "$BASE" # By default, dev environments start with a copy of 6.002x cd "$BASE" mkdir -p "$BASE/data" From 0642af0147026ca344b89726d114799a9ce3943c Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 31 Jul 2012 17:17:33 -0400 Subject: [PATCH 024/124] submodules are not on a branch, no need to pull --- create-dev-env.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index d0f23ca6e2..7ebff77fb2 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -72,11 +72,7 @@ clone_repos() { git clone git@github.com:MITx/mitx.git fi - if [[ -d "$BASE/mitx/askbot/.git" ]]; then - output "Pulling askbot" - cd "$BASE/mitx/askbot" - git pull - else + if [[ ! -d "$BASE/mitx/askbot/.git" ]]; then output "Cloning askbot as a submodule of mitx" cd "$BASE/mitx" git submodule update --init From 381d24758961355410f69e0d95aee99ad9262742 Mon Sep 17 00:00:00 2001 From: ichuang Date: Tue, 31 Jul 2012 21:43:29 -0400 Subject: [PATCH 025/124] djangoapps/student/views.py now uses soup.getText, which needs beautifulsoup4 --- cms/envs/dev.py | 3 ++- common/djangoapps/student/views.py | 3 ++- requirements.txt | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cms/envs/dev.py b/cms/envs/dev.py index b0729ba885..59bc623729 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -37,7 +37,8 @@ REPOS = { }, 'content-mit-6002x': { 'branch': 'master', - 'origin': 'git@github.com:MITx/6002x-fall-2012.git', + #'origin': 'git@github.com:MITx/6002x-fall-2012.git', + 'origin': 'git@github.com:MITx/content-mit-6002x.git', }, '6.00x': { 'branch': 'master', diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index eeda9a6b65..b739e0e37c 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -23,7 +23,8 @@ from django.http import HttpResponse, Http404 from django.shortcuts import redirect from mitxmako.shortcuts import render_to_response, render_to_string from django.core.urlresolvers import reverse -from BeautifulSoup import BeautifulSoup +#from BeautifulSoup import BeautifulSoup +from bs4 import BeautifulSoup from django.core.cache import cache from django_future.csrf import ensure_csrf_cookie diff --git a/requirements.txt b/requirements.txt index 978b5e6f1a..48362c69ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,8 @@ django_debug_toolbar -e git://github.com/MITx/django-pipeline.git#egg=django-pipeline django-staticfiles>=1.2.1 fs -beautifulsoup +beautifulsoup +beautifulsoup4 feedparser requests sympy From 8c1bb0cab2b0ab7b93febbd9e7c7075ddf41ffb9 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 1 Aug 2012 10:54:32 -0400 Subject: [PATCH 026/124] sanity checking /usr/local, updating coffee script install location, old location no longer works --- create-dev-env.sh | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index 7ebff77fb2..d0d5c30faa 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -203,6 +203,24 @@ case `uname -s` in esac ;; Darwin) + + if [[ ! -w /usr/local ]]; then + cat</dev/null || { output "Installing brew" /usr/bin/ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)" @@ -238,7 +256,7 @@ case `uname -s` in command -v coffee &>/dev/null || { output "Installing coffee script" - curl http://npmjs.org/install.sh | sh + curl --insecure https://npmjs.org/install.sh | sh npm install -g coffee-script } ;; From 71898e4869df7f88249c7b7da71e4fa851035606 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 1 Aug 2012 10:54:50 -0400 Subject: [PATCH 027/124] removing matplotlib from requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 978b5e6f1a..4cc132d6ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ django<1.4 pip numpy scipy -matplotlib markdown pygments lxml From 94c19059b58a443ad6bdd9ada47fe6d6e74d7abb Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 1 Aug 2012 11:08:41 -0400 Subject: [PATCH 028/124] trailing whitespace --- create-dev-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index d0d5c30faa..8a1a7e6f89 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -215,7 +215,7 @@ case `uname -s` in and re-run the script: $ sudo chown -R $USER /usr/local -EO +EO exit 1 From ae3da772d09b96d74bb535b1383687a08f6f87ba Mon Sep 17 00:00:00 2001 From: Bridger Maxwell Date: Wed, 1 Aug 2012 11:27:04 -0400 Subject: [PATCH 029/124] Got profile page working again. The grader for each course is now defined in the data dir as grading_policy.json. --- common/lib/xmodule/xmodule/course_module.py | 34 +++++++- common/lib/xmodule/xmodule/graders.py | 64 ++++++++++++++ lms/djangoapps/courseware/course_settings.py | 91 -------------------- lms/djangoapps/courseware/grades.py | 4 +- lms/djangoapps/courseware/views.py | 10 +-- lms/templates/profile.html | 10 +-- 6 files changed, 106 insertions(+), 107 deletions(-) delete mode 100644 lms/djangoapps/courseware/course_settings.py diff --git a/common/lib/xmodule/xmodule/course_module.py b/common/lib/xmodule/xmodule/course_module.py index dfac1ac9c6..7e0019205e 100644 --- a/common/lib/xmodule/xmodule/course_module.py +++ b/common/lib/xmodule/xmodule/course_module.py @@ -1,7 +1,9 @@ +from fs.errors import ResourceNotFoundError import time import dateutil.parser import logging +from xmodule.graders import load_grading_policy from xmodule.modulestore import Location from xmodule.seq_module import SequenceDescriptor, SequenceModule @@ -14,6 +16,9 @@ class CourseDescriptor(SequenceDescriptor): def __init__(self, system, definition=None, **kwargs): super(CourseDescriptor, self).__init__(system, definition, **kwargs) + + self._grader = None + self._grade_cutoffs = None try: self.start = time.strptime(self.metadata["start"], "%Y-%m-%dT%H:%M") @@ -27,7 +32,34 @@ class CourseDescriptor(SequenceDescriptor): def has_started(self): return time.gmtime() > self.start - + + @property + def grader(self): + self.__load_grading_policy() + return self._grader + + @property + def grade_cutoffs(self): + self.__load_grading_policy() + return self._grade_cutoffs + + + def __load_grading_policy(self): + if not self._grader or not self._grade_cutoffs: + policy_string = "" + + try: + with self.system.resources_fs.open("grading_policy.json") as grading_policy_file: + policy_string = grading_policy_file.read() + except (IOError, ResourceNotFoundError): + log.warning("Unable to load course settings file from grading_policy.json in course " + self.id) + + grading_policy = load_grading_policy(policy_string) + + self._grader = grading_policy['GRADER'] + self._grade_cutoffs = grading_policy['GRADE_CUTOFFS'] + + @staticmethod def id_to_location(course_id): '''Convert the given course_id (org/course/name) to a location object. diff --git a/common/lib/xmodule/xmodule/graders.py b/common/lib/xmodule/xmodule/graders.py index 4473c8f1b9..fca862aa9f 100644 --- a/common/lib/xmodule/xmodule/graders.py +++ b/common/lib/xmodule/xmodule/graders.py @@ -1,4 +1,5 @@ import abc +import json import logging from collections import namedtuple @@ -9,6 +10,69 @@ log = logging.getLogger("mitx.courseware") # Section either indicates the name of the problem or the name of the section Score = namedtuple("Score", "earned possible graded section") +def load_grading_policy(course_policy_string): + """ + This loads a grading policy from a string (usually read from a file), + which can be a JSON object or an empty string. + + The JSON object can have the keys GRADER and GRADE_CUTOFFS. If either is + missing, it reverts to the default. + """ + + default_policy_string = """ + { + "GRADER" : [ + { + "type" : "Homework", + "min_count" : 12, + "drop_count" : 2, + "short_label" : "HW", + "weight" : 0.15 + }, + { + "type" : "Lab", + "min_count" : 12, + "drop_count" : 2, + "category" : "Labs", + "weight" : 0.15 + }, + { + "type" : "Midterm", + "name" : "Midterm Exam", + "short_label" : "Midterm", + "weight" : 0.3 + }, + { + "type" : "Final", + "name" : "Final Exam", + "short_label" : "Final", + "weight" : 0.4 + } + ], + "GRADE_CUTOFFS" : { + "A" : 0.87, + "B" : 0.7, + "C" : 0.6 + } + } + """ + + # Load the global settings as a dictionary + grading_policy = json.loads(default_policy_string) + + # Load the course policies as a dictionary + course_policy = {} + if course_policy_string: + course_policy = json.loads(course_policy_string) + + # Override any global settings with the course settings + grading_policy.update(course_policy) + + # Here is where we should parse any configurations, so that we can fail early + grading_policy['GRADER'] = grader_from_conf(grading_policy['GRADER']) + + return grading_policy + def aggregate_scores(scores, section_name="summary"): """ diff --git a/lms/djangoapps/courseware/course_settings.py b/lms/djangoapps/courseware/course_settings.py deleted file mode 100644 index 5b2348bee6..0000000000 --- a/lms/djangoapps/courseware/course_settings.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Course settings module. All settings in the global_settings are -first applied, and then any settings in the settings.DATA_DIR/course_settings.json -are applied. A setting must be in ALL_CAPS. - -Settings are used by calling - -from courseware.course_settings import course_settings - -Note that courseware.course_settings.course_settings is not a module -- it's an object. So -importing individual settings is not possible: - -from courseware.course_settings.course_settings import GRADER # This won't work. - -""" -import json -import logging - -from django.conf import settings - -from xmodule import graders - -log = logging.getLogger("mitx.courseware") - -global_settings_json = """ -{ - "GRADER" : [ - { - "type" : "Homework", - "min_count" : 12, - "drop_count" : 2, - "short_label" : "HW", - "weight" : 0.15 - }, - { - "type" : "Lab", - "min_count" : 12, - "drop_count" : 2, - "category" : "Labs", - "weight" : 0.15 - }, - { - "type" : "Midterm", - "name" : "Midterm Exam", - "short_label" : "Midterm", - "weight" : 0.3 - }, - { - "type" : "Final", - "name" : "Final Exam", - "short_label" : "Final", - "weight" : 0.4 - } - ], - "GRADE_CUTOFFS" : { - "A" : 0.87, - "B" : 0.7, - "C" : 0.6 - } -} -""" - - -class Settings(object): - def __init__(self): - - # Load the global settings as a dictionary - global_settings = json.loads(global_settings_json) - - # Load the course settings as a dictionary - course_settings = {} - try: - # TODO: this doesn't work with multicourse - with open(settings.DATA_DIR + "/course_settings.json") as course_settings_file: - course_settings_string = course_settings_file.read() - course_settings = json.loads(course_settings_string) - except IOError: - log.warning("Unable to load course settings file from " + str(settings.DATA_DIR) + "/course_settings.json") - - # Override any global settings with the course settings - global_settings.update(course_settings) - - # Now, set the properties from the course settings on ourselves - for setting in global_settings: - setting_value = global_settings[setting] - setattr(self, setting, setting_value) - - # Here is where we should parse any configurations, so that we can fail early - self.GRADER = graders.grader_from_conf(self.GRADER) - -course_settings = Settings() diff --git a/lms/djangoapps/courseware/grades.py b/lms/djangoapps/courseware/grades.py index 66aff08dca..5a817e3d6c 100644 --- a/lms/djangoapps/courseware/grades.py +++ b/lms/djangoapps/courseware/grades.py @@ -3,7 +3,6 @@ import logging from django.conf import settings -from courseware.course_settings import course_settings from xmodule import graders from xmodule.graders import Score from models import StudentModule @@ -11,7 +10,7 @@ from models import StudentModule _log = logging.getLogger("mitx.courseware") -def grade_sheet(student, course, student_module_cache): +def grade_sheet(student, course, grader, student_module_cache): """ This pulls a summary of all problems in the course. It returns a dictionary with two datastructures: @@ -78,7 +77,6 @@ def grade_sheet(student, course, student_module_cache): 'chapter': c.metadata.get('display_name'), 'sections': sections}) - grader = course_settings.GRADER grade_summary = grader.grade(totaled_scores) return {'courseware_summary': chapters, diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 00fde8a84c..7281ab01ad 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -19,7 +19,6 @@ from django.views.decorators.cache import cache_control from module_render import toc_for_course, get_module, get_section from models import StudentModuleCache from student.models import UserProfile -from multicourse import multicourse_settings from xmodule.modulestore import Location from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem from xmodule.modulestore.django import modulestore @@ -110,8 +109,8 @@ def profile(request, course_id, student_id=None): user_info = UserProfile.objects.get(user=student) student_module_cache = StudentModuleCache(request.user, course) - course, _, _, _ = get_module(request.user, request, course.location, student_module_cache) - + course_module, _, _, _ = get_module(request.user, request, course.location, student_module_cache) + context = {'name': user_info.name, 'username': student.username, 'location': user_info.location, @@ -121,7 +120,7 @@ def profile(request, course_id, student_id=None): 'format_url_params': format_url_params, 'csrf': csrf(request)['csrf_token'] } - context.update(grades.grade_sheet(student, course, student_module_cache)) + context.update(grades.grade_sheet(student, course_module, course.grader, student_module_cache)) return render_to_response('profile.html', context) @@ -184,9 +183,6 @@ def index(request, course_id, chapter=None, section=None, chapter = clean(chapter) section = clean(section) - if settings.ENABLE_MULTICOURSE: - settings.MODULESTORE['default']['OPTIONS']['data_dir'] = settings.DATA_DIR + multicourse_settings.get_course_xmlpath(course) - context = { 'csrf': csrf(request)['csrf_token'], 'accordion': render_accordion(request, course, chapter, section), diff --git a/lms/templates/profile.html b/lms/templates/profile.html index 1e3bde5969..6cda85fb03 100644 --- a/lms/templates/profile.html +++ b/lms/templates/profile.html @@ -4,6 +4,7 @@ <%block name="headextra"> <%static:css group='course'/> + <%namespace name="profile_graphs" file="profile_graphs.js"/> <%block name="title">Profile - edX 6.002x @@ -110,9 +111,9 @@ $(function() { -<%include file="navigation.html" args="active_page='profile'" /> +<%include file="course_navigation.html" args="active_page='profile'" /> -
+
@@ -126,8 +127,7 @@ $(function() { %for chapter in courseware_summary: %if not chapter['chapter'] == "hidden":
  • -

    - ${ chapter['chapter'] }

    +

    ${ chapter['chapter'] }

      %for section in chapter['sections']: @@ -138,7 +138,7 @@ $(function() { percentageString = "{0:.0%}".format( float(earned)/total) if earned > 0 and total > 0 else "" %> -

      +

      ${ section['section'] } ${"({0:.3n}/{1:.3n}) {2}".format( float(earned), float(total), percentageString )}

      ${section['format']} %if 'due' in section and section['due']!="": From 4e6a9b0df7e3cffc8e63590b8124c69dd2369b34 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Mon, 30 Jul 2012 10:36:33 -0400 Subject: [PATCH 030/124] Add kwargs back to DescriptorSystem init()s * allow future expansion without breaking interface. --- common/lib/xmodule/xmodule/mako_module.py | 4 ++-- common/lib/xmodule/xmodule/modulestore/xml.py | 6 +++--- common/lib/xmodule/xmodule/x_module.py | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/common/lib/xmodule/xmodule/mako_module.py b/common/lib/xmodule/xmodule/mako_module.py index fcc47aaaaf..213e9077db 100644 --- a/common/lib/xmodule/xmodule/mako_module.py +++ b/common/lib/xmodule/xmodule/mako_module.py @@ -3,9 +3,9 @@ from x_module import XModuleDescriptor, DescriptorSystem class MakoDescriptorSystem(DescriptorSystem): def __init__(self, load_item, resources_fs, error_handler, - render_template): + render_template, **kwargs): super(MakoDescriptorSystem, self).__init__( - load_item, resources_fs, error_handler) + load_item, resources_fs, error_handler, **kwargs) self.render_template = render_template diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 7dd6868f78..8a084b19ee 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -29,7 +29,7 @@ def clean_out_mako_templating(xml_string): return xml_string class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): - def __init__(self, xmlstore, org, course, course_dir, error_handler): + def __init__(self, xmlstore, org, course, course_dir, error_handler, **kwargs): """ A class that handles loading from xml. Does some munging to ensure that all elements have unique slugs. @@ -88,9 +88,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): resources_fs = OSFS(xmlstore.data_dir / course_dir) MakoDescriptorSystem.__init__(self, load_item, resources_fs, - error_handler, render_template) + error_handler, render_template, **kwargs) XMLParsingSystem.__init__(self, load_item, resources_fs, - error_handler, process_xml) + error_handler, process_xml, **kwargs) diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 7bb98dcdc5..97ae307809 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -521,7 +521,7 @@ class XModuleDescriptor(Plugin, HTMLSnippet): class DescriptorSystem(object): - def __init__(self, load_item, resources_fs, error_handler): + def __init__(self, load_item, resources_fs, error_handler, **kwargs): """ load_item: Takes a Location and returns an XModuleDescriptor @@ -561,14 +561,15 @@ class DescriptorSystem(object): class XMLParsingSystem(DescriptorSystem): - def __init__(self, load_item, resources_fs, error_handler, process_xml): + def __init__(self, load_item, resources_fs, error_handler, process_xml, **kwargs): """ load_item, resources_fs, error_handler: see DescriptorSystem process_xml: Takes an xml string, and returns a XModuleDescriptor created from that xml """ - DescriptorSystem.__init__(self, load_item, resources_fs, error_handler) + DescriptorSystem.__init__(self, load_item, resources_fs, error_handler, + **kwargs) self.process_xml = process_xml From ef6da22ac35b70f313ee8f9e60c89d68e89bd745 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Mon, 30 Jul 2012 10:42:23 -0400 Subject: [PATCH 031/124] Add a fallback MalformedDescriptor * when things don't load normally, use this * separate raw editing functionality into EditingDescriptor * raw descriptor just enforces that xml is valid * add a MalformedDescriptor that just saves a string * Fallback to it on import. --- common/lib/xmodule/tests/test_import.py | 38 +++++++++++++++++ common/lib/xmodule/xmodule/editing_module.py | 24 +++++++++++ .../lib/xmodule/xmodule/malformed_module.py | 41 +++++++++++++++++++ common/lib/xmodule/xmodule/raw_module.py | 18 ++------ common/lib/xmodule/xmodule/x_module.py | 34 ++++++++++----- 5 files changed, 131 insertions(+), 24 deletions(-) create mode 100644 common/lib/xmodule/tests/test_import.py create mode 100644 common/lib/xmodule/xmodule/editing_module.py create mode 100644 common/lib/xmodule/xmodule/malformed_module.py diff --git a/common/lib/xmodule/tests/test_import.py b/common/lib/xmodule/tests/test_import.py new file mode 100644 index 0000000000..c0bd9af3d0 --- /dev/null +++ b/common/lib/xmodule/tests/test_import.py @@ -0,0 +1,38 @@ +from path import path + +import unittest + +from xmodule.x_module import XMLParsingSystem, XModuleDescriptor +from xmodule.errorhandlers import ignore_errors_handler +from xmodule.modulestore import Location + +class ImportTestCase(unittest.TestCase): + '''Make sure module imports work properly, including for malformed inputs''' + + def test_fallback(self): + '''Make sure that malformed xml loads as a MalformedDescriptorb.''' + + bad_xml = '''''' + + # Shouldn't need any system params, because the initial parse should fail + def load_item(loc): + raise Exception("Shouldn't be called") + + resources_fs = None + + def process_xml(xml): + raise Exception("Shouldn't be called") + + + def render_template(template, context): + raise Exception("Shouldn't be called") + + system = XMLParsingSystem(load_item, resources_fs, + ignore_errors_handler, process_xml) + system.render_template = render_template + + descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course', + None) + + self.assertEqual(descriptor.__class__.__name__, + 'MalformedDescriptor') diff --git a/common/lib/xmodule/xmodule/editing_module.py b/common/lib/xmodule/xmodule/editing_module.py new file mode 100644 index 0000000000..4188165a24 --- /dev/null +++ b/common/lib/xmodule/xmodule/editing_module.py @@ -0,0 +1,24 @@ +from pkg_resources import resource_string +from lxml import etree +from xmodule.mako_module import MakoModuleDescriptor +import logging + +log = logging.getLogger(__name__) + +class EditingDescriptor(MakoModuleDescriptor): + """ + Module that provides a raw editing view of its data and children. It does not + perform any validation on its definition---just passes it along to the browser. + + This class is intended to be used as a mixin. + """ + mako_template = "widgets/raw-edit.html" + + js = {'coffee': [resource_string(__name__, 'js/src/raw/edit.coffee')]} + js_module_name = "RawDescriptor" + + def get_context(self): + return { + 'module': self, + 'data': self.definition['data'], + } diff --git a/common/lib/xmodule/xmodule/malformed_module.py b/common/lib/xmodule/xmodule/malformed_module.py new file mode 100644 index 0000000000..803813dee8 --- /dev/null +++ b/common/lib/xmodule/xmodule/malformed_module.py @@ -0,0 +1,41 @@ +from pkg_resources import resource_string +from lxml import etree +from xmodule.mako_module import MakoModuleDescriptor +from xmodule.xml_module import XmlDescriptor +from xmodule.editing_module import EditingDescriptor + +import logging + +log = logging.getLogger(__name__) + +class MalformedDescriptor(EditingDescriptor): + """ + Module that provides a raw editing view of broken xml. + """ + + @classmethod + def from_xml(cls, xml_data, system, org=None, course=None): + '''Create an instance of this descriptor from the supplied data. + + Does not try to parse the data--just stores it. + ''' + + # TODO (vshnayder): how does one get back from this to a valid descriptor? + # try to parse and if successfull, send back to x_module? + + definition = { 'data' : xml_data } + # TODO (vshnayder): Do we need a valid slug here? Just pick a random + # 64-bit num? + location = ['i4x', org, course, 'malformed', 'slug'] + metadata = {} # stays in the xml_data + + return cls(system, definition, location=location, metadata=metadata) + + def export_to_xml(self, resource_fs): + ''' + Export as a string wrapped in xml + ''' + root = etree.Element('malformed') + root.text = self.definition['data'] + return etree.tostring(root) + diff --git a/common/lib/xmodule/xmodule/raw_module.py b/common/lib/xmodule/xmodule/raw_module.py index 90f4139bd5..f9f358f945 100644 --- a/common/lib/xmodule/xmodule/raw_module.py +++ b/common/lib/xmodule/xmodule/raw_module.py @@ -1,27 +1,17 @@ from pkg_resources import resource_string from lxml import etree +from xmodule.editing_module import EditingDescriptor from xmodule.mako_module import MakoModuleDescriptor from xmodule.xml_module import XmlDescriptor import logging log = logging.getLogger(__name__) - -class RawDescriptor(MakoModuleDescriptor, XmlDescriptor): +class RawDescriptor(XmlDescriptor, EditingDescriptor): """ - Module that provides a raw editing view of its data and children + Module that provides a raw editing view of its data and children. It + requires that the definition xml is valid. """ - mako_template = "widgets/raw-edit.html" - - js = {'coffee': [resource_string(__name__, 'js/src/raw/edit.coffee')]} - js_module_name = "RawDescriptor" - - def get_context(self): - return { - 'module': self, - 'data': self.definition['data'], - } - @classmethod def definition_from_xml(cls, xml_object, system): return {'data': etree.tostring(xml_object)} diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 97ae307809..c4c5110abc 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -1,10 +1,12 @@ from lxml import etree +from lxml.etree import XMLSyntaxError import pkg_resources import logging +from fs.errors import ResourceNotFoundError +from functools import partial from xmodule.modulestore import Location -from functools import partial log = logging.getLogger('mitx.' + __name__) @@ -443,16 +445,28 @@ class XModuleDescriptor(Plugin, HTMLSnippet): system is an XMLParsingSystem org and course are optional strings that will be used in the generated - modules url identifiers + module's url identifiers """ - class_ = XModuleDescriptor.load_class( - etree.fromstring(xml_data).tag, - default_class - ) - # leave next line, commented out - useful for low-level debugging - # log.debug('[XModuleDescriptor.load_from_xml] tag=%s, class_=%s' % ( - # etree.fromstring(xml_data).tag,class_)) - return class_.from_xml(xml_data, system, org, course) + try: + class_ = XModuleDescriptor.load_class( + etree.fromstring(xml_data).tag, + default_class + ) + # leave next line, commented out - useful for low-level debugging + # log.debug('[XModuleDescriptor.load_from_xml] tag=%s, class_=%s' % ( + # etree.fromstring(xml_data).tag,class_)) + + descriptor = class_.from_xml(xml_data, system, org, course) + except (ResourceNotFoundError, XMLSyntaxError) as err: + # Didn't load properly. Fall back on loading as a malformed + # descriptor. This should never error due to formatting. + + # Put import here to avoid circular import errors + from xmodule.malformed_module import MalformedDescriptor + + descriptor = MalformedDescriptor.from_xml(xml_data, system, org, course) + + return descriptor @classmethod def from_xml(cls, xml_data, system, org=None, course=None): From 53608922ba2cf74f27e5f05a23694943593704f4 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Mon, 30 Jul 2012 11:36:43 -0400 Subject: [PATCH 032/124] Make the malformed descriptor import properly * Also get rid of lazy loading of metadata and definition --- common/lib/xmodule/setup.py | 1 + .../lib/xmodule/xmodule/malformed_module.py | 11 +- common/lib/xmodule/xmodule/x_module.py | 1 + common/lib/xmodule/xmodule/xml_module.py | 101 +++++++++--------- 4 files changed, 62 insertions(+), 52 deletions(-) diff --git a/common/lib/xmodule/setup.py b/common/lib/xmodule/setup.py index b495cd0aee..09a6b7e712 100644 --- a/common/lib/xmodule/setup.py +++ b/common/lib/xmodule/setup.py @@ -25,6 +25,7 @@ setup( "discuss = xmodule.backcompat_module:TranslateCustomTagDescriptor", "html = xmodule.html_module:HtmlDescriptor", "image = xmodule.backcompat_module:TranslateCustomTagDescriptor", + "malformed = xmodule.malformed_module:MalformedDescriptor", "problem = xmodule.capa_module:CapaDescriptor", "problemset = xmodule.vertical_module:VerticalDescriptor", "section = xmodule.backcompat_module:SemanticSectionDescriptor", diff --git a/common/lib/xmodule/xmodule/malformed_module.py b/common/lib/xmodule/xmodule/malformed_module.py index 803813dee8..54032cf7d2 100644 --- a/common/lib/xmodule/xmodule/malformed_module.py +++ b/common/lib/xmodule/xmodule/malformed_module.py @@ -20,8 +20,15 @@ class MalformedDescriptor(EditingDescriptor): Does not try to parse the data--just stores it. ''' - # TODO (vshnayder): how does one get back from this to a valid descriptor? - # try to parse and if successfull, send back to x_module? + #log.debug("processing '{0}'".format(xml_data)) + try: + xml_obj = etree.fromstring(xml_data) + if xml_obj.tag == 'malformed': + xml_data = xml_obj.text + # TODO (vshnayder): how does one get back from this to a valid descriptor? + # For now, have to fix manually. + except etree.XMLSyntaxError: + pass definition = { 'data' : xml_data } # TODO (vshnayder): Do we need a valid slug here? Just pick a random diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index c4c5110abc..8803c37d78 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -464,6 +464,7 @@ class XModuleDescriptor(Plugin, HTMLSnippet): # Put import here to avoid circular import errors from xmodule.malformed_module import MalformedDescriptor + #system.error_handler("Error loading from xml.") descriptor = MalformedDescriptor.from_xml(xml_data, system, org, course) return descriptor diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index 6750906eb4..5786dbd227 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -11,66 +11,65 @@ import os log = logging.getLogger(__name__) -# TODO (cpennington): This was implemented in an attempt to improve performance, -# but the actual improvement wasn't measured (and it was implemented late at night). -# We should check if it hurts, and whether there's a better way of doing lazy loading +# # TODO (cpennington): This was implemented in an attempt to improve performance, +# # but the actual improvement wasn't measured (and it was implemented late at night). +# # We should check if it hurts, and whether there's a better way of doing lazy loading +# class LazyLoadingDict(MutableMapping): +# """ +# A dictionary object that lazily loads its contents from a provided +# function on reads (of members that haven't already been set). +# """ -class LazyLoadingDict(MutableMapping): - """ - A dictionary object that lazily loads its contents from a provided - function on reads (of members that haven't already been set). - """ +# def __init__(self, loader): +# ''' +# On the first read from this dictionary, it will call loader() to +# populate its contents. loader() must return something dict-like. Any +# elements set before the first read will be preserved. +# ''' +# self._contents = {} +# self._loaded = False +# self._loader = loader +# self._deleted = set() - def __init__(self, loader): - ''' - On the first read from this dictionary, it will call loader() to - populate its contents. loader() must return something dict-like. Any - elements set before the first read will be preserved. - ''' - self._contents = {} - self._loaded = False - self._loader = loader - self._deleted = set() +# def __getitem__(self, name): +# if not (self._loaded or name in self._contents or name in self._deleted): +# self.load() - def __getitem__(self, name): - if not (self._loaded or name in self._contents or name in self._deleted): - self.load() +# return self._contents[name] - return self._contents[name] +# def __setitem__(self, name, value): +# self._contents[name] = value +# self._deleted.discard(name) - def __setitem__(self, name, value): - self._contents[name] = value - self._deleted.discard(name) +# def __delitem__(self, name): +# del self._contents[name] +# self._deleted.add(name) - def __delitem__(self, name): - del self._contents[name] - self._deleted.add(name) +# def __contains__(self, name): +# self.load() +# return name in self._contents - def __contains__(self, name): - self.load() - return name in self._contents +# def __len__(self): +# self.load() +# return len(self._contents) - def __len__(self): - self.load() - return len(self._contents) +# def __iter__(self): +# self.load() +# return iter(self._contents) - def __iter__(self): - self.load() - return iter(self._contents) +# def __repr__(self): +# self.load() +# return repr(self._contents) - def __repr__(self): - self.load() - return repr(self._contents) +# def load(self): +# if self._loaded: +# return - def load(self): - if self._loaded: - return - - loaded_contents = self._loader() - loaded_contents.update(self._contents) - self._contents = loaded_contents - self._loaded = True +# loaded_contents = self._loader() +# loaded_contents.update(self._contents) +# self._contents = loaded_contents +# self._loaded = True _AttrMapBase = namedtuple('_AttrMap', 'metadata_key to_metadata from_metadata') @@ -230,11 +229,13 @@ class XmlDescriptor(XModuleDescriptor): cls.clean_metadata_from_xml(definition_xml) return cls.definition_from_xml(definition_xml, system) + # VS[compat] -- just have the url_name lookup once translation is done + slug = xml_object.get('url_name', xml_object.get('slug')) return cls( system, - LazyLoadingDict(definition_loader), + definition, location=location, - metadata=LazyLoadingDict(metadata_loader), + metadata=metadata, ) @classmethod From 0d83d2e645a4a701ebad7e5158493c198d552370 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Mon, 30 Jul 2012 11:51:14 -0400 Subject: [PATCH 033/124] Add roundtrip test for malformed module * also fix error message in backcompat_module --- common/lib/xmodule/tests/test_import.py | 35 ++++++++++++++++--- .../lib/xmodule/xmodule/backcompat_module.py | 2 +- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/common/lib/xmodule/tests/test_import.py b/common/lib/xmodule/tests/test_import.py index c0bd9af3d0..6643b93a67 100644 --- a/common/lib/xmodule/tests/test_import.py +++ b/common/lib/xmodule/tests/test_import.py @@ -9,11 +9,9 @@ from xmodule.modulestore import Location class ImportTestCase(unittest.TestCase): '''Make sure module imports work properly, including for malformed inputs''' - def test_fallback(self): - '''Make sure that malformed xml loads as a MalformedDescriptorb.''' - - bad_xml = '''''' - + @staticmethod + def get_system(): + '''Get a dummy system''' # Shouldn't need any system params, because the initial parse should fail def load_item(loc): raise Exception("Shouldn't be called") @@ -31,8 +29,35 @@ class ImportTestCase(unittest.TestCase): ignore_errors_handler, process_xml) system.render_template = render_template + return system + + def test_fallback(self): + '''Make sure that malformed xml loads as a MalformedDescriptorb.''' + + bad_xml = '''''' + + system = self.get_system() + descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course', None) self.assertEqual(descriptor.__class__.__name__, 'MalformedDescriptor') + + def test_reimport(self): + '''Make sure an already-exported malformed xml tag loads properly''' + + bad_xml = '''''' + system = self.get_system() + descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course', + None) + resource_fs = None + tag_xml = descriptor.export_to_xml(resource_fs) + re_import_descriptor = XModuleDescriptor.load_from_xml(tag_xml, system, + 'org', 'course', + None) + self.assertEqual(re_import_descriptor.__class__.__name__, + 'MalformedDescriptor') + + self.assertEqual(descriptor.definition['data'], + re_import_descriptor.definition['data']) diff --git a/common/lib/xmodule/xmodule/backcompat_module.py b/common/lib/xmodule/xmodule/backcompat_module.py index 997ad476c4..da0d6788e4 100644 --- a/common/lib/xmodule/xmodule/backcompat_module.py +++ b/common/lib/xmodule/xmodule/backcompat_module.py @@ -32,7 +32,7 @@ def process_includes(fn): # read in and convert to XML incxml = etree.XML(ifp.read()) - # insert new XML into tree in place of inlcude + # insert new XML into tree in place of include parent.insert(parent.index(next_include), incxml) except Exception: msg = "Error in problem xml include: %s" % (etree.tostring(next_include, pretty_print=True)) From 10054e95ce0e0962e975ff236b5237f659cd6bc7 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Mon, 30 Jul 2012 15:09:33 -0400 Subject: [PATCH 034/124] add message to import command --- cms/djangoapps/contentstore/management/commands/import.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index 69aaa35a7d..a6790ad59b 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -23,4 +23,7 @@ class Command(BaseCommand): course_dirs = args[1:] else: course_dirs = None + print "Importing. Data_dir={data}, course_dirs={courses}".format( + data=data_dir, + courses=course_dirs) import_from_xml(modulestore(), data_dir, course_dirs) From c53ed6a23830f842aa130c104e7c9125509c4b9a Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Mon, 30 Jul 2012 15:10:21 -0400 Subject: [PATCH 035/124] remove some debugging messages --- common/lib/capa/capa/capa_problem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index ed99c71635..93b6a085c1 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -329,7 +329,7 @@ class LoncapaProblem(object): # path is an absolute path or a path relative to the data dir dir = os.path.join(self.system.filestore.root_path, dir) abs_dir = os.path.normpath(dir) - log.debug("appending to path: %s" % abs_dir) + #log.debug("appending to path: %s" % abs_dir) path.append(abs_dir) return path From d750d945fdd855ea05e69d346b248b222c6e4136 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Mon, 30 Jul 2012 15:11:38 -0400 Subject: [PATCH 036/124] Remove malformed tags when contents aren't malformed anymore. --- common/lib/xmodule/tests/test_import.py | 26 +++++++++++++++- .../lib/xmodule/xmodule/malformed_module.py | 30 +++++++++++++++---- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/common/lib/xmodule/tests/test_import.py b/common/lib/xmodule/tests/test_import.py index 6643b93a67..dd55f8ff38 100644 --- a/common/lib/xmodule/tests/test_import.py +++ b/common/lib/xmodule/tests/test_import.py @@ -1,7 +1,8 @@ from path import path - import unittest +from lxml import etree + from xmodule.x_module import XMLParsingSystem, XModuleDescriptor from xmodule.errorhandlers import ignore_errors_handler from xmodule.modulestore import Location @@ -61,3 +62,26 @@ class ImportTestCase(unittest.TestCase): self.assertEqual(descriptor.definition['data'], re_import_descriptor.definition['data']) + + def test_fixed_xml_tag(self): + """Make sure a tag that's been fixed exports as the original tag type""" + + # create a malformed tag with valid xml contents + root = etree.Element('malformed') + good_xml = '''''' + root.text = good_xml + + xml_str_in = etree.tostring(root) + + # load it + system = self.get_system() + descriptor = XModuleDescriptor.load_from_xml(xml_str_in, system, 'org', 'course', + None) + # export it + resource_fs = None + xml_str_out = descriptor.export_to_xml(resource_fs) + + # Now make sure the exported xml is a sequential + xml_out = etree.fromstring(xml_str_out) + self.assertEqual(xml_out.tag, 'sequential') + diff --git a/common/lib/xmodule/xmodule/malformed_module.py b/common/lib/xmodule/xmodule/malformed_module.py index 54032cf7d2..ac419fff26 100644 --- a/common/lib/xmodule/xmodule/malformed_module.py +++ b/common/lib/xmodule/xmodule/malformed_module.py @@ -1,5 +1,6 @@ from pkg_resources import resource_string from lxml import etree +from xmodule.x_module import XModule from xmodule.mako_module import MakoModuleDescriptor from xmodule.xml_module import XmlDescriptor from xmodule.editing_module import EditingDescriptor @@ -8,10 +9,18 @@ import logging log = logging.getLogger(__name__) +class MalformedModule(XModule): + def get_html(self): + '''Show an error. + TODO (vshnayder): proper style, divs, etc. + ''' + return "Malformed content--not showing through get_html()" + class MalformedDescriptor(EditingDescriptor): """ Module that provides a raw editing view of broken xml. """ + module_class = MalformedModule @classmethod def from_xml(cls, xml_data, system, org=None, course=None): @@ -20,8 +29,8 @@ class MalformedDescriptor(EditingDescriptor): Does not try to parse the data--just stores it. ''' - #log.debug("processing '{0}'".format(xml_data)) try: + # If this is already a malformed tag, don't want to re-wrap it. xml_obj = etree.fromstring(xml_data) if xml_obj.tag == 'malformed': xml_data = xml_obj.text @@ -40,9 +49,18 @@ class MalformedDescriptor(EditingDescriptor): def export_to_xml(self, resource_fs): ''' - Export as a string wrapped in xml - ''' - root = etree.Element('malformed') - root.text = self.definition['data'] - return etree.tostring(root) + If the definition data is invalid xml, export it wrapped in a malformed + tag. If it is valid, export without the wrapper. + NOTE: There may still be problems with the valid xml--it could be + missing required attributes, could have the wrong tags, refer to missing + files, etc. + ''' + try: + xml = etree.fromstring(self.definition['data']) + return etree.tostring(xml) + except etree.XMLSyntaxError: + # still not valid. + root = etree.Element('malformed') + root.text = self.definition['data'] + return etree.tostring(root) From 46775386d376e0cba2e2d146bce0e34e3984cfda Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Mon, 30 Jul 2012 18:29:18 -0400 Subject: [PATCH 037/124] make CACHE_TIMEOUT messages go away --- cms/envs/dev.py | 3 +++ lms/envs/dev.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cms/envs/dev.py b/cms/envs/dev.py index 59bc623729..4098263829 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -76,3 +76,6 @@ CACHES = { 'KEY_FUNCTION': 'util.memcache.safe_key', } } + +# Make the keyedcache startup warnings go away +CACHE_TIMEOUT = 0 diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 1a2659cb1f..3fd86f1aee 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -17,7 +17,7 @@ MITX_FEATURES['DISABLE_START_DATES'] = True WIKI_ENABLED = True -LOGGING = get_logger_config(ENV_ROOT / "log", +LOGGING = get_logger_config(ENV_ROOT / "log", logging_env="dev", tracking_filename="tracking.log", debug=True) @@ -30,7 +30,7 @@ DATABASES = { } CACHES = { - # This is the cache used for most things. Askbot will not work without a + # This is the cache used for most things. Askbot will not work without a # functioning cache -- it relies on caching to load its settings in places. # In staging/prod envs, the sessions also live here. 'default': { @@ -52,11 +52,14 @@ CACHES = { } } +# Make the keyedcache startup warnings go away +CACHE_TIMEOUT = 0 + # Dummy secret key for dev SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' ################################ DEBUG TOOLBAR ################################# -INSTALLED_APPS += ('debug_toolbar',) +INSTALLED_APPS += ('debug_toolbar',) MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) INTERNAL_IPS = ('127.0.0.1',) @@ -71,8 +74,8 @@ DEBUG_TOOLBAR_PANELS = ( 'debug_toolbar.panels.logger.LoggingPanel', # Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and -# Django=1.3.1/1.4 where requests to views get duplicated (your method gets -# hit twice). So you can uncomment when you need to diagnose performance +# Django=1.3.1/1.4 where requests to views get duplicated (your method gets +# hit twice). So you can uncomment when you need to diagnose performance # problems, but you shouldn't leave it on. # 'debug_toolbar.panels.profiling.ProfilingDebugPanel', ) From ed35cefa295ae4b8e798d86587cbf8c838289df6 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Mon, 30 Jul 2012 18:36:02 -0400 Subject: [PATCH 038/124] Fix html file handling. * html files are now stored as follows: If the html file is valid xml, store as html/stuff.xml If it's not, store as html/stuff.xml, which contains , and html/stuff.html, which actually contains the contents. Warn if the contents are not parseable with lxml's html parser, but don't error. * for parseable html, strip out the html tag when storing, so that it isn't rendered into the middle of a page * lots of backcompat to deal with paths. Can go away soon. * fix output ordering in clean_xml --- common/lib/xmodule/html_checker.py | 14 +++ common/lib/xmodule/stringify.py | 20 ++++ common/lib/xmodule/tests/test_stringify.py | 9 ++ common/lib/xmodule/xmodule/html_module.py | 103 ++++++++++++++++-- common/lib/xmodule/xmodule/xml_module.py | 88 ++++++++------- .../management/commands/clean_xml.py | 1 + 6 files changed, 184 insertions(+), 51 deletions(-) create mode 100644 common/lib/xmodule/html_checker.py create mode 100644 common/lib/xmodule/stringify.py create mode 100644 common/lib/xmodule/tests/test_stringify.py diff --git a/common/lib/xmodule/html_checker.py b/common/lib/xmodule/html_checker.py new file mode 100644 index 0000000000..5e6b417d28 --- /dev/null +++ b/common/lib/xmodule/html_checker.py @@ -0,0 +1,14 @@ +from lxml import etree + +def check_html(html): + ''' + Check whether the passed in html string can be parsed by lxml. + Return bool success. + ''' + parser = etree.HTMLParser() + try: + etree.fromstring(html, parser) + return True + except Exception as err: + pass + return False diff --git a/common/lib/xmodule/stringify.py b/common/lib/xmodule/stringify.py new file mode 100644 index 0000000000..dad964140f --- /dev/null +++ b/common/lib/xmodule/stringify.py @@ -0,0 +1,20 @@ +from itertools import chain +from lxml import etree + +def stringify_children(node): + ''' + Return all contents of an xml tree, without the outside tags. + e.g. if node is parse of + "Hi
      there Bruce!
      " + should return + "Hi
      there Bruce!
      " + + fixed from + http://stackoverflow.com/questions/4624062/get-all-text-inside-a-tag-in-lxml + ''' + parts = ([node.text] + + list(chain(*([etree.tostring(c), c.tail] + for c in node.getchildren()) + ))) + # filter removes possible Nones in texts and tails + return ''.join(filter(None, parts)) diff --git a/common/lib/xmodule/tests/test_stringify.py b/common/lib/xmodule/tests/test_stringify.py new file mode 100644 index 0000000000..62d7683886 --- /dev/null +++ b/common/lib/xmodule/tests/test_stringify.py @@ -0,0 +1,9 @@ +from nose.tools import assert_equals +from lxml import etree +from stringify import stringify_children + +def test_stringify(): + html = '''Hi
      there Bruce!
      ''' + xml = etree.fromstring(html) + out = stringify_children(xml) + assert_equals(out, '''Hi
      there Bruce!
      ''') diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index b9bc34aed6..22d3bf163f 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -1,13 +1,17 @@ +import copy +from fs.errors import ResourceNotFoundError import logging import os from lxml import etree from xmodule.x_module import XModule -from xmodule.raw_module import RawDescriptor +from xmodule.xml_module import XmlDescriptor +from xmodule.editing_module import EditingDescriptor +from stringify import stringify_children +from html_checker import check_html log = logging.getLogger("mitx.courseware") - class HtmlModule(XModule): def get_html(self): return self.html @@ -19,33 +23,108 @@ class HtmlModule(XModule): self.html = self.definition['data'] -class HtmlDescriptor(RawDescriptor): +class HtmlDescriptor(XmlDescriptor, EditingDescriptor): """ Module for putting raw html in a course """ mako_template = "widgets/html-edit.html" module_class = HtmlModule - filename_extension = "html" + filename_extension = "xml" - # TODO (cpennington): Delete this method once all fall 2012 course are being - # edited in the cms + # VS[compat] TODO (cpennington): Delete this method once all fall 2012 course + # are being edited in the cms @classmethod def backcompat_paths(cls, path): - if path.endswith('.html.html'): - path = path[:-5] + origpath = path + if path.endswith('.html.xml'): + path = path[:-9] + '.html' #backcompat--look for html instead of xml candidates = [] while os.sep in path: candidates.append(path) _, _, path = path.partition(os.sep) + # also look for .html versions instead of .xml + if origpath.endswith('.xml'): + candidates.append(origpath[:-4] + '.html') return candidates + # NOTE: html descriptors are special. We do not want to parse and + # export them ourselves, because that can break things (e.g. lxml + # adds body tags when it exports, but they should just be html + # snippets that will be included in the middle of pages. + @classmethod - def file_to_xml(cls, file_object): - parser = etree.HTMLParser() - return etree.parse(file_object, parser).getroot() + def definition_loader(cls, xml_object, system): + '''Load a descriptor from the specified xml_object: + + If there is a filename attribute, load it as a string, and + log a warning if it is not parseable by etree.HTMLParser. + + If there is not a filename attribute, the definition is the body + of the xml_object, without the root tag (do not want in the + middle of a page) + ''' + filename = xml_object.get('filename') + if filename is None: + definition_xml = copy.deepcopy(xml_object) + cls.clean_metadata_from_xml(definition_xml) + return {'data' : stringify_children(definition_xml)} + else: + filepath = cls._format_filepath(xml_object.tag, filename) + + # VS[compat] + # TODO (cpennington): If the file doesn't exist at the right path, + # give the class a chance to fix it up. The file will be written out + # again in the correct format. This should go away once the CMS is + # online and has imported all current (fall 2012) courses from xml + if not system.resources_fs.exists(filepath): + candidates = cls.backcompat_paths(filepath) + #log.debug("candidates = {0}".format(candidates)) + for candidate in candidates: + if system.resources_fs.exists(candidate): + filepath = candidate + break + + try: + with system.resources_fs.open(filepath) as file: + html = file.read() + # Log a warning if we can't parse the file, but don't error + if not check_html(html): + log.warning("Couldn't parse html in {0}.".format(filepath)) + return {'data' : html} + except (ResourceNotFoundError) as err: + msg = 'Unable to load file contents at path {0}: {1} '.format( + filepath, err) + log.error(msg) + raise @classmethod def split_to_file(cls, xml_object): - # never include inline html + '''Never include inline html''' return True + + + # TODO (vshnayder): make export put things in the right places. + + def definition_to_xml(self, resource_fs): + '''If the contents are valid xml, write them to filename.xml. Otherwise, + write just the tag to filename.xml, and the html + string to filename.html. + ''' + try: + return etree.fromstring(self.definition['data']) + except etree.XMLSyntaxError: + pass + + # Not proper format. Write html to file, return an empty tag + filepath = u'{category}/{name}.html'.format(category=self.category, + name=self.name) + + resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) + with resource_fs.open(filepath, 'w') as file: + file.write(self.definition['data']) + + elt = etree.Element('html') + elt.set("filename", self.name) + return elt + diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index 5786dbd227..188df21130 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -162,6 +162,52 @@ class XmlDescriptor(XModuleDescriptor): """ return etree.parse(file_object).getroot() + @classmethod + def definition_loader(cls, xml_object, system, location): + '''Load a descriptor definition from the specified xml_object. + Subclasses should not need to override this except in special + cases (e.g. html module)''' + + filename = xml_object.get('filename') + if filename is None: + definition_xml = copy.deepcopy(xml_object) + else: + filepath = cls._format_filepath(xml_object.tag, filename) + + # VS[compat] + # TODO (cpennington): If the file doesn't exist at the right path, + # give the class a chance to fix it up. The file will be written out again + # in the correct format. + # This should go away once the CMS is online and has imported all current (fall 2012) + # courses from xml + if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_paths'): + candidates = cls.backcompat_paths(filepath) + for candidate in candidates: + if system.resources_fs.exists(candidate): + filepath = candidate + break + + try: + with system.resources_fs.open(filepath) as file: + definition_xml = cls.file_to_xml(file) + except (ResourceNotFoundError, etree.XMLSyntaxError): + msg = 'Unable to load file contents at path %s for item %s' % ( + filepath, location.url()) + + log.exception(msg) + system.error_handler(msg) + # if error_handler didn't reraise, work around problem. + error_elem = etree.Element('error') + message_elem = etree.SubElement(error_elem, 'error_message') + message_elem.text = msg + stack_elem = etree.SubElement(error_elem, 'stack_trace') + stack_elem.text = traceback.format_exc() + return {'data': etree.tostring(error_elem)} + + cls.clean_metadata_from_xml(definition_xml) + return cls.definition_from_xml(definition_xml, system) + + @classmethod def from_xml(cls, xml_data, system, org=None, course=None): """ @@ -191,44 +237,8 @@ class XmlDescriptor(XModuleDescriptor): metadata[attr_map.metadata_key] = attr_map.to_metadata(val) return metadata - def definition_loader(): - filename = xml_object.get('filename') - if filename is None: - definition_xml = copy.deepcopy(xml_object) - else: - filepath = cls._format_filepath(xml_object.tag, filename) - - # VS[compat] - # TODO (cpennington): If the file doesn't exist at the right path, - # give the class a chance to fix it up. The file will be written out again - # in the correct format. - # This should go away once the CMS is online and has imported all current (fall 2012) - # courses from xml - if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_paths'): - candidates = cls.backcompat_paths(filepath) - for candidate in candidates: - if system.resources_fs.exists(candidate): - filepath = candidate - break - - try: - with system.resources_fs.open(filepath) as file: - definition_xml = cls.file_to_xml(file) - except (ResourceNotFoundError, etree.XMLSyntaxError): - msg = 'Unable to load file contents at path %s for item %s' % (filepath, location.url()) - log.exception(msg) - system.error_handler(msg) - # if error_handler didn't reraise, work around problem. - error_elem = etree.Element('error') - message_elem = etree.SubElement(error_elem, 'error_message') - message_elem.text = msg - stack_elem = etree.SubElement(error_elem, 'stack_trace') - stack_elem.text = traceback.format_exc() - return {'data': etree.tostring(error_elem)} - - cls.clean_metadata_from_xml(definition_xml) - return cls.definition_from_xml(definition_xml, system) - + definition = cls.definition_loader(xml_object, system, location) + metadata = metadata_loader() # VS[compat] -- just have the url_name lookup once translation is done slug = xml_object.get('url_name', xml_object.get('slug')) return cls( @@ -283,7 +293,7 @@ class XmlDescriptor(XModuleDescriptor): # Write it to a file if necessary if self.split_to_file(xml_object): - # Put this object in it's own file + # Put this object in its own file filepath = self.__class__._format_filepath(self.category, self.name) resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) with resource_fs.open(filepath, 'w') as file: diff --git a/lms/djangoapps/courseware/management/commands/clean_xml.py b/lms/djangoapps/courseware/management/commands/clean_xml.py index 7523fd8373..41f0c39cde 100644 --- a/lms/djangoapps/courseware/management/commands/clean_xml.py +++ b/lms/djangoapps/courseware/management/commands/clean_xml.py @@ -143,6 +143,7 @@ def check_roundtrip(course_dir): # dircmp doesn't do recursive diffs. # diff = dircmp(course_dir, export_dir, ignore=[], hide=[]) print "======== Roundtrip diff: =========" + sys.stdout.flush() # needed to make diff appear in the right place os.system("diff -r {0} {1}".format(course_dir, export_dir)) print "======== ideally there is no diff above this =======" From 0ae434cc092617b970fee3968084c378d609c4f2 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 11:24:39 -0400 Subject: [PATCH 039/124] Move path_to_location out of mongo.py * also bugfix for load_definition in html_module * a bit of refactoring of Location checking code in mongo.py --- common/lib/xmodule/xmodule/html_module.py | 2 +- .../xmodule/xmodule/modulestore/__init__.py | 47 +++---- .../lib/xmodule/xmodule/modulestore/mongo.py | 120 +++--------------- .../lib/xmodule/xmodule/modulestore/search.py | 96 ++++++++++++++ .../xmodule/modulestore/tests/test_mongo.py | 15 +-- common/lib/xmodule/xmodule/xml_module.py | 8 +- lms/djangoapps/courseware/views.py | 3 +- 7 files changed, 155 insertions(+), 136 deletions(-) create mode 100644 common/lib/xmodule/xmodule/modulestore/search.py diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 22d3bf163f..996d3c4ead 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -54,7 +54,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): # snippets that will be included in the middle of pages. @classmethod - def definition_loader(cls, xml_object, system): + def load_definition(cls, xml_object, system, location): '''Load a descriptor from the specified xml_object: If there is a filename attribute, load it as a string, and diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 279782b61a..a0167d781b 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -5,7 +5,7 @@ that are stored in a database an accessible using their Location as an identifie import re from collections import namedtuple -from .exceptions import InvalidLocationError +from .exceptions import InvalidLocationError, InsufficientSpecificationError import logging log = logging.getLogger('mitx.' + 'modulestore') @@ -38,15 +38,15 @@ class Location(_LocationBase): ''' __slots__ = () - @classmethod - def clean(cls, value): + @staticmethod + def clean(value): """ Return value, made into a form legal for locations """ return re.sub('_+', '_', INVALID_CHARS.sub('_', value)) - @classmethod - def is_valid(cls, value): + @staticmethod + def is_valid(value): ''' Check if the value is a valid location, in any acceptable format. ''' @@ -56,6 +56,21 @@ class Location(_LocationBase): return False return True + @staticmethod + def ensure_fully_specified(location): + '''Make sure location is valid, and fully specified. Raises + InvalidLocationError or InsufficientSpecificationError if not. + + returns a Location object corresponding to location. + ''' + loc = Location(location) + for key, val in loc.dict().iteritems(): + if key != 'revision' and val is None: + raise InsufficientSpecificationError(location) + return loc + + + def __new__(_cls, loc_or_tag=None, org=None, course=None, category=None, name=None, revision=None): """ @@ -254,25 +269,11 @@ class ModuleStore(object): ''' raise NotImplementedError - def path_to_location(self, location, course=None, chapter=None, section=None): - ''' - Try to find a course/chapter/section[/position] path to this location. + def get_parent_locations(self, location): + '''Find all locations that are the parents of this location. Needed + for path_to_location(). - raise ItemNotFoundError if the location doesn't exist. - - If course, chapter, section are not None, restrict search to paths with those - components as specified. - - raise NoPathToItem if the location exists, but isn't accessible via - a path that matches the course/chapter/section restrictions. - - In general, a location may be accessible via many paths. This method may - return any valid path. - - Return a tuple (course, chapter, section, position). - - If the section a sequence, position should be the position of this location - in that sequence. Otherwise, position should be None. + returns an iterable of things that can be passed to Location. ''' raise NotImplementedError diff --git a/common/lib/xmodule/xmodule/modulestore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py index df4e20f3a7..061e2aafe9 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -9,11 +9,10 @@ from importlib import import_module from xmodule.errorhandlers import strict_error_handler from xmodule.x_module import XModuleDescriptor from xmodule.mako_module import MakoDescriptorSystem -from xmodule.course_module import CourseDescriptor from mitxmako.shortcuts import render_to_string from . import ModuleStore, Location -from .exceptions import (ItemNotFoundError, InsufficientSpecificationError, +from .exceptions import (ItemNotFoundError, NoPathToItem, DuplicateItemError) # TODO (cpennington): This code currently operates under the assumption that @@ -172,12 +171,17 @@ class MongoModuleStore(ModuleStore): return self.get_items(course_filter) def _find_one(self, location): - '''Look for a given location in the collection. - If revision isn't specified, returns the latest.''' - return self.collection.find_one( + '''Look for a given location in the collection. If revision is not + specified, returns the latest. If the item is not present, raise + ItemNotFoundError. + ''' + item = self.collection.find_one( location_to_query(location), sort=[('revision', pymongo.ASCENDING)], ) + if item is None: + raise ItemNotFoundError(location) + return item def get_item(self, location, depth=0): """ @@ -197,14 +201,8 @@ class MongoModuleStore(ModuleStore): calls to get_children() to cache. None indicates to cache all descendents. """ - - for key, val in Location(location).dict().iteritems(): - if key != 'revision' and val is None: - raise InsufficientSpecificationError(location) - + location = Location.ensure_fully_specified(location) item = self._find_one(location) - if item is None: - raise ItemNotFoundError(location) return self._load_items([item], depth)[0] def get_items(self, location, depth=0): @@ -282,96 +280,20 @@ class MongoModuleStore(ModuleStore): ) def get_parent_locations(self, location): - '''Find all locations that are the parents of this location. - Mostly intended for use in path_to_location, but exposed for testing - and possible other usefulness. + '''Find all locations that are the parents of this location. Needed + for path_to_location(). - returns an iterable of things that can be passed to Location. + If there is no data at location in this modulestore, raise + ItemNotFoundError. + + returns an iterable of things that can be passed to Location. This may + be empty if there are no parents. ''' - location = Location(location) + location = Location.ensure_fully_specified(location) + # Check that it's actually in this modulestore. + item = self._find_one(location) + # now get the parents items = self.collection.find({'definition.children': str(location)}, {'_id': True}) return [i['_id'] for i in items] - def path_to_location(self, location, course_name=None): - ''' - Try to find a course_id/chapter/section[/position] path to this location. - The courseware insists that the first level in the course is chapter, - but any kind of module can be a "section". - - location: something that can be passed to Location - course_name: [optional]. If not None, restrict search to paths - in that course. - - raise ItemNotFoundError if the location doesn't exist. - - raise NoPathToItem if the location exists, but isn't accessible via - a chapter/section path in the course(s) being searched. - - Return a tuple (course_id, chapter, section, position) suitable for the - courseware index view. - - A location may be accessible via many paths. This method may - return any valid path. - - If the section is a sequence, position will be the position - of this location in that sequence. Otherwise, position will - be None. TODO (vshnayder): Not true yet. - ''' - # Check that location is present at all - if self._find_one(location) is None: - raise ItemNotFoundError(location) - - def flatten(xs): - '''Convert lisp-style (a, (b, (c, ()))) lists into a python list. - Not a general flatten function. ''' - p = [] - while xs != (): - p.append(xs[0]) - xs = xs[1] - return p - - def find_path_to_course(location, course_name=None): - '''Find a path up the location graph to a node with the - specified category. If no path exists, return None. If a - path exists, return it as a list with target location - first, and the starting location last. - ''' - # Standard DFS - - # To keep track of where we came from, the work queue has - # tuples (location, path-so-far). To avoid lots of - # copying, the path-so-far is stored as a lisp-style - # list--nested hd::tl tuples, and flattened at the end. - queue = [(location, ())] - while len(queue) > 0: - (loc, path) = queue.pop() # Takes from the end - loc = Location(loc) - # print 'Processing loc={0}, path={1}'.format(loc, path) - if loc.category == "course": - if course_name is None or course_name == loc.name: - # Found it! - path = (loc, path) - return flatten(path) - - # otherwise, add parent locations at the end - newpath = (loc, path) - parents = self.get_parent_locations(loc) - queue.extend(zip(parents, repeat(newpath))) - - # If we're here, there is no path - return None - - path = find_path_to_course(location, course_name) - if path is None: - raise(NoPathToItem(location)) - - n = len(path) - course_id = CourseDescriptor.location_to_id(path[0]) - chapter = path[1].name if n > 1 else None - section = path[2].name if n > 2 else None - - # TODO (vshnayder): not handling position at all yet... - position = None - - return (course_id, chapter, section, position) diff --git a/common/lib/xmodule/xmodule/modulestore/search.py b/common/lib/xmodule/xmodule/modulestore/search.py new file mode 100644 index 0000000000..a383b3f8ec --- /dev/null +++ b/common/lib/xmodule/xmodule/modulestore/search.py @@ -0,0 +1,96 @@ +from itertools import repeat + +from xmodule.course_module import CourseDescriptor + +from .exceptions import (ItemNotFoundError, NoPathToItem) +from . import ModuleStore, Location + + +def path_to_location(modulestore, location, course_name=None): + ''' + Try to find a course_id/chapter/section[/position] path to location in + modulestore. The courseware insists that the first level in the course is + chapter, but any kind of module can be a "section". + + location: something that can be passed to Location + course_name: [optional]. If not None, restrict search to paths + in that course. + + raise ItemNotFoundError if the location doesn't exist. + + raise NoPathToItem if the location exists, but isn't accessible via + a chapter/section path in the course(s) being searched. + + Return a tuple (course_id, chapter, section, position) suitable for the + courseware index view. + + A location may be accessible via many paths. This method may + return any valid path. + + If the section is a sequence, position will be the position + of this location in that sequence. Otherwise, position will + be None. TODO (vshnayder): Not true yet. + ''' + + def flatten(xs): + '''Convert lisp-style (a, (b, (c, ()))) list into a python list. + Not a general flatten function. ''' + p = [] + while xs != (): + p.append(xs[0]) + xs = xs[1] + return p + + def find_path_to_course(location, course_name=None): + '''Find a path up the location graph to a node with the + specified category. + + If no path exists, return None. + + If a path exists, return it as a list with target location first, and + the starting location last. + ''' + # Standard DFS + + # To keep track of where we came from, the work queue has + # tuples (location, path-so-far). To avoid lots of + # copying, the path-so-far is stored as a lisp-style + # list--nested hd::tl tuples, and flattened at the end. + queue = [(location, ())] + while len(queue) > 0: + (loc, path) = queue.pop() # Takes from the end + loc = Location(loc) + + # get_parent_locations should raise ItemNotFoundError if location + # isn't found so we don't have to do it explicitly. Call this + # first to make sure the location is there (even if it's a course, and + # we would otherwise immediately exit). + parents = modulestore.get_parent_locations(loc) + + # print 'Processing loc={0}, path={1}'.format(loc, path) + if loc.category == "course": + if course_name is None or course_name == loc.name: + # Found it! + path = (loc, path) + return flatten(path) + + # otherwise, add parent locations at the end + newpath = (loc, path) + queue.extend(zip(parents, repeat(newpath))) + + # If we're here, there is no path + return None + + path = find_path_to_course(location, course_name) + if path is None: + raise(NoPathToItem(location)) + + n = len(path) + course_id = CourseDescriptor.location_to_id(path[0]) + chapter = path[1].name if n > 1 else None + section = path[2].name if n > 2 else None + + # TODO (vshnayder): not handling position at all yet... + position = None + + return (course_id, chapter, section, position) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py index cb2bc6e20c..24f0441ee0 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py @@ -8,6 +8,7 @@ from xmodule.modulestore import Location from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem from xmodule.modulestore.mongo import MongoModuleStore from xmodule.modulestore.xml_importer import import_from_xml +from xmodule.modulestore.search import path_to_location # from ~/mitx_all/mitx/common/lib/xmodule/xmodule/modulestore/tests/ # to ~/mitx_all/mitx/common/test @@ -28,7 +29,7 @@ DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor' class TestMongoModuleStore(object): - + '''Tests!''' @classmethod def setupClass(cls): cls.connection = pymongo.connection.Connection(HOST, PORT) @@ -67,7 +68,7 @@ class TestMongoModuleStore(object): def test_init(self): '''Make sure the db loads, and print all the locations in the db. - Call this directly from failing tests to see what's loaded''' + Call this directly from failing tests to see what is loaded''' ids = list(self.connection[DB][COLLECTION].find({}, {'_id': True})) pprint([Location(i['_id']).url() for i in ids]) @@ -93,8 +94,6 @@ class TestMongoModuleStore(object): self.store.get_item("i4x://edX/toy/video/Welcome"), None) - - def test_find_one(self): assert_not_equals( self.store._find_one(Location("i4x://edX/toy/course/2012_Fall")), @@ -117,13 +116,13 @@ class TestMongoModuleStore(object): ("edX/toy/2012_Fall", "Overview", "Toy_Videos", None)), ) for location, expected in should_work: - assert_equals(self.store.path_to_location(location), expected) + assert_equals(path_to_location(self.store, location), expected) not_found = ( - "i4x://edX/toy/video/WelcomeX", + "i4x://edX/toy/video/WelcomeX", "i4x://edX/toy/course/NotHome" ) for location in not_found: - assert_raises(ItemNotFoundError, self.store.path_to_location, location) + assert_raises(ItemNotFoundError, path_to_location, self.store, location) # Since our test files are valid, there shouldn't be any # elements with no path to them. But we can look for them in @@ -132,5 +131,5 @@ class TestMongoModuleStore(object): "i4x://edX/simple/video/Lost_Video", ) for location in no_path: - assert_raises(NoPathToItem, self.store.path_to_location, location, "toy") + assert_raises(NoPathToItem, path_to_location, self.store, location, "toy") diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index 188df21130..de62bc5d38 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -163,7 +163,7 @@ class XmlDescriptor(XModuleDescriptor): return etree.parse(file_object).getroot() @classmethod - def definition_loader(cls, xml_object, system, location): + def load_definition(cls, xml_object, system, location): '''Load a descriptor definition from the specified xml_object. Subclasses should not need to override this except in special cases (e.g. html module)''' @@ -225,7 +225,7 @@ class XmlDescriptor(XModuleDescriptor): slug = xml_object.get('url_name', xml_object.get('slug')) location = Location('i4x', org, course, xml_object.tag, slug) - def metadata_loader(): + def load_metadata(): metadata = {} for attr in cls.metadata_attributes: val = xml_object.get(attr) @@ -237,8 +237,8 @@ class XmlDescriptor(XModuleDescriptor): metadata[attr_map.metadata_key] = attr_map.to_metadata(val) return metadata - definition = cls.definition_loader(xml_object, system, location) - metadata = metadata_loader() + definition = cls.load_definition(xml_object, system, location) + metadata = load_metadata() # VS[compat] -- just have the url_name lookup once translation is done slug = xml_object.get('url_name', xml_object.get('slug')) return cls( diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 7281ab01ad..a52f715efd 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -20,6 +20,7 @@ from module_render import toc_for_course, get_module, get_section from models import StudentModuleCache from student.models import UserProfile from xmodule.modulestore import Location +from xmodule.modulestore.search import path_to_location from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem from xmodule.modulestore.django import modulestore from xmodule.course_module import CourseDescriptor @@ -233,7 +234,7 @@ def jump_to(request, location): # Complain if there's not data for this location try: - (course_id, chapter, section, position) = modulestore().path_to_location(location) + (course_id, chapter, section, position) = path_to_location(modulestore(), location) except ItemNotFoundError: raise Http404("No data at this location: {0}".format(location)) except NoPathToItem: From 0edc40de34f37c3322e440656e71d7dbe77c202a Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 11:25:10 -0400 Subject: [PATCH 040/124] Address minor comments in #313 --- common/lib/xmodule/xmodule/modulestore/mongo.py | 2 +- common/lib/xmodule/xmodule/x_module.py | 3 --- .../courseware/management/commands/clean_xml.py | 3 --- lms/djangoapps/courseware/tests/tests.py | 14 ++++++++------ 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py index 061e2aafe9..6a0bdad06a 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -293,7 +293,7 @@ class MongoModuleStore(ModuleStore): # Check that it's actually in this modulestore. item = self._find_one(location) # now get the parents - items = self.collection.find({'definition.children': str(location)}, + items = self.collection.find({'definition.children': location.url()}, {'_id': True}) return [i['_id'] for i in items] diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 8803c37d78..c4ba3e1e58 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -194,9 +194,6 @@ class XModule(HTMLSnippet): self.metadata = kwargs.get('metadata', {}) self._loaded_children = None - def get_name(self): - return self.name - def get_children(self): ''' Return module instances for all the children of this module. diff --git a/lms/djangoapps/courseware/management/commands/clean_xml.py b/lms/djangoapps/courseware/management/commands/clean_xml.py index 41f0c39cde..05865bb289 100644 --- a/lms/djangoapps/courseware/management/commands/clean_xml.py +++ b/lms/djangoapps/courseware/management/commands/clean_xml.py @@ -17,9 +17,6 @@ def traverse_tree(course): queue = [course] while len(queue) > 0: node = queue.pop() -# print '{0}:'.format(node.location) -# if 'data' in node.definition: -# print '{0}'.format(node.definition['data']) queue.extend(node.get_children()) return True diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index 8e9d13f8d5..0e2f7d7aa5 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -1,19 +1,20 @@ import copy import json +from path import path import os from pprint import pprint +from nose import SkipTest from django.test import TestCase from django.test.client import Client -from mock import patch, Mock -from override_settings import override_settings from django.conf import settings from django.core.urlresolvers import reverse -from path import path +from mock import patch, Mock +from override_settings import override_settings -from student.models import Registration from django.contrib.auth.models import User +from student.models import Registration from xmodule.modulestore.django import modulestore import xmodule.modulestore.django @@ -189,11 +190,12 @@ class RealCoursesLoadTestCase(PageLoader): xmodule.modulestore.django._MODULESTORES = {} xmodule.modulestore.django.modulestore().collection.drop() - # TODO: Disabled test for now.. Fix once things are cleaned up. - def Xtest_real_courses_loads(self): + def test_real_courses_loads(self): '''See if any real courses are available at the REAL_DATA_DIR. If they are, check them.''' + # TODO: Disabled test for now.. Fix once things are cleaned up. + raise SkipTest # TODO: adjust staticfiles_dirs if not os.path.isdir(REAL_DATA_DIR): # No data present. Just pass. From 707551b08d25bcf92572a323176aeeec0398aead Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 11:26:02 -0400 Subject: [PATCH 041/124] local variable naming tweak --- common/lib/xmodule/xmodule/modulestore/xml.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 8a084b19ee..b54963b513 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -70,18 +70,19 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): # log.debug('-> slug=%s' % slug) xml_data.set('url_name', slug) - module = XModuleDescriptor.load_from_xml( + descriptor = XModuleDescriptor.load_from_xml( etree.tostring(xml_data), self, org, course, xmlstore.default_class) - #log.debug('==> importing module location %s' % repr(module.location)) - module.metadata['data_dir'] = course_dir + #log.debug('==> importing descriptor location %s' % + # repr(descriptor.location)) + descriptor.metadata['data_dir'] = course_dir - xmlstore.modules[module.location] = module + xmlstore.modules[descriptor.location] = descriptor if xmlstore.eager: - module.get_children() - return module + descriptor.get_children() + return descriptor render_template = lambda: '' load_item = xmlstore.get_item From c0cdff7071431fb522a05b596bd0aa07232d1584 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 13:44:04 -0400 Subject: [PATCH 042/124] Rename MalformedDescriptor to ErrorDescriptor * change references and tests * add staff/non-staff display * added is_staff to ModuleSystem --- cms/djangoapps/contentstore/views.py | 5 ++- common/lib/xmodule/setup.py | 2 +- common/lib/xmodule/tests/__init__.py | 3 +- common/lib/xmodule/tests/test_import.py | 12 +++--- .../{malformed_module.py => error_module.py} | 41 +++++++++++-------- common/lib/xmodule/xmodule/x_module.py | 26 ++++++++---- lms/djangoapps/courseware/module_render.py | 1 + 7 files changed, 58 insertions(+), 32 deletions(-) rename common/lib/xmodule/xmodule/{malformed_module.py => error_module.py} (58%) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 6d5b2117c4..111c91e03a 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -214,7 +214,10 @@ def preview_module_system(request, preview_id, descriptor): get_module=partial(get_preview_module, request, preview_id), render_template=render_from_lms, debug=True, - replace_urls=replace_urls + replace_urls=replace_urls, + # TODO (vshnayder): CMS users won't see staff view unless they are CMS staff. + # is that what we want? + is_staff=request.user.is_staff ) diff --git a/common/lib/xmodule/setup.py b/common/lib/xmodule/setup.py index 09a6b7e712..8a0a6bb139 100644 --- a/common/lib/xmodule/setup.py +++ b/common/lib/xmodule/setup.py @@ -25,7 +25,7 @@ setup( "discuss = xmodule.backcompat_module:TranslateCustomTagDescriptor", "html = xmodule.html_module:HtmlDescriptor", "image = xmodule.backcompat_module:TranslateCustomTagDescriptor", - "malformed = xmodule.malformed_module:MalformedDescriptor", + "error = xmodule.error_module:ErrorDescriptor", "problem = xmodule.capa_module:CapaDescriptor", "problemset = xmodule.vertical_module:VerticalDescriptor", "section = xmodule.backcompat_module:SemanticSectionDescriptor", diff --git a/common/lib/xmodule/tests/__init__.py b/common/lib/xmodule/tests/__init__.py index 94bf1da65e..7f6fcfe00c 100644 --- a/common/lib/xmodule/tests/__init__.py +++ b/common/lib/xmodule/tests/__init__.py @@ -31,7 +31,8 @@ i4xs = ModuleSystem( user=Mock(), filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))), debug=True, - xqueue_callback_url='/' + xqueue_callback_url='/', + is_staff=False ) diff --git a/common/lib/xmodule/tests/test_import.py b/common/lib/xmodule/tests/test_import.py index dd55f8ff38..497354a18e 100644 --- a/common/lib/xmodule/tests/test_import.py +++ b/common/lib/xmodule/tests/test_import.py @@ -33,7 +33,7 @@ class ImportTestCase(unittest.TestCase): return system def test_fallback(self): - '''Make sure that malformed xml loads as a MalformedDescriptorb.''' + '''Make sure that malformed xml loads as an ErrorDescriptor.''' bad_xml = '''''' @@ -43,10 +43,10 @@ class ImportTestCase(unittest.TestCase): None) self.assertEqual(descriptor.__class__.__name__, - 'MalformedDescriptor') + 'ErrorDescriptor') def test_reimport(self): - '''Make sure an already-exported malformed xml tag loads properly''' + '''Make sure an already-exported error xml tag loads properly''' bad_xml = '''''' system = self.get_system() @@ -58,7 +58,7 @@ class ImportTestCase(unittest.TestCase): 'org', 'course', None) self.assertEqual(re_import_descriptor.__class__.__name__, - 'MalformedDescriptor') + 'ErrorDescriptor') self.assertEqual(descriptor.definition['data'], re_import_descriptor.definition['data']) @@ -66,8 +66,8 @@ class ImportTestCase(unittest.TestCase): def test_fixed_xml_tag(self): """Make sure a tag that's been fixed exports as the original tag type""" - # create a malformed tag with valid xml contents - root = etree.Element('malformed') + # create a error tag with valid xml contents + root = etree.Element('error') good_xml = '''''' root.text = good_xml diff --git a/common/lib/xmodule/xmodule/malformed_module.py b/common/lib/xmodule/xmodule/error_module.py similarity index 58% rename from common/lib/xmodule/xmodule/malformed_module.py rename to common/lib/xmodule/xmodule/error_module.py index ac419fff26..ee05e9c879 100644 --- a/common/lib/xmodule/xmodule/malformed_module.py +++ b/common/lib/xmodule/xmodule/error_module.py @@ -9,18 +9,26 @@ import logging log = logging.getLogger(__name__) -class MalformedModule(XModule): +class ErrorModule(XModule): def get_html(self): '''Show an error. TODO (vshnayder): proper style, divs, etc. ''' - return "Malformed content--not showing through get_html()" + if not self.system.is_staff: + return self.system.render_template('module-error.html') -class MalformedDescriptor(EditingDescriptor): + # staff get to see all the details + return self.system.render_template('module-error-staff.html', { + 'data' : self.definition['data'], + # TODO (vshnayder): need to get non-syntax errors in here somehow + 'error' : self.definition.get('error', 'Error not available') + }) + +class ErrorDescriptor(EditingDescriptor): """ Module that provides a raw editing view of broken xml. """ - module_class = MalformedModule + module_class = ErrorModule @classmethod def from_xml(cls, xml_data, system, org=None, course=None): @@ -29,38 +37,39 @@ class MalformedDescriptor(EditingDescriptor): Does not try to parse the data--just stores it. ''' + definition = {} try: - # If this is already a malformed tag, don't want to re-wrap it. + # If this is already an error tag, don't want to re-wrap it. xml_obj = etree.fromstring(xml_data) - if xml_obj.tag == 'malformed': + if xml_obj.tag == 'error': xml_data = xml_obj.text - # TODO (vshnayder): how does one get back from this to a valid descriptor? - # For now, have to fix manually. - except etree.XMLSyntaxError: - pass + except etree.XMLSyntaxError as err: + # Save the error to display later + definition['error'] = str(err) - definition = { 'data' : xml_data } - # TODO (vshnayder): Do we need a valid slug here? Just pick a random + + definition['data'] = xml_data + # TODO (vshnayder): Do we need a unique slug here? Just pick a random # 64-bit num? - location = ['i4x', org, course, 'malformed', 'slug'] + location = ['i4x', org, course, 'error', 'slug'] metadata = {} # stays in the xml_data return cls(system, definition, location=location, metadata=metadata) def export_to_xml(self, resource_fs): ''' - If the definition data is invalid xml, export it wrapped in a malformed + If the definition data is invalid xml, export it wrapped in an "error" tag. If it is valid, export without the wrapper. NOTE: There may still be problems with the valid xml--it could be missing required attributes, could have the wrong tags, refer to missing - files, etc. + files, etc. That would just get re-wrapped on import. ''' try: xml = etree.fromstring(self.definition['data']) return etree.tostring(xml) except etree.XMLSyntaxError: # still not valid. - root = etree.Element('malformed') + root = etree.Element('error') root.text = self.definition['data'] return etree.tostring(root) diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index c4ba3e1e58..2d7447b5f6 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -455,14 +455,14 @@ class XModuleDescriptor(Plugin, HTMLSnippet): descriptor = class_.from_xml(xml_data, system, org, course) except (ResourceNotFoundError, XMLSyntaxError) as err: - # Didn't load properly. Fall back on loading as a malformed + # Didn't load properly. Fall back on loading as an error # descriptor. This should never error due to formatting. # Put import here to avoid circular import errors - from xmodule.malformed_module import MalformedDescriptor + from xmodule.error_module import ErrorDescriptor #system.error_handler("Error loading from xml.") - descriptor = MalformedDescriptor.from_xml(xml_data, system, org, course) + descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course) return descriptor @@ -597,10 +597,18 @@ class ModuleSystem(object): Note that these functions can be closures over e.g. a django request and user, or other environment-specific info. ''' - def __init__(self, ajax_url, track_function, - get_module, render_template, replace_urls, - user=None, filestore=None, debug=False, - xqueue_callback_url=None, xqueue_default_queuename="null"): + def __init__(self, + ajax_url, + track_function, + get_module, + render_template, + replace_urls, + user=None, + filestore=None, + debug=False, + xqueue_callback_url=None, + xqueue_default_queuename="null", + is_staff=False): ''' Create a closure around the system environment. @@ -626,6 +634,9 @@ class ModuleSystem(object): replace_urls - TEMPORARY - A function like static_replace.replace_urls that capa_module can use to fix up the static urls in ajax results. + + is_staff - Is the user making the request a staff user? + TODO (vshnayder): this will need to change once we have real user roles. ''' self.ajax_url = ajax_url self.xqueue_callback_url = xqueue_callback_url @@ -637,6 +648,7 @@ class ModuleSystem(object): self.DEBUG = self.debug = debug self.seed = user.id if user is not None else 0 self.replace_urls = replace_urls + self.is_staff = is_staff def get(self, attr): ''' provide uniform access to attributes (like etree).''' diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 80a4ef90fc..1ff39f602e 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -172,6 +172,7 @@ def get_module(user, request, location, student_module_cache, position=None): # a module is coming through get_html and is therefore covered # by the replace_static_urls code below replace_urls=replace_urls, + is_staff=user.is_staff, ) # pass position specified in URL to module through ModuleSystem system.set('position', position) From e2e524453faa235ac90fac008839d6175dbb7379 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 13:44:40 -0400 Subject: [PATCH 043/124] Make courseware index view more bulletproof --- lms/djangoapps/courseware/views.py | 68 +++++++++++++++++++----------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index a52f715efd..fb4d2942f9 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -184,34 +184,54 @@ def index(request, course_id, chapter=None, section=None, chapter = clean(chapter) section = clean(section) - context = { - 'csrf': csrf(request)['csrf_token'], - 'accordion': render_accordion(request, course, chapter, section), - 'COURSE_TITLE': course.title, - 'course': course, - 'init': '', - 'content': '' - } + try: + context = { + 'csrf': csrf(request)['csrf_token'], + 'accordion': render_accordion(request, course, chapter, section), + 'COURSE_TITLE': course.title, + 'course': course, + 'init': '', + 'content': '' + } - look_for_module = chapter is not None and section is not None - if look_for_module: - # TODO (cpennington): Pass the right course in here + look_for_module = chapter is not None and section is not None + if look_for_module: + # TODO (cpennington): Pass the right course in here - section_descriptor = get_section(course, chapter, section) - if section_descriptor is not None: - student_module_cache = StudentModuleCache(request.user, - section_descriptor) - module, _, _, _ = get_module(request.user, request, - section_descriptor.location, - student_module_cache) - context['content'] = module.get_html() + section_descriptor = get_section(course, chapter, section) + if section_descriptor is not None: + student_module_cache = StudentModuleCache(request.user, + section_descriptor) + module, _, _, _ = get_module(request.user, request, + section_descriptor.location, + student_module_cache) + context['content'] = module.get_html() + else: + log.warning("Couldn't find a section descriptor for course_id '{0}'," + "chapter '{1}', section '{2}'".format( + course_id, chapter, section)) + + + result = render_to_response('courseware.html', context) + except: + # In production, don't want to let a 500 out for any reason + if settings.DEBUG: + raise else: - log.warning("Couldn't find a section descriptor for course_id '{0}'," - "chapter '{1}', section '{2}'".format( - course_id, chapter, section)) + log.exception("Error in index view: user={user}, course={course}," + " chapter={chapter} section={section}" + "position={position}".format( + user=request.user, + course=course, + chapter=chapter, + section=section, + position=position + )) + try: + result = render_to_response('courseware-error.html', {}) + except: + result = HttpResponse("There was an unrecoverable error") - - result = render_to_response('courseware.html', context) return result @ensure_csrf_cookie From bc14441a75c010dd98c9389ad9d83015f91cc091 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 13:45:13 -0400 Subject: [PATCH 044/124] add template I forgot --- lms/templates/module-error-staff.html | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 lms/templates/module-error-staff.html diff --git a/lms/templates/module-error-staff.html b/lms/templates/module-error-staff.html new file mode 100644 index 0000000000..955f413db3 --- /dev/null +++ b/lms/templates/module-error-staff.html @@ -0,0 +1,11 @@ +
      +

      There has been an error on the MITx servers

      +

      We're sorry, this module is temporarily unavailable. Our staff is working to fix it as soon as possible. Please email us at technical@mitx.mit.edu to report any problems or downtime.

      + +

      Staff-only details below:

      + +

      Error: ${error}

      + +

      Raw data: ${data}

      + +
      From 119ab639d0c816889a59c36540212eb1daed33fd Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 17:12:00 -0400 Subject: [PATCH 045/124] make cms users look like staff by default * so they get the staff view of problems * does NOT actually set User.is_staff --- cms/djangoapps/contentstore/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 111c91e03a..0fccc2498b 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -215,9 +215,9 @@ def preview_module_system(request, preview_id, descriptor): render_template=render_from_lms, debug=True, replace_urls=replace_urls, - # TODO (vshnayder): CMS users won't see staff view unless they are CMS staff. + # TODO (vshnayder): All CMS users get staff view by default # is that what we want? - is_staff=request.user.is_staff + is_staff=True, ) From 58543bd84b451c1da7497633e9550a3d6538e0c1 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 17:12:47 -0400 Subject: [PATCH 046/124] definition and metadata no longer lazy-loaded * get rid of dump to json * formatting --- common/djangoapps/xmodule_modifiers.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py index d0d1e0be15..b942d6726b 100644 --- a/common/djangoapps/xmodule_modifiers.py +++ b/common/djangoapps/xmodule_modifiers.py @@ -76,8 +76,9 @@ def add_histogram(get_html, module): histogram = grade_histogram(module_id) render_histogram = len(histogram) > 0 - # TODO: fixme - no filename in module.xml in general (this code block for edx4edx) - # the following if block is for summer 2012 edX course development; it will change when the CMS comes online + # TODO: fixme - no filename in module.xml in general (this code block + # for edx4edx) the following if block is for summer 2012 edX course + # development; it will change when the CMS comes online if settings.MITX_FEATURES.get('DISPLAY_EDIT_LINK') and settings.DEBUG and module_xml.get('filename') is not None: coursename = multicourse_settings.get_coursename_from_request(request) github_url = multicourse_settings.get_course_github_url(coursename) @@ -88,10 +89,8 @@ def add_histogram(get_html, module): else: edit_link = False - # Cast module.definition and module.metadata to dicts so that json can dump them - # even though they are lazily loaded - staff_context = {'definition': json.dumps(dict(module.definition), indent=4), - 'metadata': json.dumps(dict(module.metadata), indent=4), + staff_context = {'definition': dict(module.definition), + 'metadata': dict(module.metadata), 'element_id': module.location.html_id(), 'edit_link': edit_link, 'histogram': json.dumps(histogram), From 009bd230667bb722988e6aec69972ada6121d57d Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 17:13:49 -0400 Subject: [PATCH 047/124] save LazyLoadingDict in case we do want it later --- common/lib/xmodule/lazy_dict.py | 58 +++++++++++++++++++++++ common/lib/xmodule/xmodule/xml_module.py | 60 ------------------------ 2 files changed, 58 insertions(+), 60 deletions(-) create mode 100644 common/lib/xmodule/lazy_dict.py diff --git a/common/lib/xmodule/lazy_dict.py b/common/lib/xmodule/lazy_dict.py new file mode 100644 index 0000000000..fa614843ec --- /dev/null +++ b/common/lib/xmodule/lazy_dict.py @@ -0,0 +1,58 @@ +from collections import MutableMapping + +class LazyLoadingDict(MutableMapping): + """ + A dictionary object that lazily loads its contents from a provided + function on reads (of members that haven't already been set). + """ + + def __init__(self, loader): + ''' + On the first read from this dictionary, it will call loader() to + populate its contents. loader() must return something dict-like. Any + elements set before the first read will be preserved. + ''' + self._contents = {} + self._loaded = False + self._loader = loader + self._deleted = set() + + def __getitem__(self, name): + if not (self._loaded or name in self._contents or name in self._deleted): + self.load() + + return self._contents[name] + + def __setitem__(self, name, value): + self._contents[name] = value + self._deleted.discard(name) + + def __delitem__(self, name): + del self._contents[name] + self._deleted.add(name) + + def __contains__(self, name): + self.load() + return name in self._contents + + def __len__(self): + self.load() + return len(self._contents) + + def __iter__(self): + self.load() + return iter(self._contents) + + def __repr__(self): + self.load() + return repr(self._contents) + + def load(self): + if self._loaded: + return + + loaded_contents = self._loader() + loaded_contents.update(self._contents) + self._contents = loaded_contents + self._loaded = True + diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index de62bc5d38..89f1dc4ed2 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -11,66 +11,6 @@ import os log = logging.getLogger(__name__) -# # TODO (cpennington): This was implemented in an attempt to improve performance, -# # but the actual improvement wasn't measured (and it was implemented late at night). -# # We should check if it hurts, and whether there's a better way of doing lazy loading - -# class LazyLoadingDict(MutableMapping): -# """ -# A dictionary object that lazily loads its contents from a provided -# function on reads (of members that haven't already been set). -# """ - -# def __init__(self, loader): -# ''' -# On the first read from this dictionary, it will call loader() to -# populate its contents. loader() must return something dict-like. Any -# elements set before the first read will be preserved. -# ''' -# self._contents = {} -# self._loaded = False -# self._loader = loader -# self._deleted = set() - -# def __getitem__(self, name): -# if not (self._loaded or name in self._contents or name in self._deleted): -# self.load() - -# return self._contents[name] - -# def __setitem__(self, name, value): -# self._contents[name] = value -# self._deleted.discard(name) - -# def __delitem__(self, name): -# del self._contents[name] -# self._deleted.add(name) - -# def __contains__(self, name): -# self.load() -# return name in self._contents - -# def __len__(self): -# self.load() -# return len(self._contents) - -# def __iter__(self): -# self.load() -# return iter(self._contents) - -# def __repr__(self): -# self.load() -# return repr(self._contents) - -# def load(self): -# if self._loaded: -# return - -# loaded_contents = self._loader() -# loaded_contents.update(self._contents) -# self._contents = loaded_contents -# self._loaded = True - _AttrMapBase = namedtuple('_AttrMap', 'metadata_key to_metadata from_metadata') From 0b67d1c40118c3a584c32bfe5b32f0f8c0a187df Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 17:20:04 -0400 Subject: [PATCH 048/124] Turn error_handlers into error_trackers * simplify logic--tracker just tracks errors. Trackers should not raise, and are not be responsible for logging. * adapted code to use trackers. * Started cleanup of error handling code: - if need to add info and re-raise, just do that. No logging. - if working around a problem, log and track as needed. --- common/lib/xmodule/tests/test_import.py | 4 +- .../lib/xmodule/xmodule/backcompat_module.py | 11 +++-- common/lib/xmodule/xmodule/error_module.py | 15 ++++--- common/lib/xmodule/xmodule/errorhandlers.py | 45 ------------------- common/lib/xmodule/xmodule/errortracker.py | 33 ++++++++++++++ common/lib/xmodule/xmodule/mako_module.py | 4 +- .../lib/xmodule/xmodule/modulestore/mongo.py | 27 ++++++----- common/lib/xmodule/xmodule/modulestore/xml.py | 20 ++++----- common/lib/xmodule/xmodule/raw_module.py | 10 ++--- common/lib/xmodule/xmodule/x_module.py | 38 ++++++++-------- common/lib/xmodule/xmodule/xml_module.py | 29 +++++------- .../management/commands/clean_xml.py | 23 ++-------- 12 files changed, 117 insertions(+), 142 deletions(-) delete mode 100644 common/lib/xmodule/xmodule/errorhandlers.py create mode 100644 common/lib/xmodule/xmodule/errortracker.py diff --git a/common/lib/xmodule/tests/test_import.py b/common/lib/xmodule/tests/test_import.py index 497354a18e..34ace135a3 100644 --- a/common/lib/xmodule/tests/test_import.py +++ b/common/lib/xmodule/tests/test_import.py @@ -4,7 +4,7 @@ import unittest from lxml import etree from xmodule.x_module import XMLParsingSystem, XModuleDescriptor -from xmodule.errorhandlers import ignore_errors_handler +from xmodule.errortracker import null_error_tracker from xmodule.modulestore import Location class ImportTestCase(unittest.TestCase): @@ -27,7 +27,7 @@ class ImportTestCase(unittest.TestCase): raise Exception("Shouldn't be called") system = XMLParsingSystem(load_item, resources_fs, - ignore_errors_handler, process_xml) + null_error_tracker, process_xml) system.render_template = render_template return system diff --git a/common/lib/xmodule/xmodule/backcompat_module.py b/common/lib/xmodule/xmodule/backcompat_module.py index da0d6788e4..7ace342d1b 100644 --- a/common/lib/xmodule/xmodule/backcompat_module.py +++ b/common/lib/xmodule/xmodule/backcompat_module.py @@ -35,18 +35,21 @@ def process_includes(fn): # insert new XML into tree in place of include parent.insert(parent.index(next_include), incxml) except Exception: - msg = "Error in problem xml include: %s" % (etree.tostring(next_include, pretty_print=True)) - log.exception(msg) - parent = next_include.getparent() + # Log error and work around it + msg = "Error in problem xml include: %s" % ( + etree.tostring(next_include, pretty_print=True)) + system.error_tracker(msg) + + parent = next_include.getparent() errorxml = etree.Element('error') messagexml = etree.SubElement(errorxml, 'message') messagexml.text = msg stackxml = etree.SubElement(errorxml, 'stacktrace') stackxml.text = traceback.format_exc() - # insert error XML in place of include parent.insert(parent.index(next_include), errorxml) + parent.remove(next_include) next_include = xml_object.find('include') diff --git a/common/lib/xmodule/xmodule/error_module.py b/common/lib/xmodule/xmodule/error_module.py index ee05e9c879..62b3b82a39 100644 --- a/common/lib/xmodule/xmodule/error_module.py +++ b/common/lib/xmodule/xmodule/error_module.py @@ -15,7 +15,7 @@ class ErrorModule(XModule): TODO (vshnayder): proper style, divs, etc. ''' if not self.system.is_staff: - return self.system.render_template('module-error.html') + return self.system.render_template('module-error.html', {}) # staff get to see all the details return self.system.render_template('module-error-staff.html', { @@ -31,22 +31,27 @@ class ErrorDescriptor(EditingDescriptor): module_class = ErrorModule @classmethod - def from_xml(cls, xml_data, system, org=None, course=None): + def from_xml(cls, xml_data, system, org=None, course=None, err=None): '''Create an instance of this descriptor from the supplied data. Does not try to parse the data--just stores it. + + Takes an extra, optional, parameter--the error that caused an + issue. ''' definition = {} + if err is not None: + definition['error'] = err + try: # If this is already an error tag, don't want to re-wrap it. xml_obj = etree.fromstring(xml_data) if xml_obj.tag == 'error': xml_data = xml_obj.text except etree.XMLSyntaxError as err: - # Save the error to display later - definition['error'] = str(err) - + # Save the error to display later--overrides other problems + definition['error'] = err definition['data'] = xml_data # TODO (vshnayder): Do we need a unique slug here? Just pick a random diff --git a/common/lib/xmodule/xmodule/errorhandlers.py b/common/lib/xmodule/xmodule/errorhandlers.py deleted file mode 100644 index 0f97377b2a..0000000000 --- a/common/lib/xmodule/xmodule/errorhandlers.py +++ /dev/null @@ -1,45 +0,0 @@ -import logging -import sys - -log = logging.getLogger(__name__) - -def in_exception_handler(): - '''Is there an active exception?''' - return sys.exc_info() != (None, None, None) - -def strict_error_handler(msg, exc_info=None): - ''' - Do not let errors pass. If exc_info is not None, ignore msg, and just - re-raise. Otherwise, check if we are in an exception-handling context. - If so, re-raise. Otherwise, raise Exception(msg). - - Meant for use in validation, where any errors should trap. - ''' - if exc_info is not None: - raise exc_info[0], exc_info[1], exc_info[2] - - if in_exception_handler(): - raise - - raise Exception(msg) - - -def logging_error_handler(msg, exc_info=None): - '''Log all errors, but otherwise let them pass, relying on the caller to - workaround.''' - if exc_info is not None: - log.exception(msg, exc_info=exc_info) - return - - if in_exception_handler(): - log.exception(msg) - return - - log.error(msg) - - -def ignore_errors_handler(msg, exc_info=None): - '''Ignore all errors, relying on the caller to workaround. - Meant for use in the LMS, where an error in one part of the course - shouldn't bring down the whole system''' - pass diff --git a/common/lib/xmodule/xmodule/errortracker.py b/common/lib/xmodule/xmodule/errortracker.py new file mode 100644 index 0000000000..8dd893e814 --- /dev/null +++ b/common/lib/xmodule/xmodule/errortracker.py @@ -0,0 +1,33 @@ +import logging +import sys + +log = logging.getLogger(__name__) + +def in_exception_handler(): + '''Is there an active exception?''' + return sys.exc_info() != (None, None, None) + + +def make_error_tracker(): + '''Return a tuple (logger, error_list), where + the logger appends a tuple (message, exc_info=None) + to the error_list on every call. + + error_list is a simple list. If the caller messes with it, info + will be lost. + ''' + errors = [] + + def error_tracker(msg): + '''Log errors''' + exc_info = None + if in_exception_handler(): + exc_info = sys.exc_info() + + errors.append((msg, exc_info)) + + return (error_tracker, errors) + +def null_error_tracker(msg): + '''A dummy error tracker that just ignores the messages''' + pass diff --git a/common/lib/xmodule/xmodule/mako_module.py b/common/lib/xmodule/xmodule/mako_module.py index 213e9077db..eedac99aa8 100644 --- a/common/lib/xmodule/xmodule/mako_module.py +++ b/common/lib/xmodule/xmodule/mako_module.py @@ -2,10 +2,10 @@ from x_module import XModuleDescriptor, DescriptorSystem class MakoDescriptorSystem(DescriptorSystem): - def __init__(self, load_item, resources_fs, error_handler, + def __init__(self, load_item, resources_fs, error_tracker, render_template, **kwargs): super(MakoDescriptorSystem, self).__init__( - load_item, resources_fs, error_handler, **kwargs) + load_item, resources_fs, error_tracker, **kwargs) self.render_template = render_template diff --git a/common/lib/xmodule/xmodule/modulestore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py index 6a0bdad06a..76769b25b0 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -6,7 +6,7 @@ from itertools import repeat from path import path from importlib import import_module -from xmodule.errorhandlers import strict_error_handler +from xmodule.errortracker import null_error_tracker from xmodule.x_module import XModuleDescriptor from xmodule.mako_module import MakoDescriptorSystem from mitxmako.shortcuts import render_to_string @@ -26,7 +26,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem): from, with a backup of calling to the underlying modulestore for more data """ def __init__(self, modulestore, module_data, default_class, resources_fs, - error_handler, render_template): + error_tracker, render_template): """ modulestore: the module store that can be used to retrieve additional modules @@ -38,13 +38,13 @@ class CachingDescriptorSystem(MakoDescriptorSystem): resources_fs: a filesystem, as per MakoDescriptorSystem - error_handler: + error_tracker: render_template: a function for rendering templates, as per MakoDescriptorSystem """ super(CachingDescriptorSystem, self).__init__( - self.load_item, resources_fs, error_handler, render_template) + self.load_item, resources_fs, error_tracker, render_template) self.modulestore = modulestore self.module_data = module_data self.default_class = default_class @@ -79,7 +79,8 @@ class MongoModuleStore(ModuleStore): """ # TODO (cpennington): Enable non-filesystem filestores - def __init__(self, host, db, collection, fs_root, port=27017, default_class=None): + def __init__(self, host, db, collection, fs_root, port=27017, default_class=None, + error_tracker=null_error_tracker): self.collection = pymongo.connection.Connection( host=host, port=port @@ -90,13 +91,17 @@ class MongoModuleStore(ModuleStore): # Force mongo to maintain an index over _id.* that is in the same order # that is used when querying by a location - self.collection.ensure_index(zip(('_id.' + field for field in Location._fields), repeat(1))) + self.collection.ensure_index( + zip(('_id.' + field for field in Location._fields), repeat(1))) - # TODO (vshnayder): default arg default_class=None will make this error - module_path, _, class_name = default_class.rpartition('.') - class_ = getattr(import_module(module_path), class_name) - self.default_class = class_ + if default_class is not None: + module_path, _, class_name = default_class.rpartition('.') + class_ = getattr(import_module(module_path), class_name) + self.default_class = class_ + else: + self.default_class = None self.fs_root = path(fs_root) + self.error_tracker = error_tracker def _clean_item_data(self, item): """ @@ -148,7 +153,7 @@ class MongoModuleStore(ModuleStore): data_cache, self.default_class, resource_fs, - strict_error_handler, + self.error_tracker, render_to_string, ) return system.load_item(item['location']) diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index b54963b513..06e824e3d1 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -3,7 +3,7 @@ from fs.osfs import OSFS from importlib import import_module from lxml import etree from path import path -from xmodule.errorhandlers import logging_error_handler +from xmodule.errortracker import null_error_tracker from xmodule.x_module import XModuleDescriptor, XMLParsingSystem from xmodule.mako_module import MakoDescriptorSystem from cStringIO import StringIO @@ -29,7 +29,7 @@ def clean_out_mako_templating(xml_string): return xml_string class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): - def __init__(self, xmlstore, org, course, course_dir, error_handler, **kwargs): + def __init__(self, xmlstore, org, course, course_dir, error_tracker, **kwargs): """ A class that handles loading from xml. Does some munging to ensure that all elements have unique slugs. @@ -89,9 +89,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): resources_fs = OSFS(xmlstore.data_dir / course_dir) MakoDescriptorSystem.__init__(self, load_item, resources_fs, - error_handler, render_template, **kwargs) + error_tracker, render_template, **kwargs) XMLParsingSystem.__init__(self, load_item, resources_fs, - error_handler, process_xml, **kwargs) + error_tracker, process_xml, **kwargs) @@ -101,7 +101,7 @@ class XMLModuleStore(ModuleStore): """ def __init__(self, data_dir, default_class=None, eager=False, course_dirs=None, - error_handler=logging_error_handler): + error_tracker=null_error_tracker): """ Initialize an XMLModuleStore from data_dir @@ -116,8 +116,8 @@ class XMLModuleStore(ModuleStore): course_dirs: If specified, the list of course_dirs to load. Otherwise, load all course dirs - error_handler: The error handler used here and in the underlying - DescriptorSystem. By default, raise exceptions for all errors. + error_tracker: The error tracker used here and in the underlying + DescriptorSystem. By default, ignore all messages. See the comments in x_module.py:DescriptorSystem """ @@ -125,7 +125,7 @@ class XMLModuleStore(ModuleStore): self.data_dir = path(data_dir) self.modules = {} # location -> XModuleDescriptor self.courses = {} # course_dir -> XModuleDescriptor for the course - self.error_handler = error_handler + self.error_tracker = error_tracker if default_class is None: self.default_class = None @@ -154,7 +154,7 @@ class XMLModuleStore(ModuleStore): except: msg = "Failed to load course '%s'" % course_dir log.exception(msg) - error_handler(msg) + error_tracker(msg) def load_course(self, course_dir): @@ -192,7 +192,7 @@ class XMLModuleStore(ModuleStore): course = course_dir system = ImportSystem(self, org, course, course_dir, - self.error_handler) + self.error_tracker) course_descriptor = system.process_xml(etree.tostring(course_data)) log.debug('========> Done with course import from {0}'.format(course_dir)) diff --git a/common/lib/xmodule/xmodule/raw_module.py b/common/lib/xmodule/xmodule/raw_module.py index f9f358f945..9fdb5d0b38 100644 --- a/common/lib/xmodule/xmodule/raw_module.py +++ b/common/lib/xmodule/xmodule/raw_module.py @@ -1,9 +1,8 @@ -from pkg_resources import resource_string from lxml import etree from xmodule.editing_module import EditingDescriptor -from xmodule.mako_module import MakoModuleDescriptor from xmodule.xml_module import XmlDescriptor import logging +import sys log = logging.getLogger(__name__) @@ -20,13 +19,12 @@ class RawDescriptor(XmlDescriptor, EditingDescriptor): try: return etree.fromstring(self.definition['data']) except etree.XMLSyntaxError as err: + # Can't recover here, so just add some info and + # re-raise lines = self.definition['data'].split('\n') line, offset = err.position msg = ("Unable to create xml for problem {loc}. " "Context: '{context}'".format( context=lines[line - 1][offset - 40:offset + 40], loc=self.location)) - log.exception(msg) - self.system.error_handler(msg) - # no workaround possible, so just re-raise - raise + raise Exception, msg, sys.exc_info()[2] diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 2d7447b5f6..a14f83eee6 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -454,15 +454,15 @@ class XModuleDescriptor(Plugin, HTMLSnippet): # etree.fromstring(xml_data).tag,class_)) descriptor = class_.from_xml(xml_data, system, org, course) - except (ResourceNotFoundError, XMLSyntaxError) as err: + except Exception as err: # Didn't load properly. Fall back on loading as an error # descriptor. This should never error due to formatting. # Put import here to avoid circular import errors from xmodule.error_module import ErrorDescriptor - #system.error_handler("Error loading from xml.") - descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course) + system.error_tracker("Error loading from xml.") + descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course, err) return descriptor @@ -533,16 +533,19 @@ class XModuleDescriptor(Plugin, HTMLSnippet): class DescriptorSystem(object): - def __init__(self, load_item, resources_fs, error_handler, **kwargs): + def __init__(self, load_item, resources_fs, error_tracker, **kwargs): """ load_item: Takes a Location and returns an XModuleDescriptor resources_fs: A Filesystem object that contains all of the resources needed for the course - error_handler: A hook for handling errors in loading the descriptor. - Must be a function of (error_msg, exc_info=None). - See errorhandlers.py for some simple ones. + error_tracker: A hook for tracking errors in loading the descriptor. + Used for example to get a list of all non-fatal problems on course + load, and display them to the user. + + A function of (error_msg). errortracker.py provides a + handy make_error_tracker() function. Patterns for using the error handler: try: @@ -551,10 +554,8 @@ class DescriptorSystem(object): except SomeProblem: msg = 'Grommet {0} is broken'.format(x) log.exception(msg) # don't rely on handler to log - self.system.error_handler(msg) - # if we get here, work around if possible - raise # if no way to work around - OR + self.system.error_tracker(msg) + # work around return 'Oops, couldn't load grommet' OR, if not in an exception context: @@ -562,25 +563,26 @@ class DescriptorSystem(object): if not check_something(thingy): msg = "thingy {0} is broken".format(thingy) log.critical(msg) - error_handler(msg) - # if we get here, work around - pass # e.g. if no workaround needed + self.system.error_tracker(msg) + + NOTE: To avoid duplication, do not call the tracker on errors + that you're about to re-raise---let the caller track them. """ self.load_item = load_item self.resources_fs = resources_fs - self.error_handler = error_handler + self.error_tracker = error_tracker class XMLParsingSystem(DescriptorSystem): - def __init__(self, load_item, resources_fs, error_handler, process_xml, **kwargs): + def __init__(self, load_item, resources_fs, error_tracker, process_xml, **kwargs): """ - load_item, resources_fs, error_handler: see DescriptorSystem + load_item, resources_fs, error_tracker: see DescriptorSystem process_xml: Takes an xml string, and returns a XModuleDescriptor created from that xml """ - DescriptorSystem.__init__(self, load_item, resources_fs, error_handler, + DescriptorSystem.__init__(self, load_item, resources_fs, error_tracker, **kwargs) self.process_xml = process_xml diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index 89f1dc4ed2..ea13bc2640 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -1,4 +1,3 @@ -from collections import MutableMapping from xmodule.x_module import XModuleDescriptor from xmodule.modulestore import Location from lxml import etree @@ -8,13 +7,12 @@ import traceback from collections import namedtuple from fs.errors import ResourceNotFoundError import os +import sys log = logging.getLogger(__name__) - _AttrMapBase = namedtuple('_AttrMap', 'metadata_key to_metadata from_metadata') - class AttrMap(_AttrMapBase): """ A class that specifies a metadata_key, and two functions: @@ -116,11 +114,12 @@ class XmlDescriptor(XModuleDescriptor): # VS[compat] # TODO (cpennington): If the file doesn't exist at the right path, - # give the class a chance to fix it up. The file will be written out again - # in the correct format. - # This should go away once the CMS is online and has imported all current (fall 2012) - # courses from xml - if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_paths'): + # give the class a chance to fix it up. The file will be written out + # again in the correct format. This should go away once the CMS is + # online and has imported all current (fall 2012) courses from xml + if not system.resources_fs.exists(filepath) and hasattr( + cls, + 'backcompat_paths'): candidates = cls.backcompat_paths(filepath) for candidate in candidates: if system.resources_fs.exists(candidate): @@ -130,19 +129,11 @@ class XmlDescriptor(XModuleDescriptor): try: with system.resources_fs.open(filepath) as file: definition_xml = cls.file_to_xml(file) - except (ResourceNotFoundError, etree.XMLSyntaxError): + except Exception: msg = 'Unable to load file contents at path %s for item %s' % ( filepath, location.url()) - - log.exception(msg) - system.error_handler(msg) - # if error_handler didn't reraise, work around problem. - error_elem = etree.Element('error') - message_elem = etree.SubElement(error_elem, 'error_message') - message_elem.text = msg - stack_elem = etree.SubElement(error_elem, 'stack_trace') - stack_elem.text = traceback.format_exc() - return {'data': etree.tostring(error_elem)} + # Add info about where we are, but keep the traceback + raise Exception, msg, sys.exc_info()[2] cls.clean_metadata_from_xml(definition_xml) return cls.definition_from_xml(definition_xml, system) diff --git a/lms/djangoapps/courseware/management/commands/clean_xml.py b/lms/djangoapps/courseware/management/commands/clean_xml.py index 05865bb289..29ce246637 100644 --- a/lms/djangoapps/courseware/management/commands/clean_xml.py +++ b/lms/djangoapps/courseware/management/commands/clean_xml.py @@ -10,7 +10,7 @@ from lxml import etree from django.core.management.base import BaseCommand from xmodule.modulestore.xml import XMLModuleStore - +from xmodule.errortracker import make_error_tracker def traverse_tree(course): '''Load every descriptor in course. Return bool success value.''' @@ -21,23 +21,6 @@ def traverse_tree(course): return True -def make_logging_error_handler(): - '''Return a tuple (handler, error_list), where - the handler appends the message and any exc_info - to the error_list on every call. - ''' - errors = [] - - def error_handler(msg, exc_info=None): - '''Log errors''' - if exc_info is None: - if sys.exc_info() != (None, None, None): - exc_info = sys.exc_info() - - errors.append((msg, exc_info)) - - return (error_handler, errors) - def export(course, export_dir): """Export the specified course to course_dir. Creates dir if it doesn't exist. @@ -70,14 +53,14 @@ def import_with_checks(course_dir, verbose=True): data_dir = course_dir.dirname() course_dirs = [course_dir.basename()] - (error_handler, errors) = make_logging_error_handler() + (error_tracker, errors) = make_error_tracker() # No default class--want to complain if it doesn't find plugins for any # module. modulestore = XMLModuleStore(data_dir, default_class=None, eager=True, course_dirs=course_dirs, - error_handler=error_handler) + error_tracker=error_tracker) def str_of_err(tpl): (msg, exc_info) = tpl From 740c9b7df1b51dcafd5e4babc132ce894aae2411 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 17:20:18 -0400 Subject: [PATCH 049/124] fixed docstring for customtag --- common/lib/xmodule/xmodule/template_module.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/common/lib/xmodule/xmodule/template_module.py b/common/lib/xmodule/xmodule/template_module.py index 3f926555f4..b4818dc29d 100644 --- a/common/lib/xmodule/xmodule/template_module.py +++ b/common/lib/xmodule/xmodule/template_module.py @@ -7,16 +7,14 @@ from mako.template import Template class CustomTagModule(XModule): """ This module supports tags of the form - - $tagname - + In this case, $tagname should refer to a file in data/custom_tags, which contains a mako template that uses ${option} and ${option2} for the content. For instance: - data/custom_tags/book:: + data/mycourse/custom_tags/book:: More information given in the text course.xml:: From 463b758434eae2a33d9fb82f992cb139b25c50ea Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 17:20:33 -0400 Subject: [PATCH 050/124] fixed args in call to add_histogram --- lms/djangoapps/courseware/module_render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 1ff39f602e..91d4efa651 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -185,7 +185,7 @@ def get_module(user, request, location, student_module_cache, position=None): ) if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff: - module.get_html = add_histogram(module.get_html) + module.get_html = add_histogram(module.get_html, module) # If StudentModule for this instance wasn't already in the database, # and this isn't a guest user, create it. From 32253510d1892a14357a766aa97e23a9cbc3da73 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Tue, 31 Jul 2012 19:37:53 -0400 Subject: [PATCH 051/124] Import error cleanup * call error tracker when needed * remove duplicate logging--just add info and re-raise * xml modulestore uses error tracker to capture load errors * add unstyled list of import errors to courseware homepage! --- .../lib/xmodule/xmodule/backcompat_module.py | 5 +- common/lib/xmodule/xmodule/capa_module.py | 25 ++++++--- common/lib/xmodule/xmodule/course_module.py | 14 +++-- common/lib/xmodule/xmodule/errortracker.py | 10 ++-- common/lib/xmodule/xmodule/html_module.py | 5 +- .../xmodule/xmodule/modulestore/__init__.py | 13 +++++ common/lib/xmodule/xmodule/modulestore/xml.py | 54 ++++++++++++------- common/lib/xmodule/xmodule/x_module.py | 5 +- lms/djangoapps/courseware/courses.py | 1 + lms/djangoapps/courseware/views.py | 15 ++++-- lms/templates/courseware.html | 7 +++ 11 files changed, 115 insertions(+), 39 deletions(-) diff --git a/common/lib/xmodule/xmodule/backcompat_module.py b/common/lib/xmodule/xmodule/backcompat_module.py index 7ace342d1b..c49f23b99e 100644 --- a/common/lib/xmodule/xmodule/backcompat_module.py +++ b/common/lib/xmodule/xmodule/backcompat_module.py @@ -35,12 +35,13 @@ def process_includes(fn): # insert new XML into tree in place of include parent.insert(parent.index(next_include), incxml) except Exception: - # Log error and work around it + # Log error msg = "Error in problem xml include: %s" % ( etree.tostring(next_include, pretty_print=True)) - + # tell the tracker system.error_tracker(msg) + # work around parent = next_include.getparent() errorxml = etree.Element('error') messagexml = etree.SubElement(errorxml, 'message') diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 2fae8b94e2..eed2cf3ac7 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -5,6 +5,7 @@ import json import logging import traceback import re +import sys from datetime import timedelta from lxml import etree @@ -91,7 +92,8 @@ class CapaModule(XModule): display_due_date_string = self.metadata.get('due', None) if display_due_date_string is not None: self.display_due_date = dateutil.parser.parse(display_due_date_string) - #log.debug("Parsed " + display_due_date_string + " to " + str(self.display_due_date)) + #log.debug("Parsed " + display_due_date_string + + # " to " + str(self.display_due_date)) else: self.display_due_date = None @@ -99,7 +101,8 @@ class CapaModule(XModule): if grace_period_string is not None and self.display_due_date: self.grace_period = parse_timedelta(grace_period_string) self.close_date = self.display_due_date + self.grace_period - #log.debug("Then parsed " + grace_period_string + " to closing date" + str(self.close_date)) + #log.debug("Then parsed " + grace_period_string + + # " to closing date" + str(self.close_date)) else: self.grace_period = None self.close_date = self.display_due_date @@ -138,10 +141,16 @@ class CapaModule(XModule): try: self.lcp = LoncapaProblem(self.definition['data'], self.location.html_id(), instance_state, seed=seed, system=self.system) - except Exception: - msg = 'cannot create LoncapaProblem %s' % self.location.url() - log.exception(msg) + except Exception as err: + msg = 'cannot create LoncapaProblem {loc}: {err}'.format( + loc=self.location.url(), err=err) + # TODO (vshnayder): do modules need error handlers too? + # We shouldn't be switching on DEBUG. if self.system.DEBUG: + log.error(msg) + # TODO (vshnayder): This logic should be general, not here--and may + # want to preserve the data instead of replacing it. + # e.g. in the CMS msg = '

      %s

      ' % msg.replace('<', '<') msg += '

      %s

      ' % traceback.format_exc().replace('<', '<') # create a dummy problem with error message instead of failing @@ -152,7 +161,8 @@ class CapaModule(XModule): problem_text, self.location.html_id(), instance_state, seed=seed, system=self.system) else: - raise + # add extra info and raise + raise Exception(msg), None, sys.exc_info()[2] @property def rerandomize(self): @@ -191,6 +201,7 @@ class CapaModule(XModule): try: return Progress(score, total) except Exception as err: + # TODO (vshnayder): why is this still here? still needed? if self.system.DEBUG: return None raise @@ -210,6 +221,7 @@ class CapaModule(XModule): try: html = self.lcp.get_html() except Exception, err: + # TODO (vshnayder): another switch on DEBUG. if self.system.DEBUG: log.exception(err) msg = ( @@ -561,6 +573,7 @@ class CapaDescriptor(RawDescriptor): 'problems/' + path[8:], path[8:], ] + @classmethod def split_to_file(cls, xml_object): '''Problems always written in their own files''' diff --git a/common/lib/xmodule/xmodule/course_module.py b/common/lib/xmodule/xmodule/course_module.py index 7e0019205e..acdc574220 100644 --- a/common/lib/xmodule/xmodule/course_module.py +++ b/common/lib/xmodule/xmodule/course_module.py @@ -20,15 +20,22 @@ class CourseDescriptor(SequenceDescriptor): self._grader = None self._grade_cutoffs = None + msg = None try: self.start = time.strptime(self.metadata["start"], "%Y-%m-%dT%H:%M") except KeyError: self.start = time.gmtime(0) #The epoch - log.critical("Course loaded without a start date. %s", self.id) + msg = "Course loaded without a start date. id = %s" % self.id + log.critical(msg) except ValueError as e: self.start = time.gmtime(0) #The epoch - log.critical("Course loaded with a bad start date. %s '%s'", - self.id, e) + msg = "Course loaded with a bad start date. %s '%s'" % (self.id, e) + log.critical(msg) + + # Don't call the tracker from the exception handler. + if msg is not None: + system.error_tracker(msg) + def has_started(self): return time.gmtime() > self.start @@ -104,3 +111,4 @@ class CourseDescriptor(SequenceDescriptor): @property def org(self): return self.location.org + diff --git a/common/lib/xmodule/xmodule/errortracker.py b/common/lib/xmodule/xmodule/errortracker.py index 8dd893e814..09c04c061c 100644 --- a/common/lib/xmodule/xmodule/errortracker.py +++ b/common/lib/xmodule/xmodule/errortracker.py @@ -1,17 +1,21 @@ import logging import sys +from collections import namedtuple + log = logging.getLogger(__name__) +ErrorLog = namedtuple('ErrorLog', 'tracker errors') + def in_exception_handler(): '''Is there an active exception?''' return sys.exc_info() != (None, None, None) def make_error_tracker(): - '''Return a tuple (logger, error_list), where + '''Return an ErrorLog (named tuple), with fields (tracker, errors), where the logger appends a tuple (message, exc_info=None) - to the error_list on every call. + to the errors on every call. error_list is a simple list. If the caller messes with it, info will be lost. @@ -26,7 +30,7 @@ def make_error_tracker(): errors.append((msg, exc_info)) - return (error_tracker, errors) + return ErrorLog(error_tracker, errors) def null_error_tracker(msg): '''A dummy error tracker that just ignores the messages''' diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 996d3c4ead..165c402487 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -2,6 +2,7 @@ import copy from fs.errors import ResourceNotFoundError import logging import os +import sys from lxml import etree from xmodule.x_module import XModule @@ -95,8 +96,8 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): except (ResourceNotFoundError) as err: msg = 'Unable to load file contents at path {0}: {1} '.format( filepath, err) - log.error(msg) - raise + # add more info and re-raise + raise Exception(msg), None, sys.exc_info()[2] @classmethod def split_to_file(cls, xml_object): diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index a0167d781b..7241179d8e 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -213,6 +213,18 @@ class ModuleStore(object): """ raise NotImplementedError + def get_item_errors(self, location): + """ + Return a list of (msg, exception-or-None) errors that the modulestore + encountered when loading the item at location. + + location : something that can be passed to Location + + Raises the same exceptions as get_item if the location isn't found or + isn't fully specified. + """ + raise NotImplementedError + def get_items(self, location, depth=0): """ Returns a list of XModuleDescriptor instances for the items @@ -269,6 +281,7 @@ class ModuleStore(object): ''' raise NotImplementedError + def get_parent_locations(self, location): '''Find all locations that are the parents of this location. Needed for path_to_location(). diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 06e824e3d1..e7f9c9ce0a 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -1,14 +1,16 @@ import logging +import os +import re + from fs.osfs import OSFS from importlib import import_module from lxml import etree from path import path -from xmodule.errortracker import null_error_tracker +from xmodule.errortracker import ErrorLog, make_error_tracker from xmodule.x_module import XModuleDescriptor, XMLParsingSystem +from xmodule.course_module import CourseDescriptor from xmodule.mako_module import MakoDescriptorSystem from cStringIO import StringIO -import os -import re from . import ModuleStore, Location from .exceptions import ItemNotFoundError @@ -19,7 +21,6 @@ etree.set_default_parser( log = logging.getLogger('mitx.' + __name__) - # VS[compat] # TODO (cpennington): Remove this once all fall 2012 courses have been imported # into the cms from xml @@ -94,14 +95,12 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): error_tracker, process_xml, **kwargs) - class XMLModuleStore(ModuleStore): """ An XML backed ModuleStore """ def __init__(self, data_dir, default_class=None, eager=False, - course_dirs=None, - error_tracker=null_error_tracker): + course_dirs=None): """ Initialize an XMLModuleStore from data_dir @@ -115,17 +114,14 @@ class XMLModuleStore(ModuleStore): course_dirs: If specified, the list of course_dirs to load. Otherwise, load all course dirs - - error_tracker: The error tracker used here and in the underlying - DescriptorSystem. By default, ignore all messages. - See the comments in x_module.py:DescriptorSystem """ self.eager = eager self.data_dir = path(data_dir) self.modules = {} # location -> XModuleDescriptor self.courses = {} # course_dir -> XModuleDescriptor for the course - self.error_tracker = error_tracker + self.location_errors = {} # location -> ErrorLog + if default_class is None: self.default_class = None @@ -149,15 +145,18 @@ class XMLModuleStore(ModuleStore): for course_dir in course_dirs: try: - course_descriptor = self.load_course(course_dir) + # make a tracker, then stick in the right place once the course loads + # and we know its location + errorlog = make_error_tracker() + course_descriptor = self.load_course(course_dir, errorlog.tracker) self.courses[course_dir] = course_descriptor + self.location_errors[course_descriptor.location] = errorlog except: msg = "Failed to load course '%s'" % course_dir log.exception(msg) - error_tracker(msg) - def load_course(self, course_dir): + def load_course(self, course_dir, tracker): """ Load a course into this module store course_path: Course directory name @@ -191,13 +190,13 @@ class XMLModuleStore(ModuleStore): )) course = course_dir - system = ImportSystem(self, org, course, course_dir, - self.error_tracker) + system = ImportSystem(self, org, course, course_dir, tracker) course_descriptor = system.process_xml(etree.tostring(course_data)) log.debug('========> Done with course import from {0}'.format(course_dir)) return course_descriptor + def get_item(self, location, depth=0): """ Returns an XModuleDescriptor instance for the item at location. @@ -218,9 +217,28 @@ class XMLModuleStore(ModuleStore): except KeyError: raise ItemNotFoundError(location) + + def get_item_errors(self, location): + """ + Return list of errors for this location, if any. Raise the same + errors as get_item if location isn't present. + + NOTE: This only actually works for courses in the xml datastore-- + will return an empty list for all other modules. + """ + location = Location(location) + # check that item is present + self.get_item(location) + + # now look up errors + if location in self.location_errors: + return self.location_errors[location].errors + return [] + def get_courses(self, depth=0): """ - Returns a list of course descriptors + Returns a list of course descriptors. If there were errors on loading, + some of these may be ErrorDescriptors instead. """ return self.courses.values() diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index a14f83eee6..2a1769bbd7 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -460,8 +460,9 @@ class XModuleDescriptor(Plugin, HTMLSnippet): # Put import here to avoid circular import errors from xmodule.error_module import ErrorDescriptor - - system.error_tracker("Error loading from xml.") + msg = "Error loading from xml." + log.exception(msg) + system.error_tracker(msg) descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course, err) return descriptor diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index c2c391b08e..cfda6497d5 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -33,6 +33,7 @@ def check_course(course_id, course_must_be_open=True, course_required=True): try: course_loc = CourseDescriptor.id_to_location(course_id) course = modulestore().get_item(course_loc) + except (KeyError, ItemNotFoundError): raise Http404("Course not found.") diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index fb4d2942f9..a049638d1b 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -55,14 +55,20 @@ def user_groups(user): def format_url_params(params): - return [urllib.quote(string.replace(' ', '_')) for string in params] + return [urllib.quote(string.replace(' ', '_')) + if string is not None else None + for string in params] @ensure_csrf_cookie @cache_if_anonymous def courses(request): # TODO: Clean up how 'error' is done. - courses = sorted(modulestore().get_courses(), key=lambda course: course.number) + + # filter out any courses that errored. + courses = [c for c in modulestore().get_courses() + if isinstance(c, CourseDescriptor)] + courses = sorted(courses, key=lambda course: course.number) universities = defaultdict(list) for course in courses: universities[course.org].append(course) @@ -210,7 +216,10 @@ def index(request, course_id, chapter=None, section=None, log.warning("Couldn't find a section descriptor for course_id '{0}'," "chapter '{1}', section '{2}'".format( course_id, chapter, section)) - + else: + if request.user.is_staff: + # Add a list of all the errors... + context['course_errors'] = modulestore().get_item_errors(course.location) result = render_to_response('courseware.html', context) except: diff --git a/lms/templates/courseware.html b/lms/templates/courseware.html index 9c145ba8c0..0955134694 100644 --- a/lms/templates/courseware.html +++ b/lms/templates/courseware.html @@ -47,6 +47,13 @@
      ${content} + + % if course_errors is not UNDEFINED: +

      Course errors

      +
      + ${course_errors} +
      + % endif

    • From 05c22c4901158bdca2c23242473aacd0b2cf6325 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Wed, 1 Aug 2012 09:29:31 -0400 Subject: [PATCH 052/124] Prettier error display * Log formatted traceback string instead of exc_info tuple itself * display as a list --- common/lib/xmodule/xmodule/errortracker.py | 13 +++++++------ lms/templates/courseware.html | 8 +++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/common/lib/xmodule/xmodule/errortracker.py b/common/lib/xmodule/xmodule/errortracker.py index 09c04c061c..b8d42a6983 100644 --- a/common/lib/xmodule/xmodule/errortracker.py +++ b/common/lib/xmodule/xmodule/errortracker.py @@ -1,5 +1,6 @@ import logging import sys +import traceback from collections import namedtuple @@ -14,21 +15,21 @@ def in_exception_handler(): def make_error_tracker(): '''Return an ErrorLog (named tuple), with fields (tracker, errors), where - the logger appends a tuple (message, exc_info=None) - to the errors on every call. + the logger appends a tuple (message, exception_str) to the errors on every + call. exception_str is in the format returned by traceback.format_exception. - error_list is a simple list. If the caller messes with it, info + error_list is a simple list. If the caller modifies it, info will be lost. ''' errors = [] def error_tracker(msg): '''Log errors''' - exc_info = None + exc_str = '' if in_exception_handler(): - exc_info = sys.exc_info() + exc_str = ''.join(traceback.format_exception(*sys.exc_info())) - errors.append((msg, exc_info)) + errors.append((msg, exc_str)) return ErrorLog(error_tracker, errors) diff --git a/lms/templates/courseware.html b/lms/templates/courseware.html index 0955134694..a14f35d154 100644 --- a/lms/templates/courseware.html +++ b/lms/templates/courseware.html @@ -51,7 +51,13 @@ % if course_errors is not UNDEFINED:

    Course errors

    - ${course_errors} +
      + % for (msg, err) in course_errors: +
    • ${msg} + 
      • ${err}
      +
      • + % endfor +
    % endif
    From 7fb831a2e8f398e4cf1b9863b26627d6e5777a0e Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Wed, 1 Aug 2012 09:36:13 -0400 Subject: [PATCH 053/124] Record warning on bad html files --- common/lib/xmodule/xmodule/html_module.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 165c402487..b2a5df9803 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -91,7 +91,9 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): html = file.read() # Log a warning if we can't parse the file, but don't error if not check_html(html): - log.warning("Couldn't parse html in {0}.".format(filepath)) + msg = "Couldn't parse html in {0}.".format(filepath) + log.warning(msg) + system.error_tracker("Warning: " + msg) return {'data' : html} except (ResourceNotFoundError) as err: msg = 'Unable to load file contents at path {0}: {1} '.format( From 10ebcefd5427a8083581a877990517d0485ee5b1 Mon Sep 17 00:00:00 2001 From: ichuang Date: Wed, 1 Aug 2012 14:59:55 -0400 Subject: [PATCH 054/124] add openid authentication --- common/djangoapps/student/views.py | 45 +++++- lms/envs/dev.py | 12 ++ lms/envs/dev_ike.py | 222 ++++++++++++----------------- lms/templates/extauth_failure.html | 11 ++ lms/templates/index.html | 9 ++ lms/templates/login_modal.html | 3 + lms/templates/signup_modal.html | 15 ++ lms/urls.py | 12 +- requirements.txt | 2 + 9 files changed, 199 insertions(+), 132 deletions(-) create mode 100644 lms/templates/extauth_failure.html diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index b739e0e37c..692b135ff6 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -61,6 +61,15 @@ def index(request): if settings.COURSEWARE_ENABLED and request.user.is_authenticated(): return redirect(reverse('dashboard')) + return main_index() + +def main_index(extra_context = {}): + ''' + Render the edX main page. + + extra_context is used to allow immediate display of certain modal windows, eg signup, + as used by external_auth. + ''' feed_data = cache.get("students_index_rss_feed_data") if feed_data == None: if hasattr(settings, 'RSS_URL'): @@ -81,8 +90,9 @@ def index(request): for course in courses: universities[course.org].append(course) - return render_to_response('index.html', {'universities': universities, 'entries': entries}) - + context = {'universities': universities, 'entries': entries} + context.update(extra_context) + return render_to_response('index.html', context) def course_from_id(id): course_loc = CourseDescriptor.id_to_location(id) @@ -257,11 +267,26 @@ def change_setting(request): @ensure_csrf_cookie def create_account(request, post_override=None): - ''' JSON call to enroll in the course. ''' + ''' + JSON call to create new edX account. + Used by form in signup_modal.html, which is included into navigation.html + ''' js = {'success': False} post_vars = post_override if post_override else request.POST + # if doing signup for an external authorization, then get email, password, name from the eamap + # don't use the ones from the form, since the user could have hacked those + doExternalAuth = 'ExternalAuthMap' in request.session + if doExternalAuth: + eamap = request.session['ExternalAuthMap'] + email = eamap.external_email + name = eamap.external_name + password = eamap.internal_password + post_vars = dict(post_vars.items()) + post_vars.update(dict(email=email, name=name, password=password, username=post_vars['username'])) + log.debug('extauth test: post_vars = %s' % post_vars) + # Confirm we have a properly formed request for a in ['username', 'email', 'password', 'name']: if a not in post_vars: @@ -356,8 +381,9 @@ def create_account(request, post_override=None): 'key': r.activation_key, } + # composes activation email subject = render_to_string('emails/activation_email_subject.txt', d) - # Email subject *must not* contain newlines + # Email subject *must not* contain newlines subject = ''.join(subject.splitlines()) message = render_to_string('emails/activation_email.txt', d) @@ -382,6 +408,17 @@ def create_account(request, post_override=None): try_change_enrollment(request) + if doExternalAuth: + eamap.user = login_user + eamap.dtsignup = datetime.datetime.now() + eamap.save() + log.debug('Updated ExternalAuthMap for %s to be %s' % (post_vars['username'],eamap)) + + if settings.MITX_FEATURES.get('BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'): + log.debug('bypassing activation email') + login_user.is_active = True + login_user.save() + js = {'success': True} return HttpResponse(json.dumps(js), mimetype="application/json") diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 1a2659cb1f..f9b7ba10a0 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -55,6 +55,18 @@ CACHES = { # Dummy secret key for dev SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' +################################ OpenID Auth ################################# +MITX_FEATURES['AUTH_USE_OPENID'] = True +MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True + +INSTALLED_APPS += ('external_auth',) +INSTALLED_APPS += ('django_openid_auth',) + +OPENID_CREATE_USERS = False +OPENID_UPDATE_DETAILS_FROM_SREG = True +OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id' # TODO: accept more endpoints +OPENID_USE_AS_ADMIN_LOGIN = False + ################################ DEBUG TOOLBAR ################################# INSTALLED_APPS += ('debug_toolbar',) MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) diff --git a/lms/envs/dev_ike.py b/lms/envs/dev_ike.py index 79ae3354ac..fb7d980550 100644 --- a/lms/envs/dev_ike.py +++ b/lms/envs/dev_ike.py @@ -7,142 +7,110 @@ sessions. Assumes structure: /mitx # The location of this repo /log # Where we're going to write log files """ - -import socket - -if 'eecs1' in socket.gethostname(): - MITX_ROOT_URL = '/mitx2' - from .common import * from .logsettings import get_logger_config -from .dev import * - -if 'eecs1' in socket.gethostname(): - # MITX_ROOT_URL = '/mitx2' - MITX_ROOT_URL = 'https://eecs1.mit.edu/mitx2' - -#----------------------------------------------------------------------------- -# edx4edx content server - -EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' -MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] = 'ichuang@mit.edu' -EDX4EDX_ROOT = ENV_ROOT / "data/edx4edx" - -#EMAIL_BACKEND = 'django_ses.SESBackend' - -#----------------------------------------------------------------------------- -# ichuang DEBUG = True -ENABLE_MULTICOURSE = True # set to False to disable multicourse display (see lib.util.views.mitxhome) -QUICKEDIT = False +TEMPLATE_DEBUG = True -MAKO_TEMPLATES['course'] = [DATA_DIR, EDX4EDX_ROOT ] +MITX_FEATURES['DISABLE_START_DATES'] = True -#MITX_FEATURES['USE_DJANGO_PIPELINE'] = False -MITX_FEATURES['DISPLAY_HISTOGRAMS_TO_STAFF'] = False -MITX_FEATURES['DISPLAY_EDIT_LINK'] = True -MITX_FEATURES['DEBUG_LEVEL'] = 10 # 0 = lowest level, least verbose, 255 = max level, most verbose +WIKI_ENABLED = True -COURSE_SETTINGS = {'6.002x_Fall_2012': {'number' : '6.002x', - 'title' : 'Circuits and Electronics', - 'xmlpath': '/6002x-fall-2012/', - 'active' : True, - 'default_chapter' : 'Week_1', - 'default_section' : 'Administrivia_and_Circuit_Elements', - 'location': 'i4x://edx/6002xs12/course/6.002x_Fall_2012', - }, - '8.02_Spring_2013': {'number' : '8.02x', - 'title' : 'Electricity & Magnetism', - 'xmlpath': '/802x/', - 'github_url': 'https://github.com/MITx/8.02x', - 'active' : True, - 'default_chapter' : 'Introduction', - 'default_section' : 'Introduction_%28Lewin_2002%29', - }, - '6.189_Spring_2013': {'number' : '6.189x', - 'title' : 'IAP Python Programming', - 'xmlpath': '/6.189x/', - 'github_url': 'https://github.com/MITx/6.189x', - 'active' : True, - 'default_chapter' : 'Week_1', - 'default_section' : 'Variables_and_Binding', - }, - '8.01_Fall_2012': {'number' : '8.01x', - 'title' : 'Mechanics', - 'xmlpath': '/8.01x/', - 'github_url': 'https://github.com/MITx/8.01x', - 'active': True, - 'default_chapter' : 'Mechanics_Online_Spring_2012', - 'default_section' : 'Introduction_to_the_course', - 'location': 'i4x://edx/6002xs12/course/8.01_Fall_2012', - }, - 'edx4edx': {'number' : 'edX.01', - 'title' : 'edx4edx: edX Author Course', - 'xmlpath': '/edx4edx/', - 'github_url': 'https://github.com/MITx/edx4edx', - 'active' : True, - 'default_chapter' : 'Introduction', - 'default_section' : 'edx4edx_Course', - 'location': 'i4x://edx/6002xs12/course/edx4edx', - }, - '7.03x_Fall_2012': {'number' : '7.03x', - 'title' : 'Genetics', - 'xmlpath': '/7.03x/', - 'github_url': 'https://github.com/MITx/7.03x', - 'active' : True, - 'default_chapter' : 'Week_2', - 'default_section' : 'ps1_question_1', - }, - '3.091x_Fall_2012': {'number' : '3.091x', - 'title' : 'Introduction to Solid State Chemistry', - 'xmlpath': '/3.091x/', - 'github_url': 'https://github.com/MITx/3.091x', - 'active' : True, - 'default_chapter' : 'Week_1', - 'default_section' : 'Problem_Set_1', - }, - '18.06x_Linear_Algebra': {'number' : '18.06x', - 'title' : 'Linear Algebra', - 'xmlpath': '/18.06x/', - 'github_url': 'https://github.com/MITx/18.06x', - 'default_chapter' : 'Unit_1', - 'default_section' : 'Midterm_1', - 'active' : True, - }, - '6.00x_Fall_2012': {'number' : '6.00x', - 'title' : 'Introduction to Computer Science and Programming', - 'xmlpath': '/6.00x/', - 'github_url': 'https://github.com/MITx/6.00x', - 'active' : True, - 'default_chapter' : 'Week_0', - 'default_section' : 'Problem_Set_0', - 'location': 'i4x://edx/6002xs12/course/6.00x_Fall_2012', - }, - '7.00x_Fall_2012': {'number' : '7.00x', - 'title' : 'Introduction to Biology', - 'xmlpath': '/7.00x/', - 'github_url': 'https://github.com/MITx/7.00x', - 'active' : True, - 'default_chapter' : 'Unit 1', - 'default_section' : 'Introduction', - }, - } +LOGGING = get_logger_config(ENV_ROOT / "log", + logging_env="dev", + tracking_filename="tracking.log", + debug=True) -#----------------------------------------------------------------------------- +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ENV_ROOT / "db" / "mitx.db", + } +} -MIDDLEWARE_CLASSES = MIDDLEWARE_CLASSES + ( - 'ssl_auth.ssl_auth.NginxProxyHeaderMiddleware', # ssl authentication behind nginx proxy - ) +CACHES = { + # This is the cache used for most things. Askbot will not work without a + # functioning cache -- it relies on caching to load its settings in places. + # In staging/prod envs, the sessions also live here. + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'LOCATION': 'mitx_loc_mem_cache', + 'KEY_FUNCTION': 'util.memcache.safe_key', + }, -AUTHENTICATION_BACKENDS = ( - 'ssl_auth.ssl_auth.SSLLoginBackend', - 'django.contrib.auth.backends.ModelBackend', - ) + # The general cache is what you get if you use our util.cache. It's used for + # things like caching the course.xml file for different A/B test groups. + # We set it to be a DummyCache to force reloading of course.xml in dev. + # In staging environments, we would grab VERSION from data uploaded by the + # push process. + 'general': { + 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + 'KEY_PREFIX': 'general', + 'VERSION': 4, + 'KEY_FUNCTION': 'util.memcache.safe_key', + } +} -INSTALLED_APPS = INSTALLED_APPS + ( - 'ssl_auth', - ) +# Dummy secret key for dev +SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' -LOGIN_REDIRECT_URL = MITX_ROOT_URL + '/' -LOGIN_URL = MITX_ROOT_URL + '/' +################################ OpenID Auth ################################# +MITX_FEATURES['AUTH_USE_OPENID'] = True + +INSTALLED_APPS += ('external_auth',) +INSTALLED_APPS += ('django_openid_auth',) +#INSTALLED_APPS += ('ssl_auth',) + +#MIDDLEWARE_CLASSES += ( +# #'ssl_auth.ssl_auth.NginxProxyHeaderMiddleware', # ssl authentication behind nginx proxy +# ) + +#AUTHENTICATION_BACKENDS = ( +# 'django_openid_auth.auth.OpenIDBackend', +# 'django.contrib.auth.backends.ModelBackend', +# ) + +OPENID_CREATE_USERS = False +OPENID_UPDATE_DETAILS_FROM_SREG = True +OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id' +OPENID_USE_AS_ADMIN_LOGIN = False +#import external_auth.views as edXauth +#OPENID_RENDER_FAILURE = edXauth.edXauth_openid + +################################ DEBUG TOOLBAR ################################# +INSTALLED_APPS += ('debug_toolbar',) +MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) +INTERNAL_IPS = ('127.0.0.1',) + +DEBUG_TOOLBAR_PANELS = ( + 'debug_toolbar.panels.version.VersionDebugPanel', + 'debug_toolbar.panels.timer.TimerDebugPanel', + 'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel', + 'debug_toolbar.panels.headers.HeaderDebugPanel', + 'debug_toolbar.panels.request_vars.RequestVarsDebugPanel', + 'debug_toolbar.panels.sql.SQLDebugPanel', + 'debug_toolbar.panels.signals.SignalDebugPanel', + 'debug_toolbar.panels.logger.LoggingPanel', + +# Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and +# Django=1.3.1/1.4 where requests to views get duplicated (your method gets +# hit twice). So you can uncomment when you need to diagnose performance +# problems, but you shouldn't leave it on. +# 'debug_toolbar.panels.profiling.ProfilingDebugPanel', +) + +############################ FILE UPLOADS (ASKBOT) ############################# +DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' +MEDIA_ROOT = ENV_ROOT / "uploads" +MEDIA_URL = "/static/uploads/" +STATICFILES_DIRS.append(("uploads", MEDIA_ROOT)) +FILE_UPLOAD_TEMP_DIR = ENV_ROOT / "uploads" +FILE_UPLOAD_HANDLERS = ( + 'django.core.files.uploadhandler.MemoryFileUploadHandler', + 'django.core.files.uploadhandler.TemporaryFileUploadHandler', +) + +########################### PIPELINE ################################# + +PIPELINE_SASS_ARGUMENTS = '-r {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT) diff --git a/lms/templates/extauth_failure.html b/lms/templates/extauth_failure.html new file mode 100644 index 0000000000..fa53ab1084 --- /dev/null +++ b/lms/templates/extauth_failure.html @@ -0,0 +1,11 @@ + + + + OpenID failed + + +

    OpenID failed

    +

    ${message}

    + + diff --git a/lms/templates/index.html b/lms/templates/index.html index d8b0394927..dcc18d4de1 100644 --- a/lms/templates/index.html +++ b/lms/templates/index.html @@ -144,3 +144,12 @@
    + +% if show_signup_immediately is not UNDEFINED: + +% endif diff --git a/lms/templates/login_modal.html b/lms/templates/login_modal.html index 393e76ee78..e2a2754226 100644 --- a/lms/templates/login_modal.html +++ b/lms/templates/login_modal.html @@ -27,6 +27,9 @@ Not enrolled? Forgot password?

    +

    + login via openid +

    diff --git a/lms/templates/signup_modal.html b/lms/templates/signup_modal.html index aef90ab0f2..346027418d 100644 --- a/lms/templates/signup_modal.html +++ b/lms/templates/signup_modal.html @@ -19,6 +19,7 @@
    + % if has_extauth_info is UNDEFINED: @@ -27,6 +28,18 @@ + % else: +

    Welcome ${extauth_email}


    + + +

    Enter a public username:

    + + + + + + + % endif
    @@ -93,11 +106,13 @@
    + % if has_extauth_info is UNDEFINED: + % endif
    diff --git a/lms/urls.py b/lms/urls.py index 1c4a065e2b..8c36857ee3 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -160,12 +160,22 @@ if settings.DEBUG: ## Jasmine urlpatterns=urlpatterns + (url(r'^_jasmine/', include('django_jasmine.urls')),) +if settings.MITX_FEATURES.get('AUTH_USE_OPENID'): + urlpatterns += ( + url(r'^openid/login/$', 'django_openid_auth.views.login_begin', name='openid-login'), + url(r'^openid/complete/$', 'external_auth.views.edXauth_openid_login_complete', name='openid-complete'), + url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'), + ) + urlpatterns += ( + url(r'^extauth/$', 'external_auth.views.edXauth_signup', name='extauth-signup'), + ) + # urlpatterns += (url(r'^openid/', include('django_openid_auth.urls')),) + urlpatterns = patterns(*urlpatterns) if settings.DEBUG: urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) - #Custom error pages handler404 = 'static_template_view.views.render_404' handler500 = 'static_template_view.views.render_500' diff --git a/requirements.txt b/requirements.txt index 46c822642e..33b2bfeb05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ lxml boto mako python-memcached +python-openid path.py django_debug_toolbar -e git://github.com/MITx/django-pipeline.git#egg=django-pipeline @@ -37,6 +38,7 @@ django-jasmine django-keyedcache django-mako django-masquerade +django-openid-auth django-robots django-ses django-storages From 4874fa51efe4e09c16221c3319653c96778887d6 Mon Sep 17 00:00:00 2001 From: ichuang Date: Wed, 1 Aug 2012 15:00:27 -0400 Subject: [PATCH 055/124] remove cruft from student.views --- common/djangoapps/student/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index 692b135ff6..3ba83f42bb 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -23,7 +23,6 @@ from django.http import HttpResponse, Http404 from django.shortcuts import redirect from mitxmako.shortcuts import render_to_response, render_to_string from django.core.urlresolvers import reverse -#from BeautifulSoup import BeautifulSoup from bs4 import BeautifulSoup from django.core.cache import cache From 442fb6e839ab77c9eda524ace487b671fc7b8dcf Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 1 Aug 2012 16:19:29 -0400 Subject: [PATCH 056/124] Pass the cursor, rather than the output of cursor.execute, to dictfetchall --- lms/djangoapps/dashboard/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/djangoapps/dashboard/views.py b/lms/djangoapps/dashboard/views.py index 5cf3fa72c1..c4446bceaa 100644 --- a/lms/djangoapps/dashboard/views.py +++ b/lms/djangoapps/dashboard/views.py @@ -25,7 +25,7 @@ def dashboard(request): from django.db import connection cursor = connection.cursor() - results = dictfetchall(cursor.execute(query)) - + cursor.execute(query) + results = dictfetchall(cursor) return HttpResponse(json.dumps(results, indent=4)) From ea26c25cb4dc7d0651b4cf2f06209b49c0ae2642 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Wed, 1 Aug 2012 16:33:28 -0400 Subject: [PATCH 057/124] add back-compat code to customtag * apparently 6002x is using the impl-as-child structure already. --- common/lib/xmodule/xmodule/template_module.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/template_module.py b/common/lib/xmodule/xmodule/template_module.py index b4818dc29d..260ad009f9 100644 --- a/common/lib/xmodule/xmodule/template_module.py +++ b/common/lib/xmodule/xmodule/template_module.py @@ -32,7 +32,18 @@ class CustomTagModule(XModule): instance_state, shared_state, **kwargs) xmltree = etree.fromstring(self.definition['data']) - template_name = xmltree.attrib['impl'] + if 'impl' in xmltree.attrib: + template_name = xmltree.attrib['impl'] + else: + # VS[compat] backwards compatibility with old nested customtag structure + child_impl = xmltree.find('impl') + if child_impl is not None: + template_name = child_impl.text + else: + # TODO (vshnayder): better exception type + raise Exception("Could not find impl attribute in customtag {0}" + .format(location)) + params = dict(xmltree.items()) with self.system.filestore.open( 'custom_tags/{name}'.format(name=template_name)) as template: From a68cb545be5f272c5d2afde7f67d07a364e0b934 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 1 Aug 2012 16:57:13 -0400 Subject: [PATCH 058/124] updating prod-requirements --- prod-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prod-requirements.txt b/prod-requirements.txt index ff4ca151f9..98545552cb 100644 --- a/prod-requirements.txt +++ b/prod-requirements.txt @@ -15,7 +15,8 @@ django-jasmine==0.3.2 path.py==2.2.2 requests==0.12.1 BeautifulSoup==3.2.1 -newrelic==1.2.1.265 +BeautifulSoup4==4.1.1 +newrelic==1.3.0.289 ipython==0.12.1 django-pipeline==1.2.12 django-staticfiles==1.2.1 From cdd37eeb92864ee6debc5aab2e35116740445e4e Mon Sep 17 00:00:00 2001 From: Kyle Fiedler Date: Wed, 1 Aug 2012 14:29:53 -0400 Subject: [PATCH 059/124] Strip down sequence navigaiton and more stripping of the courseware nav --- .../xmodule/xmodule/css/sequence/display.scss | 133 +++++++----------- lms/static/sass/base/_variables.scss | 2 +- lms/static/sass/course/base/_base.scss | 6 + lms/static/sass/course/base/_extends.scss | 83 ++--------- .../sass/course/courseware/_courseware.scss | 8 -- .../sass/course/courseware/_sidebar.scss | 7 +- .../course/discussion/_question-view.scss | 1 - lms/templates/staticbook.html | 1 - 8 files changed, 68 insertions(+), 173 deletions(-) diff --git a/common/lib/xmodule/xmodule/css/sequence/display.scss b/common/lib/xmodule/xmodule/css/sequence/display.scss index 7658797725..6e7c5d24a7 100644 --- a/common/lib/xmodule/xmodule/css/sequence/display.scss +++ b/common/lib/xmodule/xmodule/css/sequence/display.scss @@ -2,10 +2,8 @@ nav.sequence-nav { // TODO (cpennington): This doesn't work anymore. XModules aren't able to // import from external sources. @extend .topbar; - - border-bottom: 1px solid #ddd; + border-bottom: 1px solid $border-color; margin: (-(lh())) (-(lh())) lh() (-(lh())); - background: #eee; position: relative; @include border-top-right-radius(4px); @@ -14,7 +12,7 @@ nav.sequence-nav { display: table; height: 100%; margin: 0; - padding-left: 0; + padding-left: 3px; padding-right: flex-grid(1, 9); width: 100%; @@ -23,133 +21,104 @@ nav.sequence-nav { } li { - border-left: 1px solid #eee; display: table-cell; min-width: 20px; - &:first-child { - border-left: none; - } - - .inactive { - background-repeat: no-repeat; - - &:hover { - background-color: #eee; - } - } - - .visited { - background-color: #ddd; - background-repeat: no-repeat; - - &:hover { - background-position: center center; - } - } - - .active { - background-color: #fff; - background-repeat: no-repeat; - @include box-shadow(0 1px 0 #fff); - - &:hover { - background-color: #fff; - background-position: center; - } - } - a { - background-position: center center; - border: none; + background-position: center; + background-repeat: no-repeat; + border: 1px solid transparent; + border-bottom: none; + @include border-radius(3px 3px 0 0); cursor: pointer; display: block; - height: 17px; + height: 10px; padding: 15px 0 14px; position: relative; - @include transition(all, .4s, $ease-in-out-quad); + @include transition(); width: 100%; - &.progress { - border-bottom-style: solid; - border-bottom-width: 4px; + &:hover { + background-repeat: no-repeat; + background-position: center; + background-color: #F6F6F6; + } + + &.visited { + background-color: #F6F6F6; + + &:hover { + background-position: center center; + } + } + + &.active { + border-color: $border-color; + @include box-shadow(0 2px 0 #fff); + background-color: #fff; + z-index: 9; + + &:hover { + background-position: center; + background-color: #fff; + } } &.progress-none { - @extend .progress; - border-bottom-color: red; + background-color: lighten(red, 50%); } &.progress-some { - @extend .progress; - border-bottom-color: yellow; + background-color: yellow; } &.progress-done { - @extend .progress; - border-bottom-color: green; + background-color: green; } //video &.seq_video { &.inactive { - @extend .inactive; background-image: url('../images/sequence-nav/video-icon-normal.png'); - background-position: center; } &.visited { - @extend .visited; background-image: url('../images/sequence-nav/video-icon-visited.png'); - background-position: center; } &.active { @extend .active; background-image: url('../images/sequence-nav/video-icon-current.png'); - background-position: center; } } //other &.seq_other { &.inactive { - @extend .inactive; background-image: url('../images/sequence-nav/document-icon-normal.png'); - background-position: center; } &.visited { - @extend .visited; background-image: url('../images/sequence-nav/document-icon-visited.png'); - background-position: center; } &.active { - @extend .active; background-image: url('../images/sequence-nav/document-icon-current.png'); - background-position: center; } } //vertical & problems &.seq_vertical, &.seq_problem { &.inactive { - @extend .inactive; background-image: url('../images/sequence-nav/list-icon-normal.png'); - background-position: center; } &.visited { - @extend .visited; background-image: url('../images/sequence-nav/list-icon-visited.png'); - background-position: center; } &.active { - @extend .active; background-image: url('../images/sequence-nav/list-icon-current.png'); - background-position: center; } } @@ -157,6 +126,7 @@ nav.sequence-nav { background: #333; color: #fff; display: none; + font-family: $sans-serif; line-height: lh(); left: 0px; opacity: 0; @@ -207,27 +177,29 @@ nav.sequence-nav { right: 0; top: 0; width: flex-grid(1, 9); + border: 1px solid $border-color; + border-bottom: 0; + @include border-radius(3px 3px 0 0); li { float: left; + margin-bottom: 0; width: 50%; &.prev, &.next { a { - // background-color: darken($cream, 5%); - background-position: center center; + background-position: center; background-repeat: no-repeat; - border-left: 1px solid darken(#f6efd4, 20%); - @include box-shadow(inset 1px 0 0 lighten(#f6efd4, 5%)); - @include box-sizing(border-box); - cursor: pointer; display: block; + height: 10px; + padding: 15px 0 14px; text-indent: -9999px; @include transition(all, .2s, $ease-in-out-quad); &:hover { opacity: .5; + background-color: #f4f4f4; } &.disabled { @@ -240,20 +212,13 @@ nav.sequence-nav { &.prev { a { background-image: url('../images/sequence-nav/previous-icon.png'); - - &:hover { - // background-color: $cream; - } } } &.next { a { + border-left: 1px solid lighten($border-color, 10%); background-image: url('../images/sequence-nav/next-icon.png'); - - &:hover { - // background-color: $cream; - } } } } @@ -274,10 +239,8 @@ nav.sequence-bottom { ul { @extend .clearfix; - background-color: #eee; - border: 1px solid #ddd; + border: 1px solid $border-color; @include border-radius(3px); - @include box-shadow(inset 0 0 0 1px lighten(#f6efd4, 5%)); @include inline-block(); li { @@ -312,7 +275,7 @@ nav.sequence-bottom { &.prev { a { background-image: url('../images/sequence-nav/previous-icon.png'); - border-right: 1px solid darken(#f6efd4, 20%); + border-right: 1px solid lighten($border-color, 10%); &:hover { background-color: none; diff --git a/lms/static/sass/base/_variables.scss b/lms/static/sass/base/_variables.scss index 6c8d0d4000..4a8993b200 100644 --- a/lms/static/sass/base/_variables.scss +++ b/lms/static/sass/base/_variables.scss @@ -15,6 +15,7 @@ $blue: rgb(29,157,217); $pink: rgb(182,37,104); $yellow: rgb(255, 252, 221); $error-red: rgb(253, 87, 87); +$border-color: #C8C8C8; // old variables $body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif; @@ -29,4 +30,3 @@ $dark-gray: #333; $mit-red: #993333; $cream: #F6EFD4; $text-color: $dark-gray; -$border-color: $light-gray; diff --git a/lms/static/sass/course/base/_base.scss b/lms/static/sass/course/base/_base.scss index 8849c78cfc..d04bcab103 100644 --- a/lms/static/sass/course/base/_base.scss +++ b/lms/static/sass/course/base/_base.scss @@ -12,4 +12,10 @@ table { .container { padding: lh(2); + + > div { + display: table; + width: 100%; + table-layout: fixed; + } } diff --git a/lms/static/sass/course/base/_extends.scss b/lms/static/sass/course/base/_extends.scss index e1fc953126..5710aa9639 100644 --- a/lms/static/sass/course/base/_extends.scss +++ b/lms/static/sass/course/base/_extends.scss @@ -1,43 +1,9 @@ -.wrapper { - margin: 0 auto; - max-width: $fg-max-width; - min-width: $fg-min-width; - text-align: left; - width: flex-grid(12); - - div.table-wrapper { - display: table; - width: flex-grid(12); - overflow: hidden; - } -} - h1.top-header { - background: #f3f3f3; border-bottom: 1px solid #e3e3e3; - margin: (-(lh())) (-(lh())) lh(); - padding: lh(); text-align: left; -} - -.button { - border: 1px solid darken(#888, 10%); - @include border-radius(3px); - @include box-shadow(inset 0 1px 0 lighten(#888, 10%), 0 0 3px #ccc); - color: #fff; - cursor: pointer; - font: bold $body-font-size $body-font-family; - @include linear-gradient(lighten(#888, 5%), darken(#888, 5%)); - padding: 4px 8px; - text-decoration: none; - text-shadow: none; - -webkit-font-smoothing: antialiased; - - &:hover, &:focus { - border: 1px solid darken(#888, 20%); - @include box-shadow(inset 0 1px 0 lighten(#888, 20%), 0 0 3px #ccc); - @include linear-gradient(lighten(#888, 10%), darken(#888, 5%)); - } + font-size: 24px; + font-weight: 100; + padding-bottom: lh(); } .light-button, a.light-button { @@ -147,27 +113,20 @@ h1.top-header { } header#open_close_accordion { - border-bottom: 1px solid #d3d3d3; - @include box-shadow(0 1px 0 #eee); - padding: lh(.5) lh(); position: relative; - h2 { - margin: 0; - padding-right: 20px; - } - a { - background: #eee url('../images/slide-left-icon.png') center center no-repeat; + background: #f6f6f6 url('../images/slide-left-icon.png') center center no-repeat; border: 1px solid #D3D3D3; @include border-radius(3px 0 0 3px); height: 16px; - padding: 8px; + padding: 6px; position: absolute; right: -1px; text-indent: -9999px; top: 6px; width: 16px; + z-index: 99; &:hover { background-color: white; @@ -182,32 +141,17 @@ h1.top-header { .topbar { @extend .clearfix; - border-bottom: 1px solid darken($cream, 10%); - border-top: 1px solid #fff; - font-size: 12px; - line-height: 46px; - text-shadow: 0 1px 0 #fff; + border-bottom: 1px solid $border-color; + font-size: 14px; @media print { display: none; } a { - line-height: 46px; - border-bottom: 0; - color: darken($cream, 80%); - - &:hover { - color: darken($cream, 60%); - text-decoration: none; - } - &.block-link { - // background: darken($cream, 5%); - border-left: 1px solid darken($cream, 20%); - @include box-shadow(inset 1px 0 0 lighten($cream, 5%)); + border-left: 1px solid lighten($border-color, 10%); display: block; - text-transform: uppercase; &:hover { background: none; @@ -219,12 +163,3 @@ h1.top-header { .tran { @include transition( all, .2s, $ease-in-out-quad); } - -p.ie-warning { - background: yellow; - display: block !important; - line-height: 1.3em; - margin-bottom: 0; - padding: lh(); - text-align: left; -} diff --git a/lms/static/sass/course/courseware/_courseware.scss b/lms/static/sass/course/courseware/_courseware.scss index 77f53e0241..f6c9dceb8e 100644 --- a/lms/static/sass/course/courseware/_courseware.scss +++ b/lms/static/sass/course/courseware/_courseware.scss @@ -181,17 +181,9 @@ div.course-wrapper { overflow: hidden; header#open_close_accordion { - padding: 0; - min-height: 47px; - a { background-image: url('../images/slide-right-icon.png'); } - - h2 { - visibility: hidden; - width: 10px; - } } div#accordion { diff --git a/lms/static/sass/course/courseware/_sidebar.scss b/lms/static/sass/course/courseware/_sidebar.scss index 860d588a84..fe9f54d0e3 100644 --- a/lms/static/sass/course/courseware/_sidebar.scss +++ b/lms/static/sass/course/courseware/_sidebar.scss @@ -23,7 +23,8 @@ section.course-index { } &:hover { - background: #eee; + background: #f6f6f6; + text-decoration: none; } &.ui-accordion-header { @@ -49,7 +50,7 @@ section.course-index { ul.ui-accordion-content { @include border-radius(0); - background: #FFF; + background: transparent; border: none; font-size: 12px; margin: 0; @@ -104,7 +105,7 @@ section.course-index { &:after { opacity: 1; right: 15px; - @include transition(all, 0.2s, linear); + @include transition(); } > a p { diff --git a/lms/static/sass/course/discussion/_question-view.scss b/lms/static/sass/course/discussion/_question-view.scss index 4b7765b2f9..0920f64f9b 100644 --- a/lms/static/sass/course/discussion/_question-view.scss +++ b/lms/static/sass/course/discussion/_question-view.scss @@ -313,7 +313,6 @@ div.question-header { } a.edit { - @extend .button; font-size: 12px; padding: 2px 10px; } diff --git a/lms/templates/staticbook.html b/lms/templates/staticbook.html index f5b184cc1c..eae70cdd84 100644 --- a/lms/templates/staticbook.html +++ b/lms/templates/staticbook.html @@ -67,7 +67,6 @@ $("#open_close_accordion a").click(function(){
    -

    Table of Contents

    close
    From 6c3ec99f9e325fcdfd5b71769f0b69e1058a1dd4 Mon Sep 17 00:00:00 2001 From: Kyle Fiedler Date: Wed, 1 Aug 2012 16:55:30 -0400 Subject: [PATCH 060/124] Start to change to ems and remove all of cream --- cms/static/sass/_base.scss | 2 ++ lms/static/sass/application.scss | 2 +- lms/static/sass/base/_mixins.scss | 4 ++++ lms/static/sass/base/_variables.scss | 10 ++++----- lms/static/sass/course.scss | 2 +- lms/static/sass/course/_info.scss | 22 +++++++++++-------- lms/static/sass/course/_textbook.scss | 1 - .../sass/course/discussion/_profile.scss | 1 - .../course/discussion/_question-view.scss | 4 +--- lms/static/sass/course/old/.gitignore | 1 - lms/static/sass/course/wiki/_wiki.scss | 3 --- 11 files changed, 26 insertions(+), 26 deletions(-) delete mode 100644 lms/static/sass/course/old/.gitignore diff --git a/cms/static/sass/_base.scss b/cms/static/sass/_base.scss index cad315f6e4..2ea98473d1 100644 --- a/cms/static/sass/_base.scss +++ b/cms/static/sass/_base.scss @@ -2,6 +2,7 @@ $fg-column: 70px; $fg-gutter: 26px; $fg-max-columns: 12; $body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif; +$sans-serif: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif; $body-font-size: 14px; $body-line-height: 20px; @@ -12,6 +13,7 @@ $orange: #f96e5b; $yellow: #fff8af; $cream: #F6EFD4; $mit-red: #933; +$border-color: #ddd; @mixin hide-text { background-color: transparent; diff --git a/lms/static/sass/application.scss b/lms/static/sass/application.scss index c16d72e367..240d68c5b3 100644 --- a/lms/static/sass/application.scss +++ b/lms/static/sass/application.scss @@ -2,9 +2,9 @@ @import 'base/reset'; @import 'base/font_face'; +@import 'base/mixins'; @import 'base/variables'; @import 'base/base'; -@import 'base/mixins'; @import 'base/extends'; @import 'base/animations'; diff --git a/lms/static/sass/base/_mixins.scss b/lms/static/sass/base/_mixins.scss index 7c53b6e14f..58a92d1ee6 100644 --- a/lms/static/sass/base/_mixins.scss +++ b/lms/static/sass/base/_mixins.scss @@ -1,3 +1,7 @@ +@function em($pxval, $base: 16) { + @return #{$pxval / $base}em; +} + // Line-height @function lh($amount: 1) { @return $body-line-height * $amount; diff --git a/lms/static/sass/base/_variables.scss b/lms/static/sass/base/_variables.scss index 4a8993b200..2da5855f0a 100644 --- a/lms/static/sass/base/_variables.scss +++ b/lms/static/sass/base/_variables.scss @@ -4,6 +4,8 @@ $gw-gutter: 20px; $fg-column: $gw-column; $fg-gutter: $gw-gutter; $fg-max-columns: 12; +$fg-max-width: 1400px; +$fg-min-width: 810px; $sans-serif: 'Open Sans', $verdana; $serif: $georgia; @@ -19,14 +21,10 @@ $border-color: #C8C8C8; // old variables $body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif; -$body-font-size: 14px; -$body-line-height: golden-ratio($body-font-size, 1); - -$fg-max-width: 1400px; -$fg-min-width: 810px; +$body-font-size: em(14); +$body-line-height: golden-ratio(.875em, 1); $light-gray: #ddd; $dark-gray: #333; $mit-red: #993333; -$cream: #F6EFD4; $text-color: $dark-gray; diff --git a/lms/static/sass/course.scss b/lms/static/sass/course.scss index 8fafbb8479..cc1b49a0a2 100644 --- a/lms/static/sass/course.scss +++ b/lms/static/sass/course.scss @@ -2,9 +2,9 @@ @import 'base/reset'; @import 'base/font_face'; +@import 'base/mixins'; @import 'base/variables'; @import 'base/base'; -@import 'base/mixins'; @import 'base/extends'; @import 'base/animations'; diff --git a/lms/static/sass/course/_info.scss b/lms/static/sass/course/_info.scss index 45dd2d57b3..5ba50dd8f5 100644 --- a/lms/static/sass/course/_info.scss +++ b/lms/static/sass/course/_info.scss @@ -3,6 +3,7 @@ div.info-wrapper { section.updates { @extend .content; + line-height: lh(); > h1 { @extend .top-header; @@ -19,26 +20,29 @@ div.info-wrapper { > li { @extend .clearfix; border-bottom: 1px solid #e3e3e3; - margin-bottom: lh(.5); padding-bottom: lh(.5); list-style-type: disk; &:first-child { - background: $cream; - border-bottom: 1px solid darken($cream, 10%); margin: 0 (-(lh(.5))) lh(); padding: lh(.5); } ol, ul { - margin: lh() 0 0 lh(); - list-style-type: circle; + margin: 0; + list-style-type: disk; + + ol,ul { + list-style-type: circle; + } } h2 { float: left; margin: 0 flex-gutter() 0 0; width: flex-grid(2, 9); + font-size: body-font-size; + font-weight: bold; } section.update-description { @@ -94,7 +98,7 @@ div.info-wrapper { border-bottom: 1px solid #d3d3d3; @include box-shadow(0 1px 0 #eee); @include box-sizing(border-box); - padding: 7px lh(.75); + padding: em(7) lh(.75); position: relative; &.expandable, @@ -108,13 +112,13 @@ div.info-wrapper { ul { background: none; - margin: 7px (-(lh(.75))) 0; + margin: em(7) (-(lh(.75))) 0; li { border-bottom: 0; border-top: 1px solid #d3d3d3; @include box-shadow(inset 0 1px 0 #eee); - padding-left: 18px + lh(.75); + padding-left: lh(1.5); } } @@ -150,7 +154,7 @@ div.info-wrapper { border-bottom: 0; @include box-shadow(none); color: #999; - font-size: 12px; + font-size: $body-font-size; font-weight: bold; text-transform: uppercase; } diff --git a/lms/static/sass/course/_textbook.scss b/lms/static/sass/course/_textbook.scss index ae549d723f..ed5e528809 100644 --- a/lms/static/sass/course/_textbook.scss +++ b/lms/static/sass/course/_textbook.scss @@ -62,7 +62,6 @@ div.book-wrapper { @extend .clearfix; li { - background-color: darken($cream, 4%); &.last { display: block; diff --git a/lms/static/sass/course/discussion/_profile.scss b/lms/static/sass/course/discussion/_profile.scss index 42e6b772f8..010a03ffd6 100644 --- a/lms/static/sass/course/discussion/_profile.scss +++ b/lms/static/sass/course/discussion/_profile.scss @@ -72,7 +72,6 @@ body.user-profile-page { margin-bottom: 30px; li { - background-color: lighten($cream, 3%); background-position: 10px center; background-repeat: no-repeat; @include border-radius(4px); diff --git a/lms/static/sass/course/discussion/_question-view.scss b/lms/static/sass/course/discussion/_question-view.scss index 0920f64f9b..4c2acaf9be 100644 --- a/lms/static/sass/course/discussion/_question-view.scss +++ b/lms/static/sass/course/discussion/_question-view.scss @@ -32,8 +32,6 @@ div.question-header { &.post-vote { @include border-radius(4px); - background-color: lighten($cream, 5%); - border: 1px solid darken( $cream, 10% ); @include box-shadow(inset 0 1px 0px #fff); } @@ -149,7 +147,7 @@ div.question-header { &.revision { text-align: center; - background:lighten($cream, 7%); + // background:lighten($cream, 7%); a { color: black; diff --git a/lms/static/sass/course/old/.gitignore b/lms/static/sass/course/old/.gitignore deleted file mode 100644 index b3a5267117..0000000000 --- a/lms/static/sass/course/old/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.css diff --git a/lms/static/sass/course/wiki/_wiki.scss b/lms/static/sass/course/wiki/_wiki.scss index 9c878ad263..ec53044ed1 100644 --- a/lms/static/sass/course/wiki/_wiki.scss +++ b/lms/static/sass/course/wiki/_wiki.scss @@ -24,7 +24,6 @@ div.wiki-wrapper { } p { - color: darken($cream, 55%); float: left; line-height: 46px; margin-bottom: 0; @@ -40,14 +39,12 @@ div.wiki-wrapper { input[type="button"] { @extend .block-link; - background-color: darken($cream, 5%); background-position: 12px center; background-repeat: no-repeat; border: 0; border-left: 1px solid darken(#f6efd4, 20%); @include border-radius(0); @include box-shadow(inset 1px 0 0 lighten(#f6efd4, 5%)); - color: darken($cream, 80%); display: block; font-size: 12px; font-weight: normal; From b90aa2e2dbbeef3a22d02566dd1242a91fa5e0c5 Mon Sep 17 00:00:00 2001 From: Kyle Fiedler Date: Wed, 1 Aug 2012 17:35:58 -0400 Subject: [PATCH 061/124] Added more styles for the info page and added headers for the updates and handouts so they will be consistant for every class --- lms/static/sass/base/_variables.scss | 6 ++-- lms/static/sass/course/_info.scss | 20 ++++++++---- lms/static/sass/course/base/_extends.scss | 6 ++-- lms/templates/info.html | 40 ++++++++++++----------- 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/lms/static/sass/base/_variables.scss b/lms/static/sass/base/_variables.scss index 2da5855f0a..7ad30f0c91 100644 --- a/lms/static/sass/base/_variables.scss +++ b/lms/static/sass/base/_variables.scss @@ -8,8 +8,11 @@ $fg-max-width: 1400px; $fg-min-width: 810px; $sans-serif: 'Open Sans', $verdana; +$body-font-family: $sans-serif; $serif: $georgia; +$body-font-size: em(14); +$body-line-height: golden-ratio(.875em, 1); $base-font-color: rgb(60,60,60); $lighter-base-font-color: rgb(160,160,160); @@ -20,9 +23,6 @@ $error-red: rgb(253, 87, 87); $border-color: #C8C8C8; // old variables -$body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif; -$body-font-size: em(14); -$body-line-height: golden-ratio(.875em, 1); $light-gray: #ddd; $dark-gray: #333; diff --git a/lms/static/sass/course/_info.scss b/lms/static/sass/course/_info.scss index 5ba50dd8f5..d9af9c0f82 100644 --- a/lms/static/sass/course/_info.scss +++ b/lms/static/sass/course/_info.scss @@ -16,10 +16,12 @@ div.info-wrapper { > ol { list-style: none; padding-left: 0; + margin-bottom: lh(); > li { @extend .clearfix; border-bottom: 1px solid #e3e3e3; + margin-bottom: lh(); padding-bottom: lh(.5); list-style-type: disk; @@ -41,7 +43,7 @@ div.info-wrapper { float: left; margin: 0 flex-gutter() 0 0; width: flex-grid(2, 9); - font-size: body-font-size; + font-size: $body-font-size; font-weight: bold; } @@ -68,16 +70,20 @@ div.info-wrapper { @extend .sidebar; border-left: 1px solid #d3d3d3; @include border-radius(0 4px 4px 0); + @include box-shadow(none); border-right: 0; - header { + h1 { @extend .bottom-border; - padding: lh(.5) lh(.75); + padding: lh(.5) lh(.5); + } - h1 { - font-size: 18px; - margin: 0 ; - } + header { + + // h1 { + // font-weight: 100; + // font-style: italic; + // } p { color: #666; diff --git a/lms/static/sass/course/base/_extends.scss b/lms/static/sass/course/base/_extends.scss index 5710aa9639..5927cb569e 100644 --- a/lms/static/sass/course/base/_extends.scss +++ b/lms/static/sass/course/base/_extends.scss @@ -60,11 +60,13 @@ h1.top-header { width: flex-grid(3); h1, h2 { - font-size: 18px; - font-weight: bold; + font-size: em(18); + font-weight: 100; letter-spacing: 0; text-transform: none; font-family: $sans-serif; + text-align: left; + font-style: italic; } a { diff --git a/lms/templates/info.html b/lms/templates/info.html index 25ad4f9184..a04e31896f 100644 --- a/lms/templates/info.html +++ b/lms/templates/info.html @@ -20,23 +20,25 @@ $(document).ready(function(){
    -
    -
    - % if user.is_authenticated(): -
    - ${get_course_info_section(course, 'updates')} -
    -
    - ${get_course_info_section(course, 'handouts')} -
    - % else: -
    - ${get_course_info_section(course, 'guest_updates')} -
    -
    - ${get_course_info_section(course, 'guest_handouts')} -
    - % endif -
    -
    +
    + % if user.is_authenticated(): +
    +

    Course Updates & News

    + ${get_course_info_section(course, 'updates')} +
    +
    +

    Course Handouts

    + ${get_course_info_section(course, 'handouts')} +
    + % else: +
    +

    Course Updates & News

    + ${get_course_info_section(course, 'guest_updates')} +
    +
    +

    Course Handouts

    + ${get_course_info_section(course, 'guest_handouts')} +
    + % endif +
    From 7b725a075ca9d73a13f1610aa48f6d7d1fd990e9 Mon Sep 17 00:00:00 2001 From: ichuang Date: Wed, 1 Aug 2012 17:57:21 -0400 Subject: [PATCH 062/124] external_auth djangoapps files --- common/djangoapps/external_auth/__init__.py | 0 common/djangoapps/external_auth/admin.py | 8 ++ common/djangoapps/external_auth/models.py | 29 +++++ common/djangoapps/external_auth/views.py | 137 ++++++++++++++++++++ 4 files changed, 174 insertions(+) create mode 100644 common/djangoapps/external_auth/__init__.py create mode 100644 common/djangoapps/external_auth/admin.py create mode 100644 common/djangoapps/external_auth/models.py create mode 100644 common/djangoapps/external_auth/views.py diff --git a/common/djangoapps/external_auth/__init__.py b/common/djangoapps/external_auth/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/djangoapps/external_auth/admin.py b/common/djangoapps/external_auth/admin.py new file mode 100644 index 0000000000..343492bca7 --- /dev/null +++ b/common/djangoapps/external_auth/admin.py @@ -0,0 +1,8 @@ +''' +django admin pages for courseware model +''' + +from external_auth.models import * +from django.contrib import admin + +admin.site.register(ExternalAuthMap) diff --git a/common/djangoapps/external_auth/models.py b/common/djangoapps/external_auth/models.py new file mode 100644 index 0000000000..b8b167d25d --- /dev/null +++ b/common/djangoapps/external_auth/models.py @@ -0,0 +1,29 @@ +""" +WE'RE USING MIGRATIONS! + +If you make changes to this model, be sure to create an appropriate migration +file and check it in at the same time as your model changes. To do that, + +1. Go to the mitx dir +2. django-admin.py schemamigration student --auto --settings=lms.envs.dev --pythonpath=. description_of_your_change +3. Add the migration file created in mitx/common/djangoapps/external_auth/migrations/ +""" + +from django.db import models +from django.contrib.auth.models import User + +class ExternalAuthMap(models.Model): + external_id = models.CharField(max_length=255, db_index=True, unique=True) + external_domain = models.CharField(max_length=255, db_index=True) + external_credentials = models.TextField(blank=True) # JSON dictionary + external_email = models.CharField(max_length=255, db_index=True) + external_name = models.CharField(blank=True,max_length=255, db_index=True) + user = models.OneToOneField(User, unique=True, db_index=True, null=True) + internal_password = models.CharField(blank=True, max_length=31) # randomly generated + dtcreated = models.DateTimeField('creation date',auto_now_add=True) + dtsignup = models.DateTimeField('signup date',null=True) # set after signup + + def __unicode__(self): + s = "[%s] = (%s / %s)" % (self.external_id, self.external_name, self.external_email) + return s + diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py new file mode 100644 index 0000000000..57131f21da --- /dev/null +++ b/common/djangoapps/external_auth/views.py @@ -0,0 +1,137 @@ +# from pprint import pprint + +import json +import logging +import random +import string + +from external_auth.models import ExternalAuthMap + +from django.conf import settings +from django.contrib.auth import REDIRECT_FIELD_NAME, authenticate, login +from django.contrib.auth.models import Group +from django.contrib.auth.models import User + +from django.core.urlresolvers import reverse +from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import render_to_response +from django.shortcuts import redirect +from django.template import RequestContext +from mitxmako.shortcuts import render_to_response, render_to_string +try: + from django.views.decorators.csrf import csrf_exempt +except ImportError: + from django.contrib.csrf.middleware import csrf_exempt +from django_future.csrf import ensure_csrf_cookie +from util.cache import cache_if_anonymous + +#from django_openid_auth import views as openid_views +from django_openid_auth import auth as openid_auth +from openid.consumer.consumer import (Consumer, SUCCESS, CANCEL, FAILURE) +import django_openid_auth + +import student.views as student_views + +log = logging.getLogger("mitx.external_auth") + +@csrf_exempt +def default_render_failure(request, message, status=403, template_name='extauth_failure.html', exception=None): + """Render an Openid error page to the user.""" + + message = "In openid_failure " + message + print "in openid_failure: " + data = render_to_string( template_name, dict(message=message, exception=exception)) + return HttpResponse(data, status=status) + +@csrf_exempt +def edXauth_openid_login_complete(request, redirect_field_name=REDIRECT_FIELD_NAME, render_failure=None): + """Complete the openid login process""" + + redirect_to = request.REQUEST.get(redirect_field_name, '') + render_failure = render_failure or \ + getattr(settings, 'OPENID_RENDER_FAILURE', None) or \ + default_render_failure + + openid_response = django_openid_auth.views.parse_openid_response(request) + if not openid_response: + return render_failure(request, 'This is an OpenID relying party endpoint.') + + if openid_response.status == SUCCESS: + external_id = openid_response.identity_url + oid_backend = openid_auth.OpenIDBackend() + details = oid_backened = oid_backend._extract_user_details(openid_response) + + log.debug('openid success, details=%s' % details) + + # see if we have a map from this external_id to an edX username + try: + eamap = ExternalAuthMap.objects.get(external_id=external_id) + log.debug('Found eamap=%s' % eamap) + except ExternalAuthMap.DoesNotExist: + # go render form for creating edX user + eamap = ExternalAuthMap(external_id = external_id, + external_domain = "openid:%s" % settings.OPENID_SSO_SERVER_URL, + external_credentials = json.dumps(details), + ) + eamap.external_email = details.get('email','') + eamap.external_name = '%s %s' % (details.get('first_name',''),details.get('last_name','')) + + def GenPasswd(length=12, chars=string.letters + string.digits): + return ''.join([random.choice(chars) for i in range(length)]) + eamap.internal_password = GenPasswd() + log.debug('created eamap=%s' % eamap) + + eamap.save() + + internal_user = eamap.user + if internal_user is None: + log.debug('ExtAuth: no user for %s yet, doing signup' % eamap.external_email) + return edXauth_signup(request, eamap) + + uname = internal_user.username + user = authenticate(username=uname, password=eamap.internal_password) + if user is None: + log.warning("External Auth Login failed for %s / %s" % (uname,eamap.internal_password)) + return edXauth_signup(request, eamap) + + if not user.is_active: + log.warning("External Auth: user %s is not active" % (uname)) + # TODO: improve error page + return render_failure(request, 'Account not yet activated: please look for link in your email') + + login(request, user) + request.session.set_expiry(0) + student_views.try_change_enrollment(request) + log.info("Login success - {0} ({1})".format(user.username, user.email)) + return redirect('/') + + return render_failure(request, 'Openid failure') + +@ensure_csrf_cookie +@cache_if_anonymous +def edXauth_signup(request, eamap=None): + """ + Present form to complete for signup via external authentication. + Even though the user has external credentials, he/she still needs + to create an account on the edX system, and fill in the user + registration form. + + eamap is an ExteralAuthMap object, specifying the external user + for which to complete the signup. + """ + + if eamap is None: + pass + + request.session['ExternalAuthMap'] = eamap # save this for use by student.views.create_account + + context = {'has_extauth_info': True, + 'show_signup_immediately' : True, + 'extauth_email': eamap.external_email, + 'extauth_username' : eamap.external_name.split(' ')[0], + 'extauth_name': eamap.external_name, + } + + log.debug('ExtAuth: doing signup for %s' % eamap.external_email) + + return student_views.main_index(extra_context=context) From 2f4bf5f606103e10c3aaebeae1a86624e2d76e81 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Wed, 1 Aug 2012 19:17:52 -0400 Subject: [PATCH 063/124] Minor fixes to cms * make logging work properly in cms, dev * fix a comment --- cms/djangoapps/contentstore/views.py | 2 +- cms/envs/dev.py | 7 ++++++- cms/envs/logsettings.py | 11 ++++++----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 0fccc2498b..0447254cfb 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -176,7 +176,7 @@ def load_preview_state(request, preview_id, location): def save_preview_state(request, preview_id, location, instance_state, shared_state): """ - Load the state of a preview module to the request + Save the state of a preview module to the request preview_id (str): An identifier specifying which preview this module is used for location: The Location of the module to dispatch to diff --git a/cms/envs/dev.py b/cms/envs/dev.py index 4098263829..c5e1cf5689 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -2,13 +2,18 @@ This config file runs the simplest dev environment""" from .common import * +from .logsettings import get_logger_config import logging import sys -logging.basicConfig(stream=sys.stdout, ) DEBUG = True TEMPLATE_DEBUG = DEBUG +LOGGING = get_logger_config(ENV_ROOT / "log", + logging_env="dev", + tracking_filename="tracking.log", + debug=True) + MODULESTORE = { 'default': { diff --git a/cms/envs/logsettings.py b/cms/envs/logsettings.py index 31130e33c6..3683314d02 100644 --- a/cms/envs/logsettings.py +++ b/cms/envs/logsettings.py @@ -3,19 +3,19 @@ import os.path import platform import sys -def get_logger_config(log_dir, - logging_env="no_env", +def get_logger_config(log_dir, + logging_env="no_env", tracking_filename=None, syslog_addr=None, debug=False): """Return the appropriate logging config dictionary. You should assign the - result of this to the LOGGING var in your settings. The reason it's done + result of this to the LOGGING var in your settings. The reason it's done this way instead of registering directly is because I didn't want to worry - about resetting the logging state if this is called multiple times when + about resetting the logging state if this is called multiple times when settings are extended.""" # If we're given an explicit place to put tracking logs, we do that (say for - # debugging). However, logging is not safe for multiple processes hitting + # debugging). However, logging is not safe for multiple processes hitting # the same file. So if it's left blank, we dynamically create the filename # based on the PID of this worker process. if tracking_filename: @@ -33,6 +33,7 @@ def get_logger_config(log_dir, return { 'version': 1, + 'disable_existing_loggers': False, 'formatters' : { 'standard' : { 'format' : '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s', From 82899323e86c916b4f4b175f103d2922741e94b2 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Wed, 1 Aug 2012 19:19:58 -0400 Subject: [PATCH 064/124] Add metadata inherit test * also make a placeholder roundtrip export test --- common/lib/xmodule/tests/test_export.py | 8 +++ common/lib/xmodule/tests/test_import.py | 91 +++++++++++++++++++------ 2 files changed, 80 insertions(+), 19 deletions(-) diff --git a/common/lib/xmodule/tests/test_export.py b/common/lib/xmodule/tests/test_export.py index 97da2c4fe5..eacf8352be 100644 --- a/common/lib/xmodule/tests/test_export.py +++ b/common/lib/xmodule/tests/test_export.py @@ -1,5 +1,6 @@ from xmodule.modulestore.xml import XMLModuleStore from nose.tools import assert_equals +from nose import SkipTest from tempfile import mkdtemp from fs.osfs import OSFS @@ -26,3 +27,10 @@ def check_export_roundtrip(data_dir): for location in initial_import.modules.keys(): print "Checking", location assert_equals(initial_import.modules[location], second_import.modules[location]) + + +def test_toy_roundtrip(): + dir = "" + # TODO: add paths and make this run. + raise SkipTest() + check_export_roundtrip(dir) diff --git a/common/lib/xmodule/tests/test_import.py b/common/lib/xmodule/tests/test_import.py index 34ace135a3..dab7e63f1d 100644 --- a/common/lib/xmodule/tests/test_import.py +++ b/common/lib/xmodule/tests/test_import.py @@ -1,36 +1,61 @@ from path import path import unittest +from fs.memoryfs import MemoryFS from lxml import etree from xmodule.x_module import XMLParsingSystem, XModuleDescriptor -from xmodule.errortracker import null_error_tracker +from xmodule.errortracker import make_error_tracker from xmodule.modulestore import Location +from xmodule.modulestore.exceptions import ItemNotFoundError + +ORG = 'test_org' +COURSE = 'test_course' + +class DummySystem(XMLParsingSystem): + def __init__(self): + + self.modules = {} + self.resources_fs = MemoryFS() + self.errorlog = make_error_tracker() + + def load_item(loc): + loc = Location(loc) + if loc in self.modules: + return self.modules[loc] + + print "modules: " + print self.modules + raise ItemNotFoundError("Can't find item at loc: {0}".format(loc)) + + def process_xml(xml): + print "loading {0}".format(xml) + descriptor = XModuleDescriptor.load_from_xml(xml, self, ORG, COURSE, None) + # Need to save module so we can find it later + self.modules[descriptor.location] = descriptor + + # always eager + descriptor.get_children() + return descriptor + + + XMLParsingSystem.__init__(self, load_item, self.resources_fs, + self.errorlog.tracker, process_xml) + + def render_template(self, template, context): + raise Exception("Shouldn't be called") + + + class ImportTestCase(unittest.TestCase): '''Make sure module imports work properly, including for malformed inputs''' + @staticmethod def get_system(): '''Get a dummy system''' - # Shouldn't need any system params, because the initial parse should fail - def load_item(loc): - raise Exception("Shouldn't be called") - - resources_fs = None - - def process_xml(xml): - raise Exception("Shouldn't be called") - - - def render_template(template, context): - raise Exception("Shouldn't be called") - - system = XMLParsingSystem(load_item, resources_fs, - null_error_tracker, process_xml) - system.render_template = render_template - - return system + return DummySystem() def test_fallback(self): '''Make sure that malformed xml loads as an ErrorDescriptor.''' @@ -85,3 +110,31 @@ class ImportTestCase(unittest.TestCase): xml_out = etree.fromstring(xml_str_out) self.assertEqual(xml_out.tag, 'sequential') + def test_metadata_inherit(self): + """Make sure metadata inherits properly""" + system = self.get_system() + v = "1 hour" + start_xml = ''' + + Two houses, ... + '''.format(grace=v) + descriptor = XModuleDescriptor.load_from_xml(start_xml, system, + 'org', 'course') + + print "Errors: {0}".format(system.errorlog.errors) + print descriptor, descriptor.metadata + self.assertEqual(descriptor.metadata['graceperiod'], v) + + # Check that the child inherits correctly + child = descriptor.get_children()[0] + self.assertEqual(child.metadata['graceperiod'], v) + + # Now export and see if the chapter tag has a graceperiod attribute + resource_fs = MemoryFS() + exported_xml = descriptor.export_to_xml(resource_fs) + print "Exported xml:", exported_xml + root = etree.fromstring(exported_xml) + chapter_tag = root[0] + self.assertEqual(chapter_tag.tag, 'chapter') + self.assertFalse('graceperiod' in chapter_tag.attrib) + self.assertTrue(False) From b63f05b6519b990336e1897f50f9a6e896391d0d Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Wed, 1 Aug 2012 19:20:28 -0400 Subject: [PATCH 065/124] update clean_xml script to use error trackers properly --- .../management/commands/clean_xml.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lms/djangoapps/courseware/management/commands/clean_xml.py b/lms/djangoapps/courseware/management/commands/clean_xml.py index 29ce246637..1773c3d21f 100644 --- a/lms/djangoapps/courseware/management/commands/clean_xml.py +++ b/lms/djangoapps/courseware/management/commands/clean_xml.py @@ -53,14 +53,12 @@ def import_with_checks(course_dir, verbose=True): data_dir = course_dir.dirname() course_dirs = [course_dir.basename()] - (error_tracker, errors) = make_error_tracker() # No default class--want to complain if it doesn't find plugins for any # module. modulestore = XMLModuleStore(data_dir, default_class=None, eager=True, - course_dirs=course_dirs, - error_tracker=error_tracker) + course_dirs=course_dirs) def str_of_err(tpl): (msg, exc_info) = tpl @@ -71,6 +69,15 @@ def import_with_checks(course_dir, verbose=True): return '{msg}\n{exc}'.format(msg=msg, exc=exc_str) courses = modulestore.get_courses() + + n = len(courses) + if n != 1: + print 'ERROR: Expect exactly 1 course. Loaded {n}: {lst}'.format( + n=n, lst=courses) + return (False, None) + + course = courses[0] + errors = modulestore.get_item_errors(course.location) if len(errors) != 0: all_ok = False print '\n' @@ -80,13 +87,6 @@ def import_with_checks(course_dir, verbose=True): print "=" * 40 print '\n' - n = len(courses) - if n != 1: - print 'ERROR: Expect exactly 1 course. Loaded {n}: {lst}'.format( - n=n, lst=courses) - return (False, None) - - course = courses[0] #print course validators = ( From 652e2aa24c1d8795a65afd50174216bdcda2a219 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Wed, 1 Aug 2012 19:22:14 -0400 Subject: [PATCH 066/124] fix for inheriting metadata bug * problem was on import to json--got all the metadata, but didn't preserve the _inherited_metadata * added own_metadata property, use it instead --- common/lib/xmodule/xmodule/modulestore/xml.py | 3 +++ common/lib/xmodule/xmodule/modulestore/xml_importer.py | 4 +++- common/lib/xmodule/xmodule/x_module.py | 8 ++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index e7f9c9ce0a..d6540023e8 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -41,6 +41,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): self.used_slugs = set() def process_xml(xml): + """Takes an xml string, and returns a XModuleDescriptor created from + that xml. + """ try: # VS[compat] # TODO (cpennington): Remove this once all fall 2012 courses diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index 578ade95fe..891db7e994 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -35,6 +35,8 @@ def import_from_xml(store, data_dir, course_dirs=None, eager=True, store.update_item(module.location, module.definition['data']) if 'children' in module.definition: store.update_children(module.location, module.definition['children']) - store.update_metadata(module.location, dict(module.metadata)) + # NOTE: It's important to use own_metadata here to avoid writing + # inherited metadata everywhere. + store.update_metadata(module.location, dict(module.own_metadata)) return module_store diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 2a1769bbd7..360f1b07d0 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -358,6 +358,14 @@ class XModuleDescriptor(Plugin, HTMLSnippet): self._child_instances = None self._inherited_metadata = set() + @property + def own_metadata(self): + """ + Return the metadata that is not inherited, but was defined on this module. + """ + return dict((k,v) for k,v in self.metadata.items() + if k not in self._inherited_metadata) + def inherit_metadata(self, metadata): """ Updates this module with metadata inherited from a containing module. From e2ce54612ca13577bedaeafe3c3df5b5857ef9b4 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Wed, 1 Aug 2012 19:25:38 -0400 Subject: [PATCH 067/124] remove debugging assert from test --- common/lib/xmodule/tests/test_import.py | 1 - 1 file changed, 1 deletion(-) diff --git a/common/lib/xmodule/tests/test_import.py b/common/lib/xmodule/tests/test_import.py index dab7e63f1d..6a407fe189 100644 --- a/common/lib/xmodule/tests/test_import.py +++ b/common/lib/xmodule/tests/test_import.py @@ -137,4 +137,3 @@ class ImportTestCase(unittest.TestCase): chapter_tag = root[0] self.assertEqual(chapter_tag.tag, 'chapter') self.assertFalse('graceperiod' in chapter_tag.attrib) - self.assertTrue(False) From ec78e1a126fe9a2603a7a8b75980d1b0c6b21aa5 Mon Sep 17 00:00:00 2001 From: ichuang Date: Wed, 1 Aug 2012 20:19:33 -0400 Subject: [PATCH 068/124] add missing trailing slash --- lms/templates/login_modal.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/templates/login_modal.html b/lms/templates/login_modal.html index e2a2754226..c89931695c 100644 --- a/lms/templates/login_modal.html +++ b/lms/templates/login_modal.html @@ -28,7 +28,7 @@ Forgot password?

    - login via openid + login via openid

    From 102d4906d0d1c94aec6d2854e9e46c6707c0d09e Mon Sep 17 00:00:00 2001 From: ichuang Date: Wed, 1 Aug 2012 22:41:27 -0400 Subject: [PATCH 069/124] fix problem with add_histogram (expects module as second argument) --- lms/djangoapps/courseware/module_render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 80a4ef90fc..16921d1d50 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -184,7 +184,7 @@ def get_module(user, request, location, student_module_cache, position=None): ) if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff: - module.get_html = add_histogram(module.get_html) + module.get_html = add_histogram(module.get_html, module) # If StudentModule for this instance wasn't already in the database, # and this isn't a guest user, create it. From a759850e3e45a04e5953320fdb001e98df6a81be Mon Sep 17 00:00:00 2001 From: ichuang Date: Wed, 1 Aug 2012 22:42:06 -0400 Subject: [PATCH 070/124] add SSL / MIT certificates auth; clean up external_auth.views --- common/djangoapps/external_auth/views.py | 162 +++++++++---- common/djangoapps/student/views.py | 4 + lms/djangoapps/ssl_auth/__init__.py | 0 lms/djangoapps/ssl_auth/ssl_auth.py | 290 ----------------------- lms/envs/dev.py | 3 + 5 files changed, 124 insertions(+), 335 deletions(-) delete mode 100644 lms/djangoapps/ssl_auth/__init__.py delete mode 100755 lms/djangoapps/ssl_auth/ssl_auth.py diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py index 57131f21da..5004d614d5 100644 --- a/common/djangoapps/external_auth/views.py +++ b/common/djangoapps/external_auth/views.py @@ -1,8 +1,7 @@ -# from pprint import pprint - import json import logging import random +import re import string from external_auth.models import ExternalAuthMap @@ -25,7 +24,6 @@ except ImportError: from django_future.csrf import ensure_csrf_cookie from util.cache import cache_if_anonymous -#from django_openid_auth import views as openid_views from django_openid_auth import auth as openid_auth from openid.consumer.consumer import (Consumer, SUCCESS, CANCEL, FAILURE) import django_openid_auth @@ -43,6 +41,12 @@ def default_render_failure(request, message, status=403, template_name='extauth_ data = render_to_string( template_name, dict(message=message, exception=exception)) return HttpResponse(data, status=status) +#----------------------------------------------------------------------------- +# Openid + +def GenPasswd(length=12, chars=string.letters + string.digits): + return ''.join([random.choice(chars) for i in range(length)]) + @csrf_exempt def edXauth_openid_login_complete(request, redirect_field_name=REDIRECT_FIELD_NAME, render_failure=None): """Complete the openid login process""" @@ -63,50 +67,62 @@ def edXauth_openid_login_complete(request, redirect_field_name=REDIRECT_FIELD_N log.debug('openid success, details=%s' % details) - # see if we have a map from this external_id to an edX username - try: - eamap = ExternalAuthMap.objects.get(external_id=external_id) - log.debug('Found eamap=%s' % eamap) - except ExternalAuthMap.DoesNotExist: - # go render form for creating edX user - eamap = ExternalAuthMap(external_id = external_id, - external_domain = "openid:%s" % settings.OPENID_SSO_SERVER_URL, - external_credentials = json.dumps(details), - ) - eamap.external_email = details.get('email','') - eamap.external_name = '%s %s' % (details.get('first_name',''),details.get('last_name','')) - - def GenPasswd(length=12, chars=string.letters + string.digits): - return ''.join([random.choice(chars) for i in range(length)]) - eamap.internal_password = GenPasswd() - log.debug('created eamap=%s' % eamap) - - eamap.save() - - internal_user = eamap.user - if internal_user is None: - log.debug('ExtAuth: no user for %s yet, doing signup' % eamap.external_email) - return edXauth_signup(request, eamap) - - uname = internal_user.username - user = authenticate(username=uname, password=eamap.internal_password) - if user is None: - log.warning("External Auth Login failed for %s / %s" % (uname,eamap.internal_password)) - return edXauth_signup(request, eamap) - - if not user.is_active: - log.warning("External Auth: user %s is not active" % (uname)) - # TODO: improve error page - return render_failure(request, 'Account not yet activated: please look for link in your email') - - login(request, user) - request.session.set_expiry(0) - student_views.try_change_enrollment(request) - log.info("Login success - {0} ({1})".format(user.username, user.email)) - return redirect('/') - + return edXauth_external_login_or_signup(request, + external_id, + "openid:%s" % settings.OPENID_SSO_SERVER_URL, + details, + details.get('email',''), + '%s %s' % (details.get('first_name',''),details.get('last_name','')) + ) + return render_failure(request, 'Openid failure') + +#----------------------------------------------------------------------------- +# generic external auth login or signup + +def edXauth_external_login_or_signup(request, external_id, external_domain, credentials, email, fullname): + # see if we have a map from this external_id to an edX username + try: + eamap = ExternalAuthMap.objects.get(external_id=external_id) + log.debug('Found eamap=%s' % eamap) + except ExternalAuthMap.DoesNotExist: + # go render form for creating edX user + eamap = ExternalAuthMap(external_id = external_id, + external_domain = external_domain, + external_credentials = json.dumps(credentials), + ) + eamap.external_email = email + eamap.external_name = fullname + eamap.internal_password = GenPasswd() + log.debug('created eamap=%s' % eamap) + + eamap.save() + + internal_user = eamap.user + if internal_user is None: + log.debug('ExtAuth: no user for %s yet, doing signup' % eamap.external_email) + return edXauth_signup(request, eamap) + uname = internal_user.username + user = authenticate(username=uname, password=eamap.internal_password) + if user is None: + log.warning("External Auth Login failed for %s / %s" % (uname,eamap.internal_password)) + return edXauth_signup(request, eamap) + + if not user.is_active: + log.warning("External Auth: user %s is not active" % (uname)) + # TODO: improve error page + return render_failure(request, 'Account not yet activated: please look for link in your email') + + login(request, user) + request.session.set_expiry(0) + student_views.try_change_enrollment(request) + log.info("Login success - {0} ({1})".format(user.username, user.email)) + return redirect('/') + +#----------------------------------------------------------------------------- +# generic external auth signup + @ensure_csrf_cookie @cache_if_anonymous def edXauth_signup(request, eamap=None): @@ -135,3 +151,59 @@ def edXauth_signup(request, eamap=None): log.debug('ExtAuth: doing signup for %s' % eamap.external_email) return student_views.main_index(extra_context=context) + +#----------------------------------------------------------------------------- +# MIT SSL + +def ssl_dn_extract_info(dn): + ''' + Extract username, email address (may be anyuser@anydomain.com) and full name + from the SSL DN string. Return (user,email,fullname) if successful, and None + otherwise. + ''' + ss = re.search('/emailAddress=(.*)@([^/]+)', dn) + if ss: + user = ss.group(1) + email = "%s@%s" % (user, ss.group(2)) + else: + return None + ss = re.search('/CN=([^/]+)/', dn) + if ss: + fullname = ss.group(1) + else: + return None + return (user, email, fullname) + +@csrf_exempt +def edXauth_ssl_login(request): + """ + This is called by student.views.index when MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True + + Used for MIT user authentication. This presumes the web server (nginx) has been configured + to require specific client certificates. + + If the incoming protocol is HTTPS (SSL) then authenticate via client certificate. + The certificate provides user email and fullname; this populates the ExternalAuthMap. + The user is nevertheless still asked to complete the edX signup. + + Else continues on with student.views.main_index, and no authentication. + """ + certkey = "SSL_CLIENT_S_DN" # specify the request.META field to use + + cert = request.META.get(certkey,'') + if not cert: + cert = request.META.get('HTTP_'+certkey,'') + if not cert: + cert = request._req.subprocess_env.get(certkey,'') # try the direct apache2 SSL key + if not cert: + # no certificate information - go onward to main index + return student_views.main_index() + + (user, email, fullname) = ssl_dn_extract_info(cert) + + return edXauth_external_login_or_signup(request, + external_id=email, + external_domain="ssl:MIT", + credentials=cert, + email=email, + fullname=fullname) diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index 3ba83f42bb..7937d67980 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -60,6 +60,10 @@ def index(request): if settings.COURSEWARE_ENABLED and request.user.is_authenticated(): return redirect(reverse('dashboard')) + if settings.MITX_FEATURES.get('AUTH_USE_MIT_CERTIFICATES'): + from external_auth.views import edXauth_ssl_login + return edXauth_ssl_login(request) + return main_index() def main_index(extra_context = {}): diff --git a/lms/djangoapps/ssl_auth/__init__.py b/lms/djangoapps/ssl_auth/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lms/djangoapps/ssl_auth/ssl_auth.py b/lms/djangoapps/ssl_auth/ssl_auth.py deleted file mode 100755 index adbb2bf94d..0000000000 --- a/lms/djangoapps/ssl_auth/ssl_auth.py +++ /dev/null @@ -1,290 +0,0 @@ -""" -User authentication backend for ssl (no pw required) -""" - -from django.conf import settings -from django.contrib import auth -from django.contrib.auth.models import User, check_password -from django.contrib.auth.backends import ModelBackend -from django.contrib.auth.middleware import RemoteUserMiddleware -from django.core.exceptions import ImproperlyConfigured -import os -import string -import re -from random import choice - -from student.models import UserProfile - -#----------------------------------------------------------------------------- - - -def ssl_dn_extract_info(dn): - ''' - Extract username, email address (may be anyuser@anydomain.com) and full name - from the SSL DN string. Return (user,email,fullname) if successful, and None - otherwise. - ''' - ss = re.search('/emailAddress=(.*)@([^/]+)', dn) - if ss: - user = ss.group(1) - email = "%s@%s" % (user, ss.group(2)) - else: - return None - ss = re.search('/CN=([^/]+)/', dn) - if ss: - fullname = ss.group(1) - else: - return None - return (user, email, fullname) - - -def check_nginx_proxy(request): - ''' - Check for keys in the HTTP header (META) to se if we are behind an ngix reverse proxy. - If so, get user info from the SSL DN string and return that, as (user,email,fullname) - ''' - m = request.META - if m.has_key('HTTP_X_REAL_IP'): # we're behind a nginx reverse proxy, which has already done ssl auth - if not m.has_key('HTTP_SSL_CLIENT_S_DN'): - return None - dn = m['HTTP_SSL_CLIENT_S_DN'] - return ssl_dn_extract_info(dn) - return None - -#----------------------------------------------------------------------------- - - -def get_ssl_username(request): - x = check_nginx_proxy(request) - if x: - return x[0] - env = request._req.subprocess_env - if env.has_key('SSL_CLIENT_S_DN_Email'): - email = env['SSL_CLIENT_S_DN_Email'] - user = email[:email.index('@')] - return user - return None - -#----------------------------------------------------------------------------- - - -class NginxProxyHeaderMiddleware(RemoteUserMiddleware): - ''' - Django "middleware" function for extracting user information from HTTP request. - - ''' - # this field is generated by nginx's reverse proxy - header = 'HTTP_SSL_CLIENT_S_DN' # specify the request.META field to use - - def process_request(self, request): - # AuthenticationMiddleware is required so that request.user exists. - if not hasattr(request, 'user'): - raise ImproperlyConfigured( - "The Django remote user auth middleware requires the" - " authentication middleware to be installed. Edit your" - " MIDDLEWARE_CLASSES setting to insert" - " 'django.contrib.auth.middleware.AuthenticationMiddleware'" - " before the RemoteUserMiddleware class.") - - #raise ImproperlyConfigured('[ProxyHeaderMiddleware] request.META=%s' % repr(request.META)) - - try: - username = request.META[self.header] # try the nginx META key first - except KeyError: - try: - env = request._req.subprocess_env # else try the direct apache2 SSL key - if env.has_key('SSL_CLIENT_S_DN'): - username = env['SSL_CLIENT_S_DN'] - else: - raise ImproperlyConfigured('no ssl key, env=%s' % repr(env)) - username = '' - except: - # If specified header doesn't exist then return (leaving - # request.user set to AnonymousUser by the - # AuthenticationMiddleware). - return - # If the user is already authenticated and that user is the user we are - # getting passed in the headers, then the correct user is already - # persisted in the session and we don't need to continue. - - #raise ImproperlyConfigured('[ProxyHeaderMiddleware] username=%s' % username) - - if request.user.is_authenticated(): - if request.user.username == self.clean_username(username, request): - #raise ImproperlyConfigured('%s already authenticated (%s)' % (username,request.user.username)) - return - # We are seeing this user for the first time in this session, attempt - # to authenticate the user. - #raise ImproperlyConfigured('calling auth.authenticate, remote_user=%s' % username) - user = auth.authenticate(remote_user=username) - if user: - # User is valid. Set request.user and persist user in the session - # by logging the user in. - request.user = user - if settings.DEBUG: print "[ssl_auth.ssl_auth.NginxProxyHeaderMiddleware] logging in user=%s" % user - auth.login(request, user) - - def clean_username(self, username, request): - ''' - username is the SSL DN string - extract the actual username from it and return - ''' - info = ssl_dn_extract_info(username) - if not info: - return None - (username, email, fullname) = info - return username - -#----------------------------------------------------------------------------- - - -class SSLLoginBackend(ModelBackend): - ''' - Django authentication back-end which auto-logs-in a user based on having - already authenticated with an MIT certificate (SSL). - ''' - def authenticate(self, username=None, password=None, remote_user=None): - - # remote_user is from the SSL_DN string. It will be non-empty only when - # the user has already passed the server authentication, which means - # matching with the certificate authority. - if not remote_user: - # no remote_user, so check username (but don't auto-create user) - if not username: - return None - return None # pass on to another authenticator backend - #raise ImproperlyConfigured("in SSLLoginBackend, username=%s, remote_user=%s" % (username,remote_user)) - try: - user = User.objects.get(username=username) # if user already exists don't create it - return user - except User.DoesNotExist: - return None - return None - - #raise ImproperlyConfigured("in SSLLoginBackend, username=%s, remote_user=%s" % (username,remote_user)) - #if not os.environ.has_key('HTTPS'): - # return None - #if not os.environ.get('HTTPS')=='on': # only use this back-end if HTTPS on - # return None - - def GenPasswd(length=8, chars=string.letters + string.digits): - return ''.join([choice(chars) for i in range(length)]) - - # convert remote_user to user, email, fullname - info = ssl_dn_extract_info(remote_user) - #raise ImproperlyConfigured("[SSLLoginBackend] looking up %s" % repr(info)) - if not info: - #raise ImproperlyConfigured("[SSLLoginBackend] remote_user=%s, info=%s" % (remote_user,info)) - return None - (username, email, fullname) = info - - try: - user = User.objects.get(username=username) # if user already exists don't create it - except User.DoesNotExist: - if not settings.DEBUG: - raise "User does not exist. Not creating user; potential schema consistency issues" - #raise ImproperlyConfigured("[SSLLoginBackend] creating %s" % repr(info)) - user = User(username=username, password=GenPasswd()) # create new User - user.is_staff = False - user.is_superuser = False - # get first, last name from fullname - name = fullname - if not name.count(' '): - user.first_name = " " - user.last_name = name - mn = '' - else: - user.first_name = name[:name.find(' ')] - ml = name[name.find(' '):].strip() - if ml.count(' '): - user.last_name = ml[ml.rfind(' '):] - mn = ml[:ml.rfind(' ')] - else: - user.last_name = ml - mn = '' - # set email - user.email = email - # cleanup last name - user.last_name = user.last_name.strip() - # save - user.save() - - # auto-create user profile - up = UserProfile(user=user) - up.name = fullname - up.save() - - #tui = user.get_profile() - #tui.middle_name = mn - #tui.role = 'Misc' - #tui.section = None # no section assigned at first - #tui.save() - # return None - return user - - def get_user(self, user_id): - #if not os.environ.has_key('HTTPS'): - # return None - #if not os.environ.get('HTTPS')=='on': # only use this back-end if HTTPS on - # return None - try: - return User.objects.get(pk=user_id) - except User.DoesNotExist: - return None - -#----------------------------------------------------------------------------- -# OLD! - - -class AutoLoginBackend: - def authenticate(self, username=None, password=None): - raise ImproperlyConfigured("in AutoLoginBackend, username=%s" % username) - if not os.environ.has_key('HTTPS'): - return None - if not os.environ.get('HTTPS') == 'on':# only use this back-end if HTTPS on - return None - - def GenPasswd(length=8, chars=string.letters + string.digits): - return ''.join([choice(chars) for i in range(length)]) - - try: - user = User.objects.get(username=username) - except User.DoesNotExist: - user = User(username=username, password=GenPasswd()) - user.is_staff = False - user.is_superuser = False - # get first, last name - name = os.environ.get('SSL_CLIENT_S_DN_CN').strip() - if not name.count(' '): - user.first_name = " " - user.last_name = name - mn = '' - else: - user.first_name = name[:name.find(' ')] - ml = name[name.find(' '):].strip() - if ml.count(' '): - user.last_name = ml[ml.rfind(' '):] - mn = ml[:ml.rfind(' ')] - else: - user.last_name = ml - mn = '' - # get email - user.email = os.environ.get('SSL_CLIENT_S_DN_Email') - # save - user.save() - tui = user.get_profile() - tui.middle_name = mn - tui.role = 'Misc' - tui.section = None# no section assigned at first - tui.save() - # return None - return user - - def get_user(self, user_id): - if not os.environ.has_key('HTTPS'): - return None - if not os.environ.get('HTTPS') == 'on':# only use this back-end if HTTPS on - return None - try: - return User.objects.get(pk=user_id) - except User.DoesNotExist: - return None diff --git a/lms/envs/dev.py b/lms/envs/dev.py index f9b7ba10a0..83bc596f1e 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -67,6 +67,9 @@ OPENID_UPDATE_DETAILS_FROM_SREG = True OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id' # TODO: accept more endpoints OPENID_USE_AS_ADMIN_LOGIN = False +################################ MIT Certificates SSL Auth ################################# +MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True + ################################ DEBUG TOOLBAR ################################# INSTALLED_APPS += ('debug_toolbar',) MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) From 4a0d0a08db20fe64fa0dceb08d157cce1cfaa026 Mon Sep 17 00:00:00 2001 From: ichuang Date: Wed, 1 Aug 2012 23:37:35 -0400 Subject: [PATCH 071/124] minor change so that SSL code doesn't interfere with non-nginx instances --- common/djangoapps/external_auth/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py index 5004d614d5..41062735d4 100644 --- a/common/djangoapps/external_auth/views.py +++ b/common/djangoapps/external_auth/views.py @@ -194,7 +194,10 @@ def edXauth_ssl_login(request): if not cert: cert = request.META.get('HTTP_'+certkey,'') if not cert: - cert = request._req.subprocess_env.get(certkey,'') # try the direct apache2 SSL key + try: + cert = request._req.subprocess_env.get(certkey,'') # try the direct apache2 SSL key + except Exception as err: + pass if not cert: # no certificate information - go onward to main index return student_views.main_index() From 727e51411f7e096c92745c3deb630bd77b2f119c Mon Sep 17 00:00:00 2001 From: ichuang Date: Thu, 2 Aug 2012 08:59:02 -0400 Subject: [PATCH 072/124] small change so that ssl authenticated user can logout to see main screen --- common/djangoapps/external_auth/views.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py index 41062735d4..55ff4b4194 100644 --- a/common/djangoapps/external_auth/views.py +++ b/common/djangoapps/external_auth/views.py @@ -80,7 +80,8 @@ def edXauth_openid_login_complete(request, redirect_field_name=REDIRECT_FIELD_N #----------------------------------------------------------------------------- # generic external auth login or signup -def edXauth_external_login_or_signup(request, external_id, external_domain, credentials, email, fullname): +def edXauth_external_login_or_signup(request, external_id, external_domain, credentials, email, fullname, + retfun=None): # see if we have a map from this external_id to an edX username try: eamap = ExternalAuthMap.objects.get(external_id=external_id) @@ -118,7 +119,10 @@ def edXauth_external_login_or_signup(request, external_id, external_domain, cred request.session.set_expiry(0) student_views.try_change_enrollment(request) log.info("Login success - {0} ({1})".format(user.username, user.email)) - return redirect('/') + if retfun is None: + return redirect('/') + return retfun() + #----------------------------------------------------------------------------- # generic external auth signup @@ -209,4 +213,5 @@ def edXauth_ssl_login(request): external_domain="ssl:MIT", credentials=cert, email=email, - fullname=fullname) + fullname=fullname, + retfun = student_views.main_index) From 052ae881110f7e127b3566f6b2c7c6e41e8df177 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 2 Aug 2012 09:19:25 -0400 Subject: [PATCH 073/124] Only import from github when changes are made, rather than exporting. This prevents bugs in the cms from preventing changes via git --- cms/djangoapps/github_sync/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/github_sync/views.py b/cms/djangoapps/github_sync/views.py index 941d50f986..c3b5172b29 100644 --- a/cms/djangoapps/github_sync/views.py +++ b/cms/djangoapps/github_sync/views.py @@ -5,7 +5,7 @@ from django.http import HttpResponse from django.conf import settings from django_future.csrf import csrf_exempt -from . import sync_with_github, load_repo_settings +from . import import_from_github, load_repo_settings log = logging.getLogger() @@ -46,6 +46,6 @@ def github_post_receive(request): log.info('Ignoring changes to non-tracked branch %s in repo %s' % (branch_name, repo_name)) return HttpResponse('Ignoring non-tracked branch') - sync_with_github(repo) + import_from_github(repo) return HttpResponse('Push received') From 23c3c5a6529db587e505aed01f908ea6684e3b8c Mon Sep 17 00:00:00 2001 From: ichuang Date: Thu, 2 Aug 2012 09:37:24 -0400 Subject: [PATCH 074/124] print -> log.debug, rename function from camel case --- common/djangoapps/external_auth/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py index 55ff4b4194..fdffaf0c1d 100644 --- a/common/djangoapps/external_auth/views.py +++ b/common/djangoapps/external_auth/views.py @@ -35,16 +35,16 @@ log = logging.getLogger("mitx.external_auth") @csrf_exempt def default_render_failure(request, message, status=403, template_name='extauth_failure.html', exception=None): """Render an Openid error page to the user.""" - message = "In openid_failure " + message - print "in openid_failure: " + log.debug(message) data = render_to_string( template_name, dict(message=message, exception=exception)) return HttpResponse(data, status=status) #----------------------------------------------------------------------------- # Openid -def GenPasswd(length=12, chars=string.letters + string.digits): +def edXauth_generate_password(length=12, chars=string.letters + string.digits): + """Generate internal password for externally authenticated user""" return ''.join([random.choice(chars) for i in range(length)]) @csrf_exempt @@ -94,7 +94,7 @@ def edXauth_external_login_or_signup(request, external_id, external_domain, cred ) eamap.external_email = email eamap.external_name = fullname - eamap.internal_password = GenPasswd() + eamap.internal_password = edXauth_generate_password() log.debug('created eamap=%s' % eamap) eamap.save() From b2e9d980ff4840539fd593f43c175f1040221d3b Mon Sep 17 00:00:00 2001 From: ichuang Date: Thu, 2 Aug 2012 09:42:26 -0400 Subject: [PATCH 075/124] don't overwrite oid_backend --- common/djangoapps/external_auth/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py index fdffaf0c1d..827f2bc4c5 100644 --- a/common/djangoapps/external_auth/views.py +++ b/common/djangoapps/external_auth/views.py @@ -63,7 +63,7 @@ def edXauth_openid_login_complete(request, redirect_field_name=REDIRECT_FIELD_N if openid_response.status == SUCCESS: external_id = openid_response.identity_url oid_backend = openid_auth.OpenIDBackend() - details = oid_backened = oid_backend._extract_user_details(openid_response) + details = oid_backend._extract_user_details(openid_response) log.debug('openid success, details=%s' % details) From fc0f938eae4280c714ad870459e7a05410131840 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Wed, 1 Aug 2012 20:44:27 -0400 Subject: [PATCH 076/124] Responding to comments on pull #326 * cleaned up error module: - only one template - save error message in xml and reload * better display of problem definition and metadata to staff * save error messages as string, not exception objects. --- common/djangoapps/xmodule_modifiers.py | 4 +- common/lib/xmodule/tests/test_import.py | 1 + common/lib/xmodule/xmodule/error_module.py | 49 +++++++++++++--------- common/lib/xmodule/xmodule/errortracker.py | 8 +++- common/lib/xmodule/xmodule/x_module.py | 14 ++++--- lms/templates/module-error-staff.html | 11 ----- lms/templates/module-error.html | 9 ++++ 7 files changed, 57 insertions(+), 39 deletions(-) delete mode 100644 lms/templates/module-error-staff.html diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py index b942d6726b..4d412000ec 100644 --- a/common/djangoapps/xmodule_modifiers.py +++ b/common/djangoapps/xmodule_modifiers.py @@ -89,8 +89,8 @@ def add_histogram(get_html, module): else: edit_link = False - staff_context = {'definition': dict(module.definition), - 'metadata': dict(module.metadata), + staff_context = {'definition': json.dumps(module.definition, indent=4), + 'metadata': json.dumps(module.metadata, indent=4), 'element_id': module.location.html_id(), 'edit_link': edit_link, 'histogram': json.dumps(histogram), diff --git a/common/lib/xmodule/tests/test_import.py b/common/lib/xmodule/tests/test_import.py index 6a407fe189..0d3e2260fb 100644 --- a/common/lib/xmodule/tests/test_import.py +++ b/common/lib/xmodule/tests/test_import.py @@ -73,6 +73,7 @@ class ImportTestCase(unittest.TestCase): def test_reimport(self): '''Make sure an already-exported error xml tag loads properly''' + self.maxDiff = None bad_xml = '''''' system = self.get_system() descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course', diff --git a/common/lib/xmodule/xmodule/error_module.py b/common/lib/xmodule/xmodule/error_module.py index 62b3b82a39..ecc90873b9 100644 --- a/common/lib/xmodule/xmodule/error_module.py +++ b/common/lib/xmodule/xmodule/error_module.py @@ -1,11 +1,14 @@ +import sys +import logging + from pkg_resources import resource_string from lxml import etree from xmodule.x_module import XModule from xmodule.mako_module import MakoModuleDescriptor from xmodule.xml_module import XmlDescriptor from xmodule.editing_module import EditingDescriptor +from xmodule.errortracker import exc_info_to_str -import logging log = logging.getLogger(__name__) @@ -14,14 +17,11 @@ class ErrorModule(XModule): '''Show an error. TODO (vshnayder): proper style, divs, etc. ''' - if not self.system.is_staff: - return self.system.render_template('module-error.html', {}) - # staff get to see all the details - return self.system.render_template('module-error-staff.html', { - 'data' : self.definition['data'], - # TODO (vshnayder): need to get non-syntax errors in here somehow - 'error' : self.definition.get('error', 'Error not available') + return self.system.render_template('module-error.html', { + 'data' : self.definition['data']['contents'], + 'error' : self.definition['data']['error_msg'], + 'is_staff' : self.system.is_staff, }) class ErrorDescriptor(EditingDescriptor): @@ -31,29 +31,36 @@ class ErrorDescriptor(EditingDescriptor): module_class = ErrorModule @classmethod - def from_xml(cls, xml_data, system, org=None, course=None, err=None): + def from_xml(cls, xml_data, system, org=None, course=None, + error_msg='Error not available'): '''Create an instance of this descriptor from the supplied data. Does not try to parse the data--just stores it. Takes an extra, optional, parameter--the error that caused an - issue. + issue. (should be a string, or convert usefully into one). ''' - - definition = {} - if err is not None: - definition['error'] = err + # Use a nested inner dictionary because 'data' is hardcoded + inner = {} + definition = {'data': inner} + inner['error_msg'] = str(error_msg) try: # If this is already an error tag, don't want to re-wrap it. xml_obj = etree.fromstring(xml_data) if xml_obj.tag == 'error': xml_data = xml_obj.text - except etree.XMLSyntaxError as err: - # Save the error to display later--overrides other problems - definition['error'] = err + error_node = xml_obj.find('error_msg') + if error_node is not None: + inner['error_msg'] = error_node.text + else: + inner['error_msg'] = 'Error not available' - definition['data'] = xml_data + except etree.XMLSyntaxError: + # Save the error to display later--overrides other problems + inner['error_msg'] = exc_info_to_str(sys.exc_info()) + + inner['contents'] = xml_data # TODO (vshnayder): Do we need a unique slug here? Just pick a random # 64-bit num? location = ['i4x', org, course, 'error', 'slug'] @@ -71,10 +78,12 @@ class ErrorDescriptor(EditingDescriptor): files, etc. That would just get re-wrapped on import. ''' try: - xml = etree.fromstring(self.definition['data']) + xml = etree.fromstring(self.definition['data']['contents']) return etree.tostring(xml) except etree.XMLSyntaxError: # still not valid. root = etree.Element('error') - root.text = self.definition['data'] + root.text = self.definition['data']['contents'] + err_node = etree.SubElement(root, 'error_msg') + err_node.text = self.definition['data']['error_msg'] return etree.tostring(root) diff --git a/common/lib/xmodule/xmodule/errortracker.py b/common/lib/xmodule/xmodule/errortracker.py index b8d42a6983..8ac2903149 100644 --- a/common/lib/xmodule/xmodule/errortracker.py +++ b/common/lib/xmodule/xmodule/errortracker.py @@ -8,6 +8,12 @@ log = logging.getLogger(__name__) ErrorLog = namedtuple('ErrorLog', 'tracker errors') +def exc_info_to_str(exc_info): + """Given some exception info, convert it into a string using + the traceback.format_exception() function. + """ + return ''.join(traceback.format_exception(*exc_info)) + def in_exception_handler(): '''Is there an active exception?''' return sys.exc_info() != (None, None, None) @@ -27,7 +33,7 @@ def make_error_tracker(): '''Log errors''' exc_str = '' if in_exception_handler(): - exc_str = ''.join(traceback.format_exception(*sys.exc_info())) + exc_str = exc_info_to_str(sys.exc_info()) errors.append((msg, exc_str)) diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 360f1b07d0..ac6b5db5a4 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -1,12 +1,14 @@ -from lxml import etree -from lxml.etree import XMLSyntaxError -import pkg_resources import logging +import pkg_resources +import sys + from fs.errors import ResourceNotFoundError from functools import partial +from lxml import etree +from lxml.etree import XMLSyntaxError from xmodule.modulestore import Location - +from xmodule.errortracker import exc_info_to_str log = logging.getLogger('mitx.' + __name__) @@ -471,7 +473,9 @@ class XModuleDescriptor(Plugin, HTMLSnippet): msg = "Error loading from xml." log.exception(msg) system.error_tracker(msg) - descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course, err) + err_msg = msg + "\n" + exc_info_to_str(sys.exc_info()) + descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course, + err_msg) return descriptor diff --git a/lms/templates/module-error-staff.html b/lms/templates/module-error-staff.html deleted file mode 100644 index 955f413db3..0000000000 --- a/lms/templates/module-error-staff.html +++ /dev/null @@ -1,11 +0,0 @@ -
    -

    There has been an error on the MITx servers

    -

    We're sorry, this module is temporarily unavailable. Our staff is working to fix it as soon as possible. Please email us at technical@mitx.mit.edu to report any problems or downtime.

    - -

    Staff-only details below:

    - -

    Error: ${error}

    - -

    Raw data: ${data}

    - -
    diff --git a/lms/templates/module-error.html b/lms/templates/module-error.html index 28597fa13c..7c731db17a 100644 --- a/lms/templates/module-error.html +++ b/lms/templates/module-error.html @@ -1,4 +1,13 @@

    There has been an error on the MITx servers

    We're sorry, this module is temporarily unavailable. Our staff is working to fix it as soon as possible. Please email us at technical@mitx.mit.edu to report any problems or downtime.

    + +% if is_staff: +

    Staff-only details below:

    + +

    Error: ${error | h}

    + +

    Raw data: ${data | h}

    +% endif +
    From f2a9110bdaa804bf20a91d9020113f2657d05c76 Mon Sep 17 00:00:00 2001 From: ichuang Date: Thu, 2 Aug 2012 09:56:33 -0400 Subject: [PATCH 077/124] change model to have external_id and external_domain be unique_together --- common/djangoapps/external_auth/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/djangoapps/external_auth/models.py b/common/djangoapps/external_auth/models.py index b8b167d25d..e43b306bbb 100644 --- a/common/djangoapps/external_auth/models.py +++ b/common/djangoapps/external_auth/models.py @@ -13,7 +13,9 @@ from django.db import models from django.contrib.auth.models import User class ExternalAuthMap(models.Model): - external_id = models.CharField(max_length=255, db_index=True, unique=True) + class Meta: + unique_together = (('external_id', 'external_domain'), ) + external_id = models.CharField(max_length=255, db_index=True) external_domain = models.CharField(max_length=255, db_index=True) external_credentials = models.TextField(blank=True) # JSON dictionary external_email = models.CharField(max_length=255, db_index=True) From 613c53a7109b9162acc439e202b3430f099ee35f Mon Sep 17 00:00:00 2001 From: ichuang Date: Thu, 2 Aug 2012 10:05:26 -0400 Subject: [PATCH 078/124] slight cleanup, no need to import all of django_openid_auth --- common/djangoapps/external_auth/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py index 827f2bc4c5..740b4ed1ac 100644 --- a/common/djangoapps/external_auth/views.py +++ b/common/djangoapps/external_auth/views.py @@ -26,7 +26,7 @@ from util.cache import cache_if_anonymous from django_openid_auth import auth as openid_auth from openid.consumer.consumer import (Consumer, SUCCESS, CANCEL, FAILURE) -import django_openid_auth +import django_openid_auth.views as openid_views import student.views as student_views @@ -56,7 +56,7 @@ def edXauth_openid_login_complete(request, redirect_field_name=REDIRECT_FIELD_N getattr(settings, 'OPENID_RENDER_FAILURE', None) or \ default_render_failure - openid_response = django_openid_auth.views.parse_openid_response(request) + openid_response = openid_views.parse_openid_response(request) if not openid_response: return render_failure(request, 'This is an OpenID relying party endpoint.') From 41eca8a0a54a974b6a2b231e786ffd3573e4cd81 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Thu, 2 Aug 2012 10:13:58 -0400 Subject: [PATCH 079/124] Move files into xmodule module per comment on #326 * move html_checker.py * move stringify.py * move tests into module * remove duplicate progress.py * fix module imports --- common/lib/xmodule/progress.py | 157 ------------------ common/lib/xmodule/tests/test_stringify.py | 9 - .../lib/xmodule/{ => xmodule}/html_checker.py | 0 common/lib/xmodule/xmodule/html_module.py | 10 +- common/lib/xmodule/{ => xmodule}/stringify.py | 0 .../xmodule/{ => xmodule}/tests/__init__.py | 0 .../{ => xmodule}/tests/test_export.py | 0 .../test_files/choiceresponse_checkbox.xml | 0 .../tests/test_files/choiceresponse_radio.xml | 0 .../tests/test_files/coderesponse.xml | 0 .../test_files/formularesponse_with_hint.xml | 0 .../tests/test_files/imageresponse.xml | 0 .../tests/test_files/multi_bare.xml | 0 .../tests/test_files/multichoice.xml | 0 .../tests/test_files/optionresponse.xml | 0 .../test_files/stringresponse_with_hint.xml | 0 .../tests/test_files/symbolicresponse.xml | 0 .../tests/test_files/truefalse.xml | 0 .../{ => xmodule}/tests/test_import.py | 0 .../xmodule/xmodule/tests/test_stringify.py | 10 ++ 20 files changed, 15 insertions(+), 171 deletions(-) delete mode 100644 common/lib/xmodule/progress.py delete mode 100644 common/lib/xmodule/tests/test_stringify.py rename common/lib/xmodule/{ => xmodule}/html_checker.py (100%) rename common/lib/xmodule/{ => xmodule}/stringify.py (100%) rename common/lib/xmodule/{ => xmodule}/tests/__init__.py (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_export.py (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/choiceresponse_checkbox.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/choiceresponse_radio.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/coderesponse.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/formularesponse_with_hint.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/imageresponse.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/multi_bare.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/multichoice.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/optionresponse.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/stringresponse_with_hint.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/symbolicresponse.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_files/truefalse.xml (100%) rename common/lib/xmodule/{ => xmodule}/tests/test_import.py (100%) create mode 100644 common/lib/xmodule/xmodule/tests/test_stringify.py diff --git a/common/lib/xmodule/progress.py b/common/lib/xmodule/progress.py deleted file mode 100644 index 70c8ec9da1..0000000000 --- a/common/lib/xmodule/progress.py +++ /dev/null @@ -1,157 +0,0 @@ -''' -Progress class for modules. Represents where a student is in a module. - -Useful things to know: - - Use Progress.to_js_status_str() to convert a progress into a simple - status string to pass to js. - - Use Progress.to_js_detail_str() to convert a progress into a more detailed - string to pass to js. - -In particular, these functions have a canonical handing of None. - -For most subclassing needs, you should only need to reimplement -frac() and __str__(). -''' - -from collections import namedtuple -import numbers - - -class Progress(object): - '''Represents a progress of a/b (a out of b done) - - a and b must be numeric, but not necessarily integer, with - 0 <= a <= b and b > 0. - - Progress can only represent Progress for modules where that makes sense. Other - modules (e.g. html) should return None from get_progress(). - - TODO: add tag for module type? Would allow for smarter merging. - ''' - - def __init__(self, a, b): - '''Construct a Progress object. a and b must be numbers, and must have - 0 <= a <= b and b > 0 - ''' - - # Want to do all checking at construction time, so explicitly check types - if not (isinstance(a, numbers.Number) and - isinstance(b, numbers.Number)): - raise TypeError('a and b must be numbers. Passed {0}/{1}'.format(a, b)) - - if not (0 <= a <= b and b > 0): - raise ValueError( - 'fraction a/b = {0}/{1} must have 0 <= a <= b and b > 0'.format(a, b)) - - self._a = a - self._b = b - - def frac(self): - ''' Return tuple (a,b) representing progress of a/b''' - return (self._a, self._b) - - def percent(self): - ''' Returns a percentage progress as a float between 0 and 100. - - subclassing note: implemented in terms of frac(), assumes sanity - checking is done at construction time. - ''' - (a, b) = self.frac() - return 100.0 * a / b - - def started(self): - ''' Returns True if fractional progress is greater than 0. - - subclassing note: implemented in terms of frac(), assumes sanity - checking is done at construction time. - ''' - return self.frac()[0] > 0 - - def inprogress(self): - ''' Returns True if fractional progress is strictly between 0 and 1. - - subclassing note: implemented in terms of frac(), assumes sanity - checking is done at construction time. - ''' - (a, b) = self.frac() - return a > 0 and a < b - - def done(self): - ''' Return True if this represents done. - - subclassing note: implemented in terms of frac(), assumes sanity - checking is done at construction time. - ''' - (a, b) = self.frac() - return a == b - - def ternary_str(self): - ''' Return a string version of this progress: either - "none", "in_progress", or "done". - - subclassing note: implemented in terms of frac() - ''' - (a, b) = self.frac() - if a == 0: - return "none" - if a < b: - return "in_progress" - return "done" - - def __eq__(self, other): - ''' Two Progress objects are equal if they have identical values. - Implemented in terms of frac()''' - if not isinstance(other, Progress): - return False - (a, b) = self.frac() - (a2, b2) = other.frac() - return a == a2 and b == b2 - - def __ne__(self, other): - ''' The opposite of equal''' - return not self.__eq__(other) - - def __str__(self): - ''' Return a string representation of this string. - - subclassing note: implemented in terms of frac(). - ''' - (a, b) = self.frac() - return "{0}/{1}".format(a, b) - - @staticmethod - def add_counts(a, b): - '''Add two progress indicators, assuming that each represents items done: - (a / b) + (c / d) = (a + c) / (b + d). - If either is None, returns the other. - ''' - if a is None: - return b - if b is None: - return a - # get numerators + denominators - (n, d) = a.frac() - (n2, d2) = b.frac() - return Progress(n + n2, d + d2) - - @staticmethod - def to_js_status_str(progress): - ''' - Return the "status string" version of the passed Progress - object that should be passed to js. Use this function when - sending Progress objects to js to limit dependencies. - ''' - if progress is None: - return "NA" - return progress.ternary_str() - - @staticmethod - def to_js_detail_str(progress): - ''' - Return the "detail string" version of the passed Progress - object that should be passed to js. Use this function when - passing Progress objects to js to limit dependencies. - ''' - if progress is None: - return "NA" - return str(progress) diff --git a/common/lib/xmodule/tests/test_stringify.py b/common/lib/xmodule/tests/test_stringify.py deleted file mode 100644 index 62d7683886..0000000000 --- a/common/lib/xmodule/tests/test_stringify.py +++ /dev/null @@ -1,9 +0,0 @@ -from nose.tools import assert_equals -from lxml import etree -from stringify import stringify_children - -def test_stringify(): - html = '''Hi
    there Bruce!
    ''' - xml = etree.fromstring(html) - out = stringify_children(xml) - assert_equals(out, '''Hi
    there Bruce!
    ''') diff --git a/common/lib/xmodule/html_checker.py b/common/lib/xmodule/xmodule/html_checker.py similarity index 100% rename from common/lib/xmodule/html_checker.py rename to common/lib/xmodule/xmodule/html_checker.py diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index b2a5df9803..7c3456e5ad 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -5,11 +5,11 @@ import os import sys from lxml import etree -from xmodule.x_module import XModule -from xmodule.xml_module import XmlDescriptor -from xmodule.editing_module import EditingDescriptor -from stringify import stringify_children -from html_checker import check_html +from .x_module import XModule +from .xml_module import XmlDescriptor +from .editing_module import EditingDescriptor +from .stringify import stringify_children +from .html_checker import check_html log = logging.getLogger("mitx.courseware") diff --git a/common/lib/xmodule/stringify.py b/common/lib/xmodule/xmodule/stringify.py similarity index 100% rename from common/lib/xmodule/stringify.py rename to common/lib/xmodule/xmodule/stringify.py diff --git a/common/lib/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py similarity index 100% rename from common/lib/xmodule/tests/__init__.py rename to common/lib/xmodule/xmodule/tests/__init__.py diff --git a/common/lib/xmodule/tests/test_export.py b/common/lib/xmodule/xmodule/tests/test_export.py similarity index 100% rename from common/lib/xmodule/tests/test_export.py rename to common/lib/xmodule/xmodule/tests/test_export.py diff --git a/common/lib/xmodule/tests/test_files/choiceresponse_checkbox.xml b/common/lib/xmodule/xmodule/tests/test_files/choiceresponse_checkbox.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/choiceresponse_checkbox.xml rename to common/lib/xmodule/xmodule/tests/test_files/choiceresponse_checkbox.xml diff --git a/common/lib/xmodule/tests/test_files/choiceresponse_radio.xml b/common/lib/xmodule/xmodule/tests/test_files/choiceresponse_radio.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/choiceresponse_radio.xml rename to common/lib/xmodule/xmodule/tests/test_files/choiceresponse_radio.xml diff --git a/common/lib/xmodule/tests/test_files/coderesponse.xml b/common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/coderesponse.xml rename to common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml diff --git a/common/lib/xmodule/tests/test_files/formularesponse_with_hint.xml b/common/lib/xmodule/xmodule/tests/test_files/formularesponse_with_hint.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/formularesponse_with_hint.xml rename to common/lib/xmodule/xmodule/tests/test_files/formularesponse_with_hint.xml diff --git a/common/lib/xmodule/tests/test_files/imageresponse.xml b/common/lib/xmodule/xmodule/tests/test_files/imageresponse.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/imageresponse.xml rename to common/lib/xmodule/xmodule/tests/test_files/imageresponse.xml diff --git a/common/lib/xmodule/tests/test_files/multi_bare.xml b/common/lib/xmodule/xmodule/tests/test_files/multi_bare.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/multi_bare.xml rename to common/lib/xmodule/xmodule/tests/test_files/multi_bare.xml diff --git a/common/lib/xmodule/tests/test_files/multichoice.xml b/common/lib/xmodule/xmodule/tests/test_files/multichoice.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/multichoice.xml rename to common/lib/xmodule/xmodule/tests/test_files/multichoice.xml diff --git a/common/lib/xmodule/tests/test_files/optionresponse.xml b/common/lib/xmodule/xmodule/tests/test_files/optionresponse.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/optionresponse.xml rename to common/lib/xmodule/xmodule/tests/test_files/optionresponse.xml diff --git a/common/lib/xmodule/tests/test_files/stringresponse_with_hint.xml b/common/lib/xmodule/xmodule/tests/test_files/stringresponse_with_hint.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/stringresponse_with_hint.xml rename to common/lib/xmodule/xmodule/tests/test_files/stringresponse_with_hint.xml diff --git a/common/lib/xmodule/tests/test_files/symbolicresponse.xml b/common/lib/xmodule/xmodule/tests/test_files/symbolicresponse.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/symbolicresponse.xml rename to common/lib/xmodule/xmodule/tests/test_files/symbolicresponse.xml diff --git a/common/lib/xmodule/tests/test_files/truefalse.xml b/common/lib/xmodule/xmodule/tests/test_files/truefalse.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/truefalse.xml rename to common/lib/xmodule/xmodule/tests/test_files/truefalse.xml diff --git a/common/lib/xmodule/tests/test_import.py b/common/lib/xmodule/xmodule/tests/test_import.py similarity index 100% rename from common/lib/xmodule/tests/test_import.py rename to common/lib/xmodule/xmodule/tests/test_import.py diff --git a/common/lib/xmodule/xmodule/tests/test_stringify.py b/common/lib/xmodule/xmodule/tests/test_stringify.py new file mode 100644 index 0000000000..1c6ee855f3 --- /dev/null +++ b/common/lib/xmodule/xmodule/tests/test_stringify.py @@ -0,0 +1,10 @@ +from nose.tools import assert_equals +from lxml import etree +from xmodule.stringify import stringify_children + +def test_stringify(): + text = 'Hi
    there Bruce!
    ' + html = '''{0}'''.format(text) + xml = etree.fromstring(html) + out = stringify_children(xml) + assert_equals(out, text) From 31b8270cfb1442be97ada91b157f67690125a2a2 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Thu, 2 Aug 2012 10:14:35 -0400 Subject: [PATCH 080/124] remove too-clever lazy dictionary --- common/lib/xmodule/lazy_dict.py | 58 --------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 common/lib/xmodule/lazy_dict.py diff --git a/common/lib/xmodule/lazy_dict.py b/common/lib/xmodule/lazy_dict.py deleted file mode 100644 index fa614843ec..0000000000 --- a/common/lib/xmodule/lazy_dict.py +++ /dev/null @@ -1,58 +0,0 @@ -from collections import MutableMapping - -class LazyLoadingDict(MutableMapping): - """ - A dictionary object that lazily loads its contents from a provided - function on reads (of members that haven't already been set). - """ - - def __init__(self, loader): - ''' - On the first read from this dictionary, it will call loader() to - populate its contents. loader() must return something dict-like. Any - elements set before the first read will be preserved. - ''' - self._contents = {} - self._loaded = False - self._loader = loader - self._deleted = set() - - def __getitem__(self, name): - if not (self._loaded or name in self._contents or name in self._deleted): - self.load() - - return self._contents[name] - - def __setitem__(self, name, value): - self._contents[name] = value - self._deleted.discard(name) - - def __delitem__(self, name): - del self._contents[name] - self._deleted.add(name) - - def __contains__(self, name): - self.load() - return name in self._contents - - def __len__(self): - self.load() - return len(self._contents) - - def __iter__(self): - self.load() - return iter(self._contents) - - def __repr__(self): - self.load() - return repr(self._contents) - - def load(self): - if self._loaded: - return - - loaded_contents = self._loader() - loaded_contents.update(self._contents) - self._contents = loaded_contents - self._loaded = True - From fd796478d83fbbd9571710edc3b21eb7b4b96d5a Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Thu, 2 Aug 2012 11:33:04 -0400 Subject: [PATCH 081/124] add a handy supertrace script --- common/lib/supertrace.py | 52 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 common/lib/supertrace.py diff --git a/common/lib/supertrace.py b/common/lib/supertrace.py new file mode 100644 index 0000000000..e17cd7a8ba --- /dev/null +++ b/common/lib/supertrace.py @@ -0,0 +1,52 @@ +""" +A handy util to print a django-debug-screen-like stack trace with +values of local variables. +""" + +import sys, traceback +from django.utils.encoding import smart_unicode + + +def supertrace(max_len=160): + """ + Print the usual traceback information, followed by a listing of all the + local variables in each frame. Should be called from an exception handler. + + if max_len is not None, will print up to max_len chars for each local variable. + + (cite: modified from somewhere on stackoverflow) + """ + tb = sys.exc_info()[2] + while True: + if not tb.tb_next: + break + tb = tb.tb_next + stack = [] + frame = tb.tb_frame + while frame: + stack.append(f) + frame = frame.f_back + stack.reverse() + # First print the regular traceback + traceback.print_exc() + + print "Locals by frame, innermost last" + for frame in stack: + print + print "Frame %s in %s at line %s" % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + for key, value in frame.f_locals.items(): + print ("\t%20s = " % smart_unicode(key, errors='ignore')), + # We have to be careful not to cause a new error in our error + # printer! Calling str() on an unknown object could cause an + # error. + try: + s = smart_unicode(value, errors='ignore') + if max_len is not None: + s = s[:max_len] + print s + except: + print "" + + From 64346d727b3c142ad74de87e2fa37c12667e236f Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Thu, 2 Aug 2012 11:33:53 -0400 Subject: [PATCH 082/124] todos and comments --- common/lib/xmodule/xmodule/editing_module.py | 7 ++++++- lms/djangoapps/courseware/views.py | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/common/lib/xmodule/xmodule/editing_module.py b/common/lib/xmodule/xmodule/editing_module.py index 4188165a24..67a4d66dad 100644 --- a/common/lib/xmodule/xmodule/editing_module.py +++ b/common/lib/xmodule/xmodule/editing_module.py @@ -20,5 +20,10 @@ class EditingDescriptor(MakoModuleDescriptor): def get_context(self): return { 'module': self, - 'data': self.definition['data'], + 'data': self.definition.get('data', ''), + # TODO (vshnayder): allow children and metadata to be edited. + #'children' : self.definition.get('children, ''), + + # TODO: show both own metadata and inherited? + #'metadata' : self.own_metadata, } diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index a049638d1b..2b55b48caf 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -117,7 +117,7 @@ def profile(request, course_id, student_id=None): student_module_cache = StudentModuleCache(request.user, course) course_module, _, _, _ = get_module(request.user, request, course.location, student_module_cache) - + context = {'name': user_info.name, 'username': student.username, 'location': user_info.location, @@ -243,6 +243,7 @@ def index(request, course_id, chapter=None, section=None, return result + @ensure_csrf_cookie def jump_to(request, location): ''' @@ -269,7 +270,7 @@ def jump_to(request, location): except NoPathToItem: raise Http404("This location is not in any class: {0}".format(location)) - + # Rely on index to do all error handling return index(request, course_id, chapter, section, position) @ensure_csrf_cookie From 3eff9ffecd2382ba0bad9b877376388daffa7a8e Mon Sep 17 00:00:00 2001 From: ichuang Date: Thu, 2 Aug 2012 13:28:52 -0400 Subject: [PATCH 083/124] match external_domain as well when retrieving ExternalAuthMap objects --- common/djangoapps/external_auth/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py index 740b4ed1ac..d00a0a7182 100644 --- a/common/djangoapps/external_auth/views.py +++ b/common/djangoapps/external_auth/views.py @@ -84,7 +84,9 @@ def edXauth_external_login_or_signup(request, external_id, external_domain, cred retfun=None): # see if we have a map from this external_id to an edX username try: - eamap = ExternalAuthMap.objects.get(external_id=external_id) + eamap = ExternalAuthMap.objects.get(external_id = external_id, + external_domain = external_domain, + ) log.debug('Found eamap=%s' % eamap) except ExternalAuthMap.DoesNotExist: # go render form for creating edX user From a7103ff8932ddfdcd10e844aa71fd3901690de47 Mon Sep 17 00:00:00 2001 From: ichuang Date: Thu, 2 Aug 2012 13:39:12 -0400 Subject: [PATCH 084/124] switch to PascalCase, remove unnecessary assignment --- common/djangoapps/student/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index 7937d67980..35ce225011 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -280,14 +280,14 @@ def create_account(request, post_override=None): # if doing signup for an external authorization, then get email, password, name from the eamap # don't use the ones from the form, since the user could have hacked those - doExternalAuth = 'ExternalAuthMap' in request.session - if doExternalAuth: + DoExternalAuth = 'ExternalAuthMap' in request.session + if DoExternalAuth: eamap = request.session['ExternalAuthMap'] email = eamap.external_email name = eamap.external_name password = eamap.internal_password post_vars = dict(post_vars.items()) - post_vars.update(dict(email=email, name=name, password=password, username=post_vars['username'])) + post_vars.update(dict(email=email, name=name, password=password)) log.debug('extauth test: post_vars = %s' % post_vars) # Confirm we have a properly formed request @@ -411,7 +411,7 @@ def create_account(request, post_override=None): try_change_enrollment(request) - if doExternalAuth: + if DoExternalAuth: eamap.user = login_user eamap.dtsignup = datetime.datetime.now() eamap.save() From 26ae88faac8c620fa14d0cabfee7c03ac5938169 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Thu, 2 Aug 2012 13:41:54 -0400 Subject: [PATCH 085/124] Factor out get_item_error() into a new ModuleStoreBase class * may be temporary if we move errors into the items themselves. --- .../xmodule/xmodule/modulestore/__init__.py | 44 +++++++++++++++++-- .../lib/xmodule/xmodule/modulestore/mongo.py | 9 ++-- common/lib/xmodule/xmodule/modulestore/xml.py | 36 +++++---------- 3 files changed, 59 insertions(+), 30 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 7241179d8e..e59e4bd68e 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -3,10 +3,13 @@ This module provides an abstraction for working with XModuleDescriptors that are stored in a database an accessible using their Location as an identifier """ -import re -from collections import namedtuple -from .exceptions import InvalidLocationError, InsufficientSpecificationError import logging +import re + +from collections import namedtuple + +from .exceptions import InvalidLocationError, InsufficientSpecificationError +from xmodule.errortracker import ErrorLog, make_error_tracker log = logging.getLogger('mitx.' + 'modulestore') @@ -290,3 +293,38 @@ class ModuleStore(object): ''' raise NotImplementedError + +class ModuleStoreBase(ModuleStore): + ''' + Implement interface functionality that can be shared. + ''' + def __init__(self): + ''' + Set up the error-tracking logic. + ''' + self._location_errors = {} # location -> ErrorLog + + def _get_errorlog(self, location): + """ + If we already have an errorlog for this location, return it. Otherwise, + create one. + """ + location = Location(location) + if location not in self._location_errors: + self._location_errors[location] = make_error_tracker() + return self._location_errors[location] + + def get_item_errors(self, location): + """ + Return list of errors for this location, if any. Raise the same + errors as get_item if location isn't present. + + NOTE: For now, the only items that track errors are CourseDescriptors in + the xml datastore. This will return an empty list for all other items + and datastores. + """ + # check that item is present and raise the promised exceptions if needed + self.get_item(location) + + errorlog = self._get_errorlog(location) + return errorlog.errors diff --git a/common/lib/xmodule/xmodule/modulestore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py index 76769b25b0..1cec6c7f87 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -11,7 +11,7 @@ from xmodule.x_module import XModuleDescriptor from xmodule.mako_module import MakoDescriptorSystem from mitxmako.shortcuts import render_to_string -from . import ModuleStore, Location +from . import ModuleStoreBase, Location from .exceptions import (ItemNotFoundError, NoPathToItem, DuplicateItemError) @@ -38,7 +38,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem): resources_fs: a filesystem, as per MakoDescriptorSystem - error_tracker: + error_tracker: a function that logs errors for later display to users render_template: a function for rendering templates, as per MakoDescriptorSystem @@ -73,7 +73,7 @@ def location_to_query(location): return query -class MongoModuleStore(ModuleStore): +class MongoModuleStore(ModuleStoreBase): """ A Mongodb backed ModuleStore """ @@ -81,6 +81,9 @@ class MongoModuleStore(ModuleStore): # TODO (cpennington): Enable non-filesystem filestores def __init__(self, host, db, collection, fs_root, port=27017, default_class=None, error_tracker=null_error_tracker): + + ModuleStoreBase.__init__(self) + self.collection = pymongo.connection.Connection( host=host, port=port diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index d6540023e8..0567e4e7a7 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -12,7 +12,7 @@ from xmodule.course_module import CourseDescriptor from xmodule.mako_module import MakoDescriptorSystem from cStringIO import StringIO -from . import ModuleStore, Location +from . import ModuleStoreBase, Location from .exceptions import ItemNotFoundError etree.set_default_parser( @@ -98,7 +98,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): error_tracker, process_xml, **kwargs) -class XMLModuleStore(ModuleStore): +class XMLModuleStore(ModuleStoreBase): """ An XML backed ModuleStore """ @@ -118,13 +118,12 @@ class XMLModuleStore(ModuleStore): course_dirs: If specified, the list of course_dirs to load. Otherwise, load all course dirs """ + ModuleStoreBase.__init__(self) self.eager = eager self.data_dir = path(data_dir) self.modules = {} # location -> XModuleDescriptor self.courses = {} # course_dir -> XModuleDescriptor for the course - self.location_errors = {} # location -> ErrorLog - if default_class is None: self.default_class = None @@ -148,12 +147,14 @@ class XMLModuleStore(ModuleStore): for course_dir in course_dirs: try: - # make a tracker, then stick in the right place once the course loads - # and we know its location + # Special-case code here, since we don't have a location for the + # course before it loads. + # So, make a tracker to track load-time errors, then put in the right + # place after the course loads and we have its location errorlog = make_error_tracker() course_descriptor = self.load_course(course_dir, errorlog.tracker) self.courses[course_dir] = course_descriptor - self.location_errors[course_descriptor.location] = errorlog + self._location_errors[course_descriptor.location] = errorlog except: msg = "Failed to load course '%s'" % course_dir log.exception(msg) @@ -221,23 +222,6 @@ class XMLModuleStore(ModuleStore): raise ItemNotFoundError(location) - def get_item_errors(self, location): - """ - Return list of errors for this location, if any. Raise the same - errors as get_item if location isn't present. - - NOTE: This only actually works for courses in the xml datastore-- - will return an empty list for all other modules. - """ - location = Location(location) - # check that item is present - self.get_item(location) - - # now look up errors - if location in self.location_errors: - return self.location_errors[location].errors - return [] - def get_courses(self, depth=0): """ Returns a list of course descriptors. If there were errors on loading, @@ -245,9 +229,11 @@ class XMLModuleStore(ModuleStore): """ return self.courses.values() + def create_item(self, location): raise NotImplementedError("XMLModuleStores are read-only") + def update_item(self, location, data): """ Set the data in the item specified by the location to @@ -258,6 +244,7 @@ class XMLModuleStore(ModuleStore): """ raise NotImplementedError("XMLModuleStores are read-only") + def update_children(self, location, children): """ Set the children for the item specified by the location to @@ -268,6 +255,7 @@ class XMLModuleStore(ModuleStore): """ raise NotImplementedError("XMLModuleStores are read-only") + def update_metadata(self, location, metadata): """ Set the metadata for the item specified by the location to From 46bc7bc499fb63005d5225a465470c9d7481aa3d Mon Sep 17 00:00:00 2001 From: ichuang Date: Thu, 2 Aug 2012 13:44:37 -0400 Subject: [PATCH 086/124] remove unnecessary hidden fields --- lms/templates/signup_modal.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lms/templates/signup_modal.html b/lms/templates/signup_modal.html index 346027418d..1510eb407b 100644 --- a/lms/templates/signup_modal.html +++ b/lms/templates/signup_modal.html @@ -30,15 +30,9 @@ % else:

    Welcome ${extauth_email}


    - -

    Enter a public username:

    - - - - % endif From 10b2c212b6d9159acf430f8d65738471292eea5c Mon Sep 17 00:00:00 2001 From: ichuang Date: Thu, 2 Aug 2012 13:52:25 -0400 Subject: [PATCH 087/124] fix javascript for signup modal .click() in index.html for safari --- lms/templates/index.html | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lms/templates/index.html b/lms/templates/index.html index dcc18d4de1..4255251604 100644 --- a/lms/templates/index.html +++ b/lms/templates/index.html @@ -148,7 +148,26 @@ % if show_signup_immediately is not UNDEFINED: From dff380e8076a49a82d10bda267d89e9d7259133d Mon Sep 17 00:00:00 2001 From: ichuang Date: Thu, 2 Aug 2012 14:19:56 -0400 Subject: [PATCH 088/124] cleanup lms urls (remove cruft from debugging openid) --- lms/urls.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lms/urls.py b/lms/urls.py index 8c36857ee3..35a71920a5 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -166,10 +166,6 @@ if settings.MITX_FEATURES.get('AUTH_USE_OPENID'): url(r'^openid/complete/$', 'external_auth.views.edXauth_openid_login_complete', name='openid-complete'), url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'), ) - urlpatterns += ( - url(r'^extauth/$', 'external_auth.views.edXauth_signup', name='extauth-signup'), - ) - # urlpatterns += (url(r'^openid/', include('django_openid_auth.urls')),) urlpatterns = patterns(*urlpatterns) From 3b6f33f5a79c76d4bed9cad90b663bcf8deb7342 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Thu, 2 Aug 2012 14:26:24 -0400 Subject: [PATCH 089/124] Fix github_sync tests * CMS no longer syncs, it just imports when it gets a commit note from github --- .../github_sync/tests/test_views.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cms/djangoapps/github_sync/tests/test_views.py b/cms/djangoapps/github_sync/tests/test_views.py index 212d707340..37030d6a1b 100644 --- a/cms/djangoapps/github_sync/tests/test_views.py +++ b/cms/djangoapps/github_sync/tests/test_views.py @@ -11,33 +11,33 @@ class PostReceiveTestCase(TestCase): def setUp(self): self.client = Client() - @patch('github_sync.views.sync_with_github') - def test_non_branch(self, sync_with_github): + @patch('github_sync.views.import_from_github') + def test_non_branch(self, import_from_github): self.client.post('/github_service_hook', {'payload': json.dumps({ 'ref': 'refs/tags/foo'}) }) - self.assertFalse(sync_with_github.called) + self.assertFalse(import_from_github.called) - @patch('github_sync.views.sync_with_github') - def test_non_watched_repo(self, sync_with_github): + @patch('github_sync.views.import_from_github') + def test_non_watched_repo(self, import_from_github): self.client.post('/github_service_hook', {'payload': json.dumps({ 'ref': 'refs/heads/branch', 'repository': {'name': 'bad_repo'}}) }) - self.assertFalse(sync_with_github.called) + self.assertFalse(import_from_github.called) - @patch('github_sync.views.sync_with_github') - def test_non_tracked_branch(self, sync_with_github): + @patch('github_sync.views.import_from_github') + def test_non_tracked_branch(self, import_from_github): self.client.post('/github_service_hook', {'payload': json.dumps({ 'ref': 'refs/heads/non_branch', 'repository': {'name': 'repo'}}) }) - self.assertFalse(sync_with_github.called) + self.assertFalse(import_from_github.called) - @patch('github_sync.views.sync_with_github') - def test_tracked_branch(self, sync_with_github): + @patch('github_sync.views.import_from_github') + def test_tracked_branch(self, import_from_github): self.client.post('/github_service_hook', {'payload': json.dumps({ 'ref': 'refs/heads/branch', 'repository': {'name': 'repo'}}) }) - sync_with_github.assert_called_with(load_repo_settings('repo')) + import_from_github.assert_called_with(load_repo_settings('repo')) From 987b9c11a94372f6d4c888755a1ce5ff5a036e30 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Thu, 2 Aug 2012 14:05:42 -0400 Subject: [PATCH 090/124] Use url_name for chapters and sections in lms views * got rid of the hackish conversions between ' ' and '_' * use url_name and display_name where appropriate * update templates to match. --- common/lib/xmodule/xmodule/x_module.py | 2 ++ lms/djangoapps/courseware/grades.py | 33 ++++++++++++++-------- lms/djangoapps/courseware/module_render.py | 28 ++++++++++-------- lms/djangoapps/courseware/views.py | 29 ++++--------------- lms/templates/accordion.html | 6 ++-- lms/templates/profile.html | 28 +++++++++--------- 6 files changed, 62 insertions(+), 64 deletions(-) diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index ac6b5db5a4..1d16849d67 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -339,6 +339,8 @@ class XModuleDescriptor(Plugin, HTMLSnippet): module display_name: The name to use for displaying this module to the user + url_name: The name to use for this module in urls and other places + where a unique name is needed. format: The format of this module ('Homework', 'Lab', etc) graded (bool): Whether this module is should be graded or not start (string): The date for which this module will be available diff --git a/lms/djangoapps/courseware/grades.py b/lms/djangoapps/courseware/grades.py index 5a817e3d6c..717cbde140 100644 --- a/lms/djangoapps/courseware/grades.py +++ b/lms/djangoapps/courseware/grades.py @@ -12,18 +12,23 @@ _log = logging.getLogger("mitx.courseware") def grade_sheet(student, course, grader, student_module_cache): """ - This pulls a summary of all problems in the course. It returns a dictionary with two datastructures: + This pulls a summary of all problems in the course. It returns a dictionary + with two datastructures: - - courseware_summary is a summary of all sections with problems in the course. It is organized as an array of chapters, - each containing an array of sections, each containing an array of scores. This contains information for graded and ungraded - problems, and is good for displaying a course summary with due dates, etc. + - courseware_summary is a summary of all sections with problems in the + course. It is organized as an array of chapters, each containing an array of + sections, each containing an array of scores. This contains information for + graded and ungraded problems, and is good for displaying a course summary + with due dates, etc. - - grade_summary is the output from the course grader. More information on the format is in the docstring for CourseGrader. + - grade_summary is the output from the course grader. More information on + the format is in the docstring for CourseGrader. Arguments: student: A User object for the student to grade course: An XModule containing the course to grade - student_module_cache: A StudentModuleCache initialized with all instance_modules for the student + student_module_cache: A StudentModuleCache initialized with all + instance_modules for the student """ totaled_scores = {} chapters = [] @@ -51,12 +56,16 @@ def grade_sheet(student, course, grader, student_module_cache): correct = total if not total > 0: - #We simply cannot grade a problem that is 12/0, because we might need it as a percentage + #We simply cannot grade a problem that is 12/0, because we + #might need it as a percentage graded = False - scores.append(Score(correct, total, graded, module.metadata.get('display_name'))) + scores.append(Score(correct, total, graded, + module.metadata.get('display_name'))) + + section_total, graded_total = graders.aggregate_scores( + scores, s.metadata.get('display_name')) - section_total, graded_total = graders.aggregate_scores(scores, s.metadata.get('display_name')) #Add the graded total to totaled_scores format = s.metadata.get('format', "") if format and graded_total.possible > 0: @@ -65,7 +74,8 @@ def grade_sheet(student, course, grader, student_module_cache): totaled_scores[format] = format_scores sections.append({ - 'section': s.metadata.get('display_name'), + 'display_name': s.metadata.get('display_name'), + 'url_name': s.metadata.get('url_name'), 'scores': scores, 'section_total': section_total, 'format': format, @@ -74,7 +84,8 @@ def grade_sheet(student, course, grader, student_module_cache): }) chapters.append({'course': course.metadata.get('display_name'), - 'chapter': c.metadata.get('display_name'), + 'display_name': c.metadata.get('display_name'), + 'url_name': c.metadata.get('url_name'), 'sections': sections}) grade_summary = grader.grade(totaled_scores) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 91d4efa651..4699ed50a4 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -35,10 +35,12 @@ def toc_for_course(user, request, course, active_chapter, active_section): Create a table of contents from the module store Return format: - [ {'name': name, 'sections': SECTIONS, 'active': bool}, ... ] + [ {'display_name': name, 'url_name': url_name, + 'sections': SECTIONS, 'active': bool}, ... ] where SECTIONS is a list - [ {'name': name, 'format': format, 'due': due, 'active' : bool}, ...] + [ {'display_name': name, 'url_name': url_name, + 'format': format, 'due': due, 'active' : bool}, ...] active is set for the section and chapter corresponding to the passed parameters. Everything else comes from the xml, or defaults to "". @@ -59,12 +61,14 @@ def toc_for_course(user, request, course, active_chapter, active_section): hide_from_toc = section.metadata.get('hide_from_toc', 'false').lower() == 'true' if not hide_from_toc: - sections.append({'name': section.metadata.get('display_name'), + sections.append({'display_name': section.metadata.get('display_name'), + 'url_name': section.metadata.get('url_name'), 'format': section.metadata.get('format', ''), 'due': section.metadata.get('due', ''), 'active': active}) - chapters.append({'name': chapter.metadata.get('display_name'), + chapters.append({'display_name': chapter.metadata.get('display_name'), + 'url_name': chapter.metadata.get('url_name'), 'sections': sections, 'active': chapter.metadata.get('display_name') == active_chapter}) return chapters @@ -76,8 +80,8 @@ def get_section(course_module, chapter, section): or None if this doesn't specify a valid section course: Course url - chapter: Chapter name - section: Section name + chapter: Chapter url_name + section: Section url_name """ if course_module is None: @@ -85,7 +89,7 @@ def get_section(course_module, chapter, section): chapter_module = None for _chapter in course_module.get_children(): - if _chapter.metadata.get('display_name') == chapter: + if _chapter.metadata.get('url_name') == chapter: chapter_module = _chapter break @@ -94,7 +98,7 @@ def get_section(course_module, chapter, section): section_module = None for _section in chapter_module.get_children(): - if _section.metadata.get('display_name') == section: + if _section.metadata.get('url_name') == section: section_module = _section break @@ -141,12 +145,12 @@ def get_module(user, request, location, student_module_cache, position=None): # Setup system context for module instance ajax_url = settings.MITX_ROOT_URL + '/modx/' + descriptor.location.url() + '/' - # Fully qualified callback URL for external queueing system - xqueue_callback_url = (request.build_absolute_uri('/') + settings.MITX_ROOT_URL + - 'xqueue/' + str(user.id) + '/' + descriptor.location.url() + '/' + + # Fully qualified callback URL for external queueing system + xqueue_callback_url = (request.build_absolute_uri('/') + settings.MITX_ROOT_URL + + 'xqueue/' + str(user.id) + '/' + descriptor.location.url() + '/' + 'score_update') - # Default queuename is course-specific and is derived from the course that + # Default queuename is course-specific and is derived from the course that # contains the current module. # TODO: Queuename should be derived from 'course_settings.json' of each course xqueue_default_queuename = descriptor.location.org + '-' + descriptor.location.course diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 2b55b48caf..18b710e108 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -54,11 +54,6 @@ def user_groups(user): return group_names -def format_url_params(params): - return [urllib.quote(string.replace(' ', '_')) - if string is not None else None - for string in params] - @ensure_csrf_cookie @cache_if_anonymous @@ -124,7 +119,6 @@ def profile(request, course_id, student_id=None): 'language': user_info.language, 'email': student.email, 'course': course, - 'format_url_params': format_url_params, 'csrf': csrf(request)['csrf_token'] } context.update(grades.grade_sheet(student, course_module, course.grader, student_module_cache)) @@ -138,9 +132,9 @@ def render_accordion(request, course, chapter, section): If chapter and section are '' or None, renders a default accordion. - Returns (initialization_javascript, content)''' + Returns the html string''' - # TODO (cpennington): do the right thing with courses + # grab the table of contents toc = toc_for_course(request.user, request, course, chapter, section) active_chapter = 1 @@ -152,7 +146,6 @@ def render_accordion(request, course, chapter, section): ('toc', toc), ('course_name', course.title), ('course_id', course.id), - ('format_url_params', format_url_params), ('csrf', csrf(request)['csrf_token'])] + template_imports.items()) return render_to_string('accordion.html', context) @@ -169,9 +162,9 @@ def index(request, course_id, chapter=None, section=None, Arguments: - request : HTTP request - - course : coursename (str) - - chapter : chapter name (str) - - section : section name (str) + - course_id : course id (str: ORG/course/URL_NAME) + - chapter : chapter url_name (str) + - section : section url_name (str) - position : position in module, eg of module (str) Returns: @@ -180,16 +173,6 @@ def index(request, course_id, chapter=None, section=None, ''' course = check_course(course_id) - def clean(s): - ''' Fixes URLs -- we convert spaces to _ in URLs to prevent - funny encoding characters and keep the URLs readable. This undoes - that transformation. - ''' - return s.replace('_', ' ') if s is not None else None - - chapter = clean(chapter) - section = clean(section) - try: context = { 'csrf': csrf(request)['csrf_token'], @@ -202,8 +185,6 @@ def index(request, course_id, chapter=None, section=None, look_for_module = chapter is not None and section is not None if look_for_module: - # TODO (cpennington): Pass the right course in here - section_descriptor = get_section(course, chapter, section) if section_descriptor is not None: student_module_cache = StudentModuleCache(request.user, diff --git a/lms/templates/accordion.html b/lms/templates/accordion.html index defb424a29..353b83db70 100644 --- a/lms/templates/accordion.html +++ b/lms/templates/accordion.html @@ -1,13 +1,13 @@ <%! from django.core.urlresolvers import reverse %> <%def name="make_chapter(chapter)"> -

    ${chapter['name']}

    +

    ${chapter['display_name']}

      % for section in chapter['sections']: - -

      ${section['name']} + +

      ${section['display_name']} ${section['format']} ${"due " + section['due'] if 'due' in section and section['due'] != '' else ''} diff --git a/lms/templates/profile.html b/lms/templates/profile.html index 6cda85fb03..8107bb1923 100644 --- a/lms/templates/profile.html +++ b/lms/templates/profile.html @@ -72,7 +72,7 @@ $(function() { var new_email = $('#new_email_field').val(); var new_password = $('#new_email_password').val(); - postJSON('/change_email',{"new_email":new_email, + postJSON('/change_email',{"new_email":new_email, "password":new_password}, function(data){ if(data.success){ @@ -81,7 +81,7 @@ $(function() { $("#change_email_error").html(data.error); } }); - log_event("profile", {"type":"email_change_request", + log_event("profile", {"type":"email_change_request", "old_email":"${email}", "new_email":new_email}); return false; @@ -91,7 +91,7 @@ $(function() { var new_name = $('#new_name_field').val(); var rationale = $('#name_rationale_field').val(); - postJSON('/change_name',{"new_name":new_name, + postJSON('/change_name',{"new_name":new_name, "rationale":rationale}, function(data){ if(data.success){ @@ -100,7 +100,7 @@ $(function() { $("#change_name_error").html(data.error); } }); - log_event("profile", {"type":"name_change_request", + log_event("profile", {"type":"name_change_request", "new_name":new_name, "rationale":rationale}); return false; @@ -125,9 +125,9 @@ $(function() {

        %for chapter in courseware_summary: - %if not chapter['chapter'] == "hidden": + %if not chapter['display_name'] == "hidden":
      1. -

        ${ chapter['chapter'] }

        +

        ${ chapter['display_name'] }

          %for section in chapter['sections']: @@ -137,14 +137,14 @@ $(function() { total = section['section_total'].possible percentageString = "{0:.0%}".format( float(earned)/total) if earned > 0 and total > 0 else "" %> - -

          - ${ section['section'] } ${"({0:.3n}/{1:.3n}) {2}".format( float(earned), float(total), percentageString )}

          + +

          + ${ section['display_name'] } ${"({0:.3n}/{1:.3n}) {2}".format( float(earned), float(total), percentageString )}

          ${section['format']} %if 'due' in section and section['due']!="": due ${section['due']} %endif - + %if len(section['scores']) > 0:
            ${ "Problem Scores: " if section['graded'] else "Practice Scores: "} @@ -153,7 +153,7 @@ $(function() { %endfor
          %endif - + %endfor
        @@ -181,7 +181,7 @@ $(function() {
      2. - Forum name: ${username} + Forum name: ${username}
      3. @@ -215,7 +215,7 @@ $(function() {
        -

        To uphold the credibility of edX certificates, name changes must go through an approval process. A member of the course staff will review your request, and if approved, update your information. Please allow up to a week for your request to be processed. Thank you.

        +

        To uphold the credibility of edX certificates, name changes must go through an approval process. A member of the course staff will review your request, and if approved, update your information. Please allow up to a week for your request to be processed. Thank you.

        • @@ -234,7 +234,7 @@ $(function() {
          -

          Change e-mail

          +

          Change e-mail

          From 2f911fd3b22be91d8f5a542e52f9ff1303d619dc Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Thu, 2 Aug 2012 14:08:28 -0400 Subject: [PATCH 091/124] add error-handling TODOs to capa_problem --- common/lib/capa/capa/capa_problem.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 93b6a085c1..cb3c19487d 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -288,20 +288,30 @@ class LoncapaProblem(object): try: ifp = self.system.filestore.open(file) # open using ModuleSystem OSFS filestore except Exception as err: - log.error('Error %s in problem xml include: %s' % (err, etree.tostring(inc, pretty_print=True))) - log.error('Cannot find file %s in %s' % (file, self.system.filestore)) - if not self.system.get('DEBUG'): # if debugging, don't fail - just log error + log.error('Error %s in problem xml include: %s' % ( + err, etree.tostring(inc, pretty_print=True))) + log.error('Cannot find file %s in %s' % ( + file, self.system.filestore)) + # if debugging, don't fail - just log error + # TODO (vshnayder): need real error handling, display to users + if not self.system.get('DEBUG'): raise - else: continue + else: + continue try: incxml = etree.XML(ifp.read()) # read in and convert to XML except Exception as err: - log.error('Error %s in problem xml include: %s' % (err, etree.tostring(inc, pretty_print=True))) + log.error('Error %s in problem xml include: %s' % ( + err, etree.tostring(inc, pretty_print=True))) log.error('Cannot parse XML in %s' % (file)) - if not self.system.get('DEBUG'): # if debugging, don't fail - just log error + # if debugging, don't fail - just log error + # TODO (vshnayder): same as above + if not self.system.get('DEBUG'): raise - else: continue - parent = inc.getparent() # insert new XML into tree in place of inlcude + else: + continue + # insert new XML into tree in place of inlcude + parent = inc.getparent() parent.insert(parent.index(inc), incxml) parent.remove(inc) log.debug('Included %s into %s' % (file, self.problem_id)) From b46172da9b00d224a327da64b2168e648b4a0717 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Fri, 3 Aug 2012 11:36:54 -0400 Subject: [PATCH 092/124] Rename module.name and descriptor.name to url_name * update templates and code references * also a display_name property that defaults to cleaned url_name --- cms/djangoapps/contentstore/views.py | 2 +- cms/templates/unit.html | 2 +- cms/templates/widgets/navigation.html | 4 ++-- cms/templates/widgets/sequence-edit.html | 2 +- common/lib/xmodule/xmodule/html_module.py | 4 ++-- .../lib/xmodule/xmodule/modulestore/search.py | 4 ++-- common/lib/xmodule/xmodule/video_module.py | 9 ++++---- common/lib/xmodule/xmodule/x_module.py | 22 +++++++++++++++++-- common/lib/xmodule/xmodule/xml_module.py | 6 ++--- lms/djangoapps/courseware/courses.py | 2 +- lms/templates/video.html | 2 +- 11 files changed, 39 insertions(+), 20 deletions(-) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 0447254cfb..0305795e52 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -108,7 +108,7 @@ def edit_item(request): 'contents': item.get_html(), 'js_module': item.js_module_name, 'category': item.category, - 'name': item.name, + 'url_name': item.url_name, 'previews': get_module_previews(request, item), }) diff --git a/cms/templates/unit.html b/cms/templates/unit.html index 6aa780d42a..828f95ed47 100644 --- a/cms/templates/unit.html +++ b/cms/templates/unit.html @@ -1,7 +1,7 @@
          -

          ${name}

          +

          ${url_name}

          ${category}

          diff --git a/cms/templates/widgets/navigation.html b/cms/templates/widgets/navigation.html index 9f9b37d571..ce18e867bd 100644 --- a/cms/templates/widgets/navigation.html +++ b/cms/templates/widgets/navigation.html @@ -41,7 +41,7 @@ % for week in weeks:
        • -

          ${week.name}

          +

          ${week.url_name}

            % if 'goals' in week.metadata: % for goal in week.metadata['goals']: @@ -60,7 +60,7 @@ data-type="${module.js_module_name}" data-preview-type="${module.module_class.js_module_name}"> - ${module.name} + ${module.url_name} handle % endfor diff --git a/cms/templates/widgets/sequence-edit.html b/cms/templates/widgets/sequence-edit.html index f7108e366e..c623eb4ec2 100644 --- a/cms/templates/widgets/sequence-edit.html +++ b/cms/templates/widgets/sequence-edit.html @@ -39,7 +39,7 @@ ${child.name} + data-preview-type="${child.module_class.js_module_name}">${child.url_name} handle %endfor diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 7c3456e5ad..7a09004e33 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -121,13 +121,13 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): # Not proper format. Write html to file, return an empty tag filepath = u'{category}/{name}.html'.format(category=self.category, - name=self.name) + name=self.url_name) resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) with resource_fs.open(filepath, 'w') as file: file.write(self.definition['data']) elt = etree.Element('html') - elt.set("filename", self.name) + elt.set("filename", self.url_name) return elt diff --git a/common/lib/xmodule/xmodule/modulestore/search.py b/common/lib/xmodule/xmodule/modulestore/search.py index a383b3f8ec..b3eae64fcc 100644 --- a/common/lib/xmodule/xmodule/modulestore/search.py +++ b/common/lib/xmodule/xmodule/modulestore/search.py @@ -87,8 +87,8 @@ def path_to_location(modulestore, location, course_name=None): n = len(path) course_id = CourseDescriptor.location_to_id(path[0]) - chapter = path[1].name if n > 1 else None - section = path[2].name if n > 2 else None + chapter = path[1].url_name if n > 1 else None + section = path[2].url_name if n > 2 else None # TODO (vshnayder): not handling position at all yet... position = None diff --git a/common/lib/xmodule/xmodule/video_module.py b/common/lib/xmodule/xmodule/video_module.py index da10e4bc91..fb68ba982b 100644 --- a/common/lib/xmodule/xmodule/video_module.py +++ b/common/lib/xmodule/xmodule/video_module.py @@ -23,11 +23,12 @@ class VideoModule(XModule): css = {'scss': [resource_string(__name__, 'css/video/display.scss')]} js_module_name = "Video" - def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs): - XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs) + def __init__(self, system, location, definition, + instance_state=None, shared_state=None, **kwargs): + XModule.__init__(self, system, location, definition, + instance_state, shared_state, **kwargs) xmltree = etree.fromstring(self.definition['data']) self.youtube = xmltree.get('youtube') - self.name = xmltree.get('name') self.position = 0 if instance_state is not None: @@ -71,7 +72,7 @@ class VideoModule(XModule): 'streams': self.video_list(), 'id': self.location.html_id(), 'position': self.position, - 'name': self.name, + 'display_name': self.display_name, # TODO (cpennington): This won't work when we move to data that isn't on the filesystem 'data_dir': self.metadata['data_dir'], }) diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 1d16849d67..f6a43f2612 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -191,11 +191,20 @@ class XModule(HTMLSnippet): self.instance_state = instance_state self.shared_state = shared_state self.id = self.location.url() - self.name = self.location.name + self.url_name = self.location.name self.category = self.location.category self.metadata = kwargs.get('metadata', {}) self._loaded_children = None + @property + def display_name(self): + ''' + Return a display name for the module: use display_name if defined in + metadata, otherwise convert the url name. + ''' + return self.metadata.get('display_name', + self.url_name.replace('_', ' ')) + def get_children(self): ''' Return module instances for all the children of this module. @@ -355,13 +364,22 @@ class XModuleDescriptor(Plugin, HTMLSnippet): self.metadata = kwargs.get('metadata', {}) self.definition = definition if definition is not None else {} self.location = Location(kwargs.get('location')) - self.name = self.location.name + self.url_name = self.location.name self.category = self.location.category self.shared_state_key = kwargs.get('shared_state_key') self._child_instances = None self._inherited_metadata = set() + @property + def display_name(self): + ''' + Return a display name for the module: use display_name if defined in + metadata, otherwise convert the url name. + ''' + return self.metadata.get('display_name', + self.url_name.replace('_', ' ')) + @property def own_metadata(self): """ diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index ea13bc2640..b0a289d149 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -225,7 +225,7 @@ class XmlDescriptor(XModuleDescriptor): # Write it to a file if necessary if self.split_to_file(xml_object): # Put this object in its own file - filepath = self.__class__._format_filepath(self.category, self.name) + filepath = self.__class__._format_filepath(self.category, self.url_name) resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) with resource_fs.open(filepath, 'w') as file: file.write(etree.tostring(xml_object, pretty_print=True)) @@ -238,10 +238,10 @@ class XmlDescriptor(XModuleDescriptor): xml_object.tail = '' - xml_object.set('filename', self.name) + xml_object.set('filename', self.url_name) # Add the metadata - xml_object.set('url_name', self.name) + xml_object.set('url_name', self.url_name) for attr in self.metadata_attributes: attr_map = self.xml_attribute_map.get(attr, AttrMap(attr)) metadata_key = attr_map.metadata_key diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index cfda6497d5..19eef3ee80 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -83,7 +83,7 @@ def get_course_about_section(course, section_key): log.warning("Missing about section {key} in course {url}".format(key=section_key, url=course.location.url())) return None elif section_key == "title": - return course.metadata.get('display_name', course.name) + return course.metadata.get('display_name', course.url_name) elif section_key == "university": return course.location.org elif section_key == "number": diff --git a/lms/templates/video.html b/lms/templates/video.html index 65ff44e8fa..93273ddb87 100644 --- a/lms/templates/video.html +++ b/lms/templates/video.html @@ -1,5 +1,5 @@ % if name is not UNDEFINED and name is not None: -

            ${name}

            +

            ${display_name}

            % endif
            From 0b2069c61959d9509703c84d503d3dc6c551ea90 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Thu, 2 Aug 2012 15:14:13 -0400 Subject: [PATCH 093/124] make clean_xml script work with stringified errors --- .../courseware/management/commands/clean_xml.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lms/djangoapps/courseware/management/commands/clean_xml.py b/lms/djangoapps/courseware/management/commands/clean_xml.py index 1773c3d21f..de845df572 100644 --- a/lms/djangoapps/courseware/management/commands/clean_xml.py +++ b/lms/djangoapps/courseware/management/commands/clean_xml.py @@ -61,11 +61,7 @@ def import_with_checks(course_dir, verbose=True): course_dirs=course_dirs) def str_of_err(tpl): - (msg, exc_info) = tpl - if exc_info is None: - return msg - - exc_str = '\n'.join(traceback.format_exception(*exc_info)) + (msg, exc_str) = tpl return '{msg}\n{exc}'.format(msg=msg, exc=exc_str) courses = modulestore.get_courses() @@ -83,7 +79,7 @@ def import_with_checks(course_dir, verbose=True): print '\n' print "=" * 40 print 'ERRORs during import:' - print '\n'.join(map(str_of_err,errors)) + print '\n'.join(map(str_of_err, errors)) print "=" * 40 print '\n' From 94e24c16260b85b672cebf9bdfa14b7e86d3bf7c Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Fri, 3 Aug 2012 14:45:43 -0400 Subject: [PATCH 094/124] Leftover name->url_name fixes --- .../lib/xmodule/xmodule/modulestore/search.py | 5 +++-- lms/djangoapps/courseware/grades.py | 10 +++++----- lms/djangoapps/courseware/module_render.py | 18 +++++++++--------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/search.py b/common/lib/xmodule/xmodule/modulestore/search.py index b3eae64fcc..baf3d46b57 100644 --- a/common/lib/xmodule/xmodule/modulestore/search.py +++ b/common/lib/xmodule/xmodule/modulestore/search.py @@ -87,8 +87,9 @@ def path_to_location(modulestore, location, course_name=None): n = len(path) course_id = CourseDescriptor.location_to_id(path[0]) - chapter = path[1].url_name if n > 1 else None - section = path[2].url_name if n > 2 else None + # pull out the location names + chapter = path[1].name if n > 1 else None + section = path[2].name if n > 2 else None # TODO (vshnayder): not handling position at all yet... position = None diff --git a/lms/djangoapps/courseware/grades.py b/lms/djangoapps/courseware/grades.py index 717cbde140..6e7a0ce102 100644 --- a/lms/djangoapps/courseware/grades.py +++ b/lms/djangoapps/courseware/grades.py @@ -74,8 +74,8 @@ def grade_sheet(student, course, grader, student_module_cache): totaled_scores[format] = format_scores sections.append({ - 'display_name': s.metadata.get('display_name'), - 'url_name': s.metadata.get('url_name'), + 'display_name': s.display_name, + 'url_name': s.url_name, 'scores': scores, 'section_total': section_total, 'format': format, @@ -83,9 +83,9 @@ def grade_sheet(student, course, grader, student_module_cache): 'graded': graded, }) - chapters.append({'course': course.metadata.get('display_name'), - 'display_name': c.metadata.get('display_name'), - 'url_name': c.metadata.get('url_name'), + chapters.append({'course': course.display_name, + 'display_name': c.display_name, + 'url_name': c.url_name, 'sections': sections}) grade_summary = grader.grade(totaled_scores) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 4699ed50a4..9260e15c61 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -56,21 +56,21 @@ def toc_for_course(user, request, course, active_chapter, active_section): sections = list() for section in chapter.get_display_items(): - active = (chapter.metadata.get('display_name') == active_chapter and - section.metadata.get('display_name') == active_section) + active = (chapter.display_name == active_chapter and + section.display_name == active_section) hide_from_toc = section.metadata.get('hide_from_toc', 'false').lower() == 'true' if not hide_from_toc: - sections.append({'display_name': section.metadata.get('display_name'), - 'url_name': section.metadata.get('url_name'), + sections.append({'display_name': section.display_name, + 'url_name': section.url_name, 'format': section.metadata.get('format', ''), 'due': section.metadata.get('due', ''), 'active': active}) - chapters.append({'display_name': chapter.metadata.get('display_name'), - 'url_name': chapter.metadata.get('url_name'), + chapters.append({'display_name': chapter.display_name, + 'url_name': chapter.url_name, 'sections': sections, - 'active': chapter.metadata.get('display_name') == active_chapter}) + 'active': chapter.display_name == active_chapter}) return chapters @@ -89,7 +89,7 @@ def get_section(course_module, chapter, section): chapter_module = None for _chapter in course_module.get_children(): - if _chapter.metadata.get('url_name') == chapter: + if _chapter.url_name == chapter: chapter_module = _chapter break @@ -98,7 +98,7 @@ def get_section(course_module, chapter, section): section_module = None for _section in chapter_module.get_children(): - if _section.metadata.get('url_name') == section: + if _section.url_name == section: section_module = _section break From 84ed806f0dd4da076f5396816b1242a9a5e68451 Mon Sep 17 00:00:00 2001 From: ichuang Date: Fri, 3 Aug 2012 20:11:58 -0400 Subject: [PATCH 095/124] start on lms migration path: view for loaded modules, and reload method --- common/lib/xmodule/xmodule/modulestore/xml.py | 35 ++++++++++++------- common/lib/xmodule/xmodule/x_module.py | 2 ++ lms/envs/dev.py | 5 +++ lms/urls.py | 6 ++++ 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 0567e4e7a7..46fcf19469 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -146,19 +146,30 @@ class XMLModuleStore(ModuleStoreBase): os.path.exists(self.data_dir / d / "course.xml")] for course_dir in course_dirs: - try: - # Special-case code here, since we don't have a location for the - # course before it loads. - # So, make a tracker to track load-time errors, then put in the right - # place after the course loads and we have its location - errorlog = make_error_tracker() - course_descriptor = self.load_course(course_dir, errorlog.tracker) - self.courses[course_dir] = course_descriptor - self._location_errors[course_descriptor.location] = errorlog - except: - msg = "Failed to load course '%s'" % course_dir - log.exception(msg) + self.try_load_course(course_dir) + def try_load_course(self,course_dir): + ''' + Load a course, keeping track of errors as we go along. + ''' + try: + # Special-case code here, since we don't have a location for the + # course before it loads. + # So, make a tracker to track load-time errors, then put in the right + # place after the course loads and we have its location + errorlog = make_error_tracker() + course_descriptor = self.load_course(course_dir, errorlog.tracker) + self.courses[course_dir] = course_descriptor + self._location_errors[course_descriptor.location] = errorlog + except: + msg = "Failed to load course '%s'" % course_dir + log.exception(msg) + + def __unicode__(self): + ''' + String representation - for debugging + ''' + return 'data_dir=%s, %d courses, %d modules' % (self.data_dir,len(self.courses),len(self.modules)) def load_course(self, course_dir, tracker): """ diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index f6a43f2612..60670767f7 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -204,6 +204,8 @@ class XModule(HTMLSnippet): ''' return self.metadata.get('display_name', self.url_name.replace('_', ' ')) + def __unicode__(self): + return '' % (self.name, self.category, self.id) def get_children(self): ''' diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 813471fb54..a4655bf763 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -58,6 +58,11 @@ CACHE_TIMEOUT = 0 # Dummy secret key for dev SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' +################################ LMS Migration ################################# +MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True + +LMS_MIGRATION_ALLOWED_IPS = ['any'] + ################################ OpenID Auth ################################# MITX_FEATURES['AUTH_USE_OPENID'] = True MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True diff --git a/lms/urls.py b/lms/urls.py index 78198b2dfb..c74e92ea6e 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -169,6 +169,12 @@ if settings.MITX_FEATURES.get('AUTH_USE_OPENID'): url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'), ) +if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): + urlpatterns += ( + url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'), + url(r'^migrate/reload/(?P[^/]+)$', 'lms_migration.migrate.manage_modulestores'), + ) + urlpatterns = patterns(*urlpatterns) if settings.DEBUG: From f1ba26b007a181fb2962d493ddd7f1a825bf47ed Mon Sep 17 00:00:00 2001 From: ichuang Date: Fri, 3 Aug 2012 20:36:17 -0400 Subject: [PATCH 096/124] require login and enrollment in course to be able to view its courseware --- lms/djangoapps/courseware/views.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 18b710e108..831e8ced29 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -150,6 +150,7 @@ def render_accordion(request, course, chapter, section): return render_to_string('accordion.html', context) +@login_required @ensure_csrf_cookie @cache_control(no_cache=True, no_store=True, must_revalidate=True) def index(request, course_id, chapter=None, section=None, @@ -172,6 +173,10 @@ def index(request, course_id, chapter=None, section=None, - HTTPresponse ''' course = check_course(course_id) + registered = registered_for_course(course, request.user) + if not registered: + log.debug('User %s tried to view course %s but is not enrolled' % (request.user,course.location.url())) + return redirect('/') try: context = { @@ -266,14 +271,18 @@ def course_info(request, course_id): return render_to_response('info.html', {'course': course}) +def registered_for_course(course, user): + '''Return CourseEnrollment if user is registered for course, else False''' + if user is None: + return False + if user.is_authenticated(): + return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists() + else: + return False + @ensure_csrf_cookie @cache_if_anonymous def course_about(request, course_id): - def registered_for_course(course, user): - if user.is_authenticated(): - return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists() - else: - return False course = check_course(course_id, course_must_be_open=False) registered = registered_for_course(course, request.user) return render_to_response('portal/course_about.html', {'course': course, 'registered': registered}) From 30922fb4491d752a46f269150446618b7abdb444 Mon Sep 17 00:00:00 2001 From: ichuang Date: Fri, 3 Aug 2012 21:39:23 -0400 Subject: [PATCH 097/124] add ACCESS_REQUIRE_STAFF_FOR_COURSE feature for enrollment check --- common/djangoapps/student/views.py | 9 +++++++++ lms/djangoapps/courseware/courses.py | 20 ++++++++++++++++++++ lms/envs/dev.py | 3 ++- lms/templates/portal/course_about.html | 3 +++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index 35ce225011..ace1f8f576 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -37,6 +37,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError from models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment from datetime import date from collections import namedtuple +from courseware.courses import course_staff_group_name, has_staff_access_to_course log = logging.getLogger("mitx.student") Article = namedtuple('Article', 'title url author image deck publication publish_date') @@ -184,6 +185,14 @@ def change_enrollment(request): .format(user.username, enrollment.course_id)) return {'success': False, 'error': 'The course requested does not exist.'} + if settings.MITX_FEATURES.get('ACCESS_REQUIRE_STAFF_FOR_COURSE'): + # require that user be in the staff_* group (or be an overall admin) to be able to enroll + # eg staff_6.002x or staff_6.00x + if not has_staff_access_to_course(user,course): + staff_group = course_staff_group_name(course) + log.debug('user %s denied enrollment to %s ; not in %s' % (user,course.location.url(),staff_group)) + return {'success': False, 'error' : '%s membership required to access course.' % staff_group} + enrollment, created = CourseEnrollment.objects.get_or_create(user=user, course_id=course.id) return {'success': True} diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 19eef3ee80..e11fb566f4 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -114,3 +114,23 @@ def get_course_info_section(course, section_key): return "! Info section missing !" raise KeyError("Invalid about key " + str(section_key)) + +def course_staff_group_name(course): + return 'staff_%s' % course.metadata['course'] + +def has_staff_access_to_course(user,course): + ''' + Returns True if the given user has staff access to the course. + This means that user is in the staff_* group, or is an overall admin. + ''' + if user.is_staff: + return True + user_groups = [x[1] for x in user.groups.values_list()] # note this is the Auth group, not UserTestGroup + log.debug('user is in groups %s' % user_groups) + staff_group = course_staff_group_name(course) + if staff_group in user_groups: + return True + return False + + + diff --git a/lms/envs/dev.py b/lms/envs/dev.py index a4655bf763..3e681e6b0d 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -60,12 +60,13 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' ################################ LMS Migration ################################# MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True +MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True LMS_MIGRATION_ALLOWED_IPS = ['any'] ################################ OpenID Auth ################################# MITX_FEATURES['AUTH_USE_OPENID'] = True -MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True +MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True # require that user be in the staff_* group to be able to enroll INSTALLED_APPS += ('external_auth',) INSTALLED_APPS += ('django_openid_auth',) diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html index e6359d0542..afecc2f795 100644 --- a/lms/templates/portal/course_about.html +++ b/lms/templates/portal/course_about.html @@ -19,6 +19,8 @@ $(document).delegate('#class_enroll_form', 'ajax:success', function(data, json, xhr) { if(json.success) { location.href="${reverse('dashboard')}"; + }else{ + document.getElementById('register_message').innerHTML = "

            " + json.error + "

            "; } }); })(this) @@ -63,6 +65,7 @@ You are registered for this course (${course.number}). %else: Register for ${course.number} +
            %endif %else: Register for ${course.number} From 10f53d62e3a312b58509d399ea97e2bb7128e4d2 Mon Sep 17 00:00:00 2001 From: ichuang Date: Fri, 3 Aug 2012 22:38:33 -0400 Subject: [PATCH 098/124] migration views - see modulestore contents, and force reload of course --- lms/djangoapps/lms_migration/__init__.py | 0 lms/djangoapps/lms_migration/migrate.py | 104 +++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 lms/djangoapps/lms_migration/__init__.py create mode 100644 lms/djangoapps/lms_migration/migrate.py diff --git a/lms/djangoapps/lms_migration/__init__.py b/lms/djangoapps/lms_migration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/lms_migration/migrate.py b/lms/djangoapps/lms_migration/migrate.py new file mode 100644 index 0000000000..4da346c657 --- /dev/null +++ b/lms/djangoapps/lms_migration/migrate.py @@ -0,0 +1,104 @@ +# +# migration tools for content team to go from stable-edx4edx to LMS+CMS +# + +import logging +from pprint import pprint +import xmodule.modulestore.django as xmodule_django +from xmodule.modulestore.django import modulestore + +from django.http import HttpResponse +from django.conf import settings + +log = logging.getLogger("mitx.lms_migrate") +LOCAL_DEBUG = True +ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS + +def escape(s): + """escape HTML special characters in string""" + return str(s).replace('<','<').replace('>','>') + +def manage_modulestores(request,reload_dir=None): + ''' + Manage the static in-memory modulestores. + + If reload_dir is not None, then instruct the xml loader to reload that course directory. + ''' + html = "" + + def_ms = modulestore() + courses = def_ms.get_courses() + + #---------------------------------------- + # check on IP address of requester + + ip = request.META.get('HTTP_X_REAL_IP','') # nginx reverse proxy + if not ip: + ip = request.META.get('REMOTE_ADDR','None') + + if LOCAL_DEBUG: + html += '

            IP address: %s ' % ip + + if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): + html += 'Permission denied' + html += "" + return HttpResponse(html) + + #---------------------------------------- + # reload course if specified + + if reload_dir is not None: + if reload_dir not in def_ms.courses: + html += "

            Error: '%s' is not a valid course directory

            " % reload_dir + else: + html += "

            Reloaded course directory '%s'

            " % reload_dir + def_ms.try_load_course(reload_dir) + + #---------------------------------------- + + html += '

            Courses loaded in the modulestore

            ' + html += '
              ' + for cdir, course in def_ms.courses.items(): + html += '
            1. %s (%s)
              2. ' % (settings.MITX_ROOT_URL, + escape(cdir), + escape(cdir), + course.location.url()) + html += '
            ' + + #---------------------------------------- + + dumpfields = ['definition','location','metadata'] + + for cdir, course in def_ms.courses.items(): + html += '
            ' + html += '

            Course: %s (%s)

            ' % (course.metadata['display_name'],cdir) + + for field in dumpfields: + data = getattr(course,field) + html += '

            %s

            ' % field + if type(data)==dict: + html += '
              ' + for k,v in data.items(): + html += '
            • %s:%s
              • ' % (escape(k),escape(v)) + html += '
            ' + else: + html += '
            • %s
            ' % escape(data) + + + #---------------------------------------- + + html += '
            ' + html += "courses: 
            %s
            " % escape(courses) + + ms = xmodule_django._MODULESTORES + html += "modules: 
            %s
            " % escape(ms) + html += "default modulestore: 
            %s
            " % escape(unicode(def_ms)) + + #---------------------------------------- + + log.debug('_MODULESTORES=%s' % ms) + log.debug('courses=%s' % courses) + log.debug('def_ms=%s' % unicode(def_ms)) + + html += "" + return HttpResponse(html) From 9de6e28180021c4f0310a981c808635d33f4309b Mon Sep 17 00:00:00 2001 From: ichuang Date: Fri, 3 Aug 2012 22:52:48 -0400 Subject: [PATCH 099/124] limit course reload to localhost or user.is_staff --- lms/djangoapps/lms_migration/migrate.py | 11 ++++++++--- lms/envs/dev.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lms/djangoapps/lms_migration/migrate.py b/lms/djangoapps/lms_migration/migrate.py index 4da346c657..285b26b483 100644 --- a/lms/djangoapps/lms_migration/migrate.py +++ b/lms/djangoapps/lms_migration/migrate.py @@ -38,11 +38,16 @@ def manage_modulestores(request,reload_dir=None): if LOCAL_DEBUG: html += '

            IP address: %s ' % ip + log.debug('request from ip=%s' % ip) if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): - html += 'Permission denied' - html += "" - return HttpResponse(html) + if request.user and request.user.is_staff: + log.debug('request allowed because user=%s is staff' % request.user) + else: + html += 'Permission denied' + html += "" + log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS) + return HttpResponse(html) #---------------------------------------- # reload course if specified diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 3e681e6b0d..d61c2e8b39 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -62,7 +62,7 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True -LMS_MIGRATION_ALLOWED_IPS = ['any'] +LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1'] ################################ OpenID Auth ################################# MITX_FEATURES['AUTH_USE_OPENID'] = True From c20ebd13c47f6053516fe25b5b15719e6abea5e2 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 08:22:06 -0400 Subject: [PATCH 100/124] in course_about.html make "You are registered..." a link to course --- lms/templates/portal/course_about.html | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html index afecc2f795..bdea0a47d1 100644 --- a/lms/templates/portal/course_about.html +++ b/lms/templates/portal/course_about.html @@ -62,7 +62,20 @@
            %if user.is_authenticated(): %if registered: + <% + if course.has_started() or settings.MITX_FEATURES['DISABLE_START_DATES']: + course_target = reverse('info', args=[course.id]) + else: + course_target = reverse('about_course', args=[course.id]) + show_link = settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION') + %> + %if show_link: + + %endif You are registered for this course (${course.number}). + %if show_link: + + %endif %else: Register for ${course.number}
            From 3f83904c128bf3bd8dcf01bb23039c96cf5c1d61 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 10:19:54 -0400 Subject: [PATCH 101/124] if AUTH_REQUIRE_STAFF_FOR_COURSE then course list = those accessible --- common/djangoapps/student/views.py | 14 +++++--------- lms/djangoapps/courseware/courses.py | 23 ++++++++++++++++++++++- lms/djangoapps/courseware/views.py | 18 +++++------------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index ace1f8f576..87490786c1 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -8,7 +8,6 @@ import uuid import feedparser import urllib import itertools -from collections import defaultdict from django.conf import settings from django.contrib.auth import logout, authenticate, login @@ -37,7 +36,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError from models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment from datetime import date from collections import namedtuple -from courseware.courses import course_staff_group_name, has_staff_access_to_course +from courseware.courses import course_staff_group_name, has_staff_access_to_course, get_courses_by_university log = logging.getLogger("mitx.student") Article = namedtuple('Article', 'title url author image deck publication publish_date') @@ -65,9 +64,9 @@ def index(request): from external_auth.views import edXauth_ssl_login return edXauth_ssl_login(request) - return main_index() + return main_index(user=request.user) -def main_index(extra_context = {}): +def main_index(extra_context = {}, user=None): ''' Render the edX main page. @@ -89,11 +88,8 @@ def main_index(extra_context = {}): entry.image = soup.img['src'] if soup.img else None entry.summary = soup.getText() - universities = defaultdict(list) - courses = sorted(modulestore().get_courses(), key=lambda course: course.number) - for course in courses: - universities[course.org].append(course) - + # The course selection work is done in courseware.courses. + universities = get_courses_by_university(None) context = {'universities': universities, 'entries': entries} context.update(extra_context) return render_to_response('index.html', context) diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index e11fb566f4..c050084fff 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -1,3 +1,4 @@ +from collections import defaultdict from fs.errors import ResourceNotFoundError from functools import wraps import logging @@ -123,6 +124,8 @@ def has_staff_access_to_course(user,course): Returns True if the given user has staff access to the course. This means that user is in the staff_* group, or is an overall admin. ''' + if user is None or (not user.is_authenticated()) or course is None: + return False if user.is_staff: return True user_groups = [x[1] for x in user.groups.values_list()] # note this is the Auth group, not UserTestGroup @@ -132,5 +135,23 @@ def has_staff_access_to_course(user,course): return True return False - +def get_courses_by_university(user): + ''' + Returns dict of lists of courses available, keyed by course.org (ie university). + Courses are sorted by course.number. + + if ACCESS_REQUIRE_STAFF_FOR_COURSE then list only includes those accessible to user. + ''' + # TODO: Clean up how 'error' is done. + # filter out any courses that errored. + courses = [c for c in modulestore().get_courses() + if isinstance(c, CourseDescriptor)] + courses = sorted(courses, key=lambda course: course.number) + universities = defaultdict(list) + for course in courses: + if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): + if not has_staff_access_to_course(user,course): + continue + universities[course.org].append(course) + return universities diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 831e8ced29..41b2101b44 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -1,4 +1,3 @@ -from collections import defaultdict import json import logging import urllib @@ -28,7 +27,7 @@ from xmodule.course_module import CourseDescriptor from util.cache import cache, cache_if_anonymous from student.models import UserTestGroup, CourseEnrollment from courseware import grades -from courseware.courses import check_course +from courseware.courses import check_course, get_courses_by_university log = logging.getLogger("mitx.courseware") @@ -58,19 +57,12 @@ def user_groups(user): @ensure_csrf_cookie @cache_if_anonymous def courses(request): - # TODO: Clean up how 'error' is done. - - # filter out any courses that errored. - courses = [c for c in modulestore().get_courses() - if isinstance(c, CourseDescriptor)] - courses = sorted(courses, key=lambda course: course.number) - universities = defaultdict(list) - for course in courses: - universities[course.org].append(course) - + ''' + Render "find courses" page. The course selection work is done in courseware.courses. + ''' + universities = get_courses_by_university(request.user) return render_to_response("courses.html", {'universities': universities}) - @cache_control(no_cache=True, no_store=True, must_revalidate=True) def gradebook(request, course_id): if 'course_admin' not in user_groups(request.user): From fb7b48e10af14f643ce038459f1d3cf56197414e Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 10:28:05 -0400 Subject: [PATCH 102/124] minor - have migrate also show user when debugging --- lms/djangoapps/lms_migration/migrate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/lms_migration/migrate.py b/lms/djangoapps/lms_migration/migrate.py index 285b26b483..2bf893507b 100644 --- a/lms/djangoapps/lms_migration/migrate.py +++ b/lms/djangoapps/lms_migration/migrate.py @@ -38,7 +38,8 @@ def manage_modulestores(request,reload_dir=None): if LOCAL_DEBUG: html += '

            IP address: %s ' % ip - log.debug('request from ip=%s' % ip) + html += '

            User: %s ' % request.user + log.debug('request from ip=%s, user=%s' % (ip,request.user)) if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): if request.user and request.user.is_staff: From d50af5765e009ff9c142f24c9f87b061007d20b0 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 11:03:54 -0400 Subject: [PATCH 103/124] make university profile pages also use get_courses_by_university --- common/lib/xmodule/xmodule/xml_module.py | 1 + lms/djangoapps/courseware/courses.py | 7 ++++++- lms/djangoapps/courseware/views.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index b0a289d149..d46304b067 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -41,6 +41,7 @@ class XmlDescriptor(XModuleDescriptor): # to definition_from_xml, and from the xml returned by definition_to_xml metadata_attributes = ('format', 'graceperiod', 'showanswer', 'rerandomize', 'start', 'due', 'graded', 'display_name', 'url_name', 'hide_from_toc', + 'ispublic', # if True, then course is listed for all users; see # VS[compat] Remove once unused. 'name', 'slug') diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index c050084fff..78025c2fae 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -135,6 +135,11 @@ def has_staff_access_to_course(user,course): return True return False +def has_access_to_course(user,course): + if course.metadata.get('ispublic'): + return True + return has_staff_access_to_course(user,course) + def get_courses_by_university(user): ''' Returns dict of lists of courses available, keyed by course.org (ie university). @@ -150,7 +155,7 @@ def get_courses_by_university(user): universities = defaultdict(list) for course in courses: if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): - if not has_staff_access_to_course(user,course): + if not has_access_to_course(user,course): continue universities[course.org].append(course) return universities diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 41b2101b44..5db3bcf91a 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -289,7 +289,7 @@ def university_profile(request, org_id): raise Http404("University Profile not found for {0}".format(org_id)) # Only grab courses for this org... - courses = [c for c in all_courses if c.org == org_id] + courses = get_courses_by_university(request.user)[org_id] context = dict(courses=courses, org_id=org_id) template_file = "university_profile/{0}.html".format(org_id).lower() From b8ae026c2937b0a3bbe66278033fe2fc9fc326e4 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 11:16:47 -0400 Subject: [PATCH 104/124] fail gracefully if course.xml missing metadata in course_staff_group_name --- lms/djangoapps/courseware/courses.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 78025c2fae..133a593ac8 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -117,7 +117,10 @@ def get_course_info_section(course, section_key): raise KeyError("Invalid about key " + str(section_key)) def course_staff_group_name(course): - return 'staff_%s' % course.metadata['course'] + coursename = course.metadata.get('course','') + if not coursename: # Fall 2012: not all course.xml have metadata correct yet + coursename = course.metadata.get('data_dir','UnknownCourseName') + return 'staff_%s' % coursename def has_staff_access_to_course(user,course): ''' From 85af1d88cf46c2279caa4974ba364ebf5bbfb020 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 15:30:36 -0400 Subject: [PATCH 105/124] utility scripts to create new users, and staff_* groups for courses --- utility-scripts/create_groups.py | 32 ++++++++ utility-scripts/create_user.py | 137 +++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 utility-scripts/create_groups.py create mode 100644 utility-scripts/create_user.py diff --git a/utility-scripts/create_groups.py b/utility-scripts/create_groups.py new file mode 100644 index 0000000000..841242fd54 --- /dev/null +++ b/utility-scripts/create_groups.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# +# File: create_groups.py +# Date: 04-Aug-12 +# Author: I. Chuang +# +# Create all staff_* groups for classes in data directory. + +import os, sys, string, re + +sys.path.append(os.path.abspath('.')) +os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev' +from lms.envs.dev import * + +from django.conf import settings +from django.contrib.auth.models import User, Group +from path import path + +data_dir = settings.DATA_DIR +print "data_dir = %s" % data_dir + +for course_dir in os.listdir(data_dir): + # print course_dir + if not os.path.isdir(path(data_dir) / course_dir): + continue + gname = 'staff_%s' % course_dir + if Group.objects.filter(name=gname): + print "group exists for %s" % gname + continue + g = Group(name=gname) + g.save() + print "created group %s" % gname diff --git a/utility-scripts/create_user.py b/utility-scripts/create_user.py new file mode 100644 index 0000000000..19bdb6e743 --- /dev/null +++ b/utility-scripts/create_user.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +# +# File: create_user.py +# Date: 04-Aug-12 +# Author: I. Chuang +# +# Create user. Prompt for groups and ExternalAuthMap + +import os, sys, string, re +import datetime +from getpass import getpass +import json +import readline + +sys.path.append(os.path.abspath('.')) +os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev' +from lms.envs.dev import * + +from student.models import UserProfile, Registration +from external_auth.models import ExternalAuthMap +from django.contrib.auth.models import User, Group +from random import choice + +class MyCompleter(object): # Custom completer + + def __init__(self, options): + self.options = sorted(options) + + def complete(self, text, state): + if state == 0: # on first trigger, build possible matches + if text: # cache matches (entries that start with entered text) + self.matches = [s for s in self.options + if s and s.startswith(text)] + else: # no text entered, all matches possible + self.matches = self.options[:] + + # return match indexed by state + try: + return self.matches[state] + except IndexError: + return None + +def GenPasswd(length=8, chars=string.letters + string.digits): + return ''.join([choice(chars) for i in range(length)]) + +#----------------------------------------------------------------------------- +# main + +while True: + uname = raw_input('username: ') + if User.objects.filter(username=uname): + print "username %s already taken" % uname + else: + break + +while True: + email = raw_input('email: ') + if User.objects.filter(email=email): + print "email %s already taken" % email + else: + break + +name = raw_input('Full name: ') + +make_eamap = False +if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y': + if not email.endswith('@MIT.EDU'): + print "Failed - email must be @MIT.EDU" + sys.exit(-1) + mit_domain = 'ssl:MIT' + if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain): + print "Failed - email %s already exists as external_id" % email + sys.exit(-1) + make_eamap = True + password = GenPasswd(12) +else: + while True: + password = getpass() + password2 = getpass() + if password == password2: + break + print "Oops, passwords do not match, please retry" + +user = User(username=uname, email=email, is_active=True) +user.set_password(password) +try: + user.save() +except IntegrityError: + print "Oops, failed to create user %s, IntegrityError" % user + raise + +r = Registration() +r.register(user) + +up = UserProfile(user=user) +up.name = name +up.save() + +if make_eamap: + credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email) + eamap = ExternalAuthMap(external_id = email, + external_email = email, + external_domain = mit_domain, + external_name = name, + internal_password = password, + external_credentials = json.dumps(credentials), + ) + eamap.user = user + eamap.dtsignup = datetime.datetime.now() + eamap.save() + +print "User %s created successfully!" % user + +if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y': + sys.exit(0) + +print "Here are the groups available:" + +groups = [str(g.name) for g in Group.objects.all()] +print groups + +completer = MyCompleter(groups) +readline.set_completer(completer.complete) +readline.parse_and_bind('tab: complete') + +while True: + gname = raw_input("Add group (tab to autocomplete, empty line to end): ") + if not gname: + break + if not gname in groups: + print "Unknown group %s" % gname + continue + g = Group.objects.get(name=gname) + user.groups.add(g) + print "Added %s to group %s" % (user,g) + +print "Done!" From 7fe75030cc64050e8997d011bab39174aa9c24ad Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 17:52:14 -0400 Subject: [PATCH 106/124] fix staff edit link in module content display (goes to github) --- common/djangoapps/xmodule_modifiers.py | 35 +++++++++++++++------- common/lib/xmodule/xmodule/html_module.py | 10 ++++++- common/lib/xmodule/xmodule/xml_module.py | 11 +++++-- lms/djangoapps/courseware/courses.py | 8 ++++- lms/djangoapps/courseware/module_render.py | 5 +++- lms/templates/staff_problem_info.html | 8 ++--- 6 files changed, 57 insertions(+), 20 deletions(-) diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py index 4d412000ec..082c5f5122 100644 --- a/common/djangoapps/xmodule_modifiers.py +++ b/common/djangoapps/xmodule_modifiers.py @@ -1,9 +1,15 @@ +import re import json +import logging + from django.conf import settings from functools import wraps from static_replace import replace_urls from mitxmako.shortcuts import render_to_string +from xmodule.seq_module import SequenceModule +from xmodule.vertical_module import VerticalModule +log = logging.getLogger("mitx.xmodule_modifiers") def wrap_xmodule(get_html, module, template): """ @@ -69,27 +75,33 @@ def add_histogram(get_html, module): the output of the old get_html function with additional information for admin users only, including a histogram of student answers and the definition of the xmodule + + Does nothing if module is a SequenceModule """ @wraps(get_html) def _get_html(): + + if type(module) in [SequenceModule, VerticalModule]: # TODO: make this more general, eg use an XModule attribute instead + return get_html() + module_id = module.id histogram = grade_histogram(module_id) render_histogram = len(histogram) > 0 - # TODO: fixme - no filename in module.xml in general (this code block - # for edx4edx) the following if block is for summer 2012 edX course - # development; it will change when the CMS comes online - if settings.MITX_FEATURES.get('DISPLAY_EDIT_LINK') and settings.DEBUG and module_xml.get('filename') is not None: - coursename = multicourse_settings.get_coursename_from_request(request) - github_url = multicourse_settings.get_course_github_url(coursename) - fn = module_xml.get('filename') - if module_xml.tag=='problem': fn = 'problems/' + fn # grrr - edit_link = (github_url + '/tree/master/' + fn) if github_url is not None else None - if module_xml.tag=='problem': edit_link += '.xml' # grrr + # TODO (ichuang): Remove after fall 2012 LMS migration done + if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): + filename = module.definition.get('filename','') + log.debug('filename = %s' % filename) + data_dir = module.system.filestore.root_path.rsplit('/')[-1] + edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filename) + log.debug('edit_link = %s' % edit_link) + log.debug('module = %s' % dir(module)) + log.debug('module type = %s' % type(module)) + log.debug('location = %s' % str(module.location)) else: edit_link = False - staff_context = {'definition': json.dumps(module.definition, indent=4), + staff_context = {'definition': module.definition.get('data'), 'metadata': json.dumps(module.metadata, indent=4), 'element_id': module.location.html_id(), 'edit_link': edit_link, @@ -99,3 +111,4 @@ def add_histogram(get_html, module): return render_to_string("staff_problem_info.html", staff_context) return _get_html + diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 7a09004e33..8af1a9c5ba 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -94,7 +94,15 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): msg = "Couldn't parse html in {0}.".format(filepath) log.warning(msg) system.error_tracker("Warning: " + msg) - return {'data' : html} + + definition = {'data' : html} + + # TODO (ichuang): remove this after migration + # for Fall 2012 LMS migration: keep filename + definition['filename'] = filepath + + return definition + except (ResourceNotFoundError) as err: msg = 'Unable to load file contents at path {0}: {1} '.format( filepath, err) diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index d46304b067..dee87921d9 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -9,7 +9,7 @@ from fs.errors import ResourceNotFoundError import os import sys -log = logging.getLogger(__name__) +log = logging.getLogger('mitx.' + __name__) _AttrMapBase = namedtuple('_AttrMap', 'metadata_key to_metadata from_metadata') @@ -110,6 +110,7 @@ class XmlDescriptor(XModuleDescriptor): filename = xml_object.get('filename') if filename is None: definition_xml = copy.deepcopy(xml_object) + filepath = '' else: filepath = cls._format_filepath(xml_object.tag, filename) @@ -137,7 +138,13 @@ class XmlDescriptor(XModuleDescriptor): raise Exception, msg, sys.exc_info()[2] cls.clean_metadata_from_xml(definition_xml) - return cls.definition_from_xml(definition_xml, system) + definition = cls.definition_from_xml(definition_xml, system) + + # TODO (ichuang): remove this after migration + # for Fall 2012 LMS migration: keep filename + definition['filename'] = filepath + + return definition @classmethod diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 133a593ac8..e568f97f56 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -117,7 +117,13 @@ def get_course_info_section(course, section_key): raise KeyError("Invalid about key " + str(section_key)) def course_staff_group_name(course): - coursename = course.metadata.get('course','') + ''' + course should be either a CourseDescriptor instance, or a string (the .course entry of a Location) + ''' + if type(course)==str: + coursename = course + else: + coursename = course.metadata.get('course','') if not coursename: # Fall 2012: not all course.xml have metadata correct yet coursename = course.metadata.get('data_dir','UnknownCourseName') return 'staff_%s' % coursename diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 9260e15c61..cdb9dc40f3 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -15,6 +15,8 @@ from xmodule.exceptions import NotFoundError from xmodule.x_module import ModuleSystem from xmodule_modifiers import replace_static_urls, add_histogram, wrap_xmodule +from courseware.courses import has_staff_access_to_course + log = logging.getLogger("mitx.courseware") @@ -188,7 +190,8 @@ def get_module(user, request, location, student_module_cache, position=None): module.metadata['data_dir'] ) - if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff: + if (settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and + (user.is_staff or has_staff_access_to_course(user, module.location.course))): module.get_html = add_histogram(module.get_html, module) # If StudentModule for this instance wasn't already in the database, diff --git a/lms/templates/staff_problem_info.html b/lms/templates/staff_problem_info.html index f9fa999ae9..c9b92c51db 100644 --- a/lms/templates/staff_problem_info.html +++ b/lms/templates/staff_problem_info.html @@ -1,11 +1,11 @@ ${module_content} -
            -definition = ${definition | h} -metadata = ${metadata | h} -
            %if edit_link: % endif +
            +definition = 
            ${definition | h}
            +metadata = ${metadata | h} +
            %if render_histogram:
            %endif From 23669f5aa1fc23b049f53032fa073f171e2aec89 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 17:56:32 -0400 Subject: [PATCH 107/124] add some error handling to utility scripts --- utility-scripts/create_groups.py | 7 ++++++- utility-scripts/create_user.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/utility-scripts/create_groups.py b/utility-scripts/create_groups.py index 841242fd54..063d2ae392 100644 --- a/utility-scripts/create_groups.py +++ b/utility-scripts/create_groups.py @@ -10,7 +10,12 @@ import os, sys, string, re sys.path.append(os.path.abspath('.')) os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev' -from lms.envs.dev import * + +try: + from lms.envs.dev import * +except Exception as err: + print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory." + sys.exit(-1) from django.conf import settings from django.contrib.auth.models import User, Group diff --git a/utility-scripts/create_user.py b/utility-scripts/create_user.py index 19bdb6e743..c9708f537d 100644 --- a/utility-scripts/create_user.py +++ b/utility-scripts/create_user.py @@ -14,7 +14,12 @@ import readline sys.path.append(os.path.abspath('.')) os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev' -from lms.envs.dev import * + +try: + from lms.envs.dev import * +except Exception as err: + print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory." + sys.exit(-1) from student.models import UserProfile, Registration from external_auth.models import ExternalAuthMap From ebe6bf4888611364f8f36a2b38632bf4f29121a1 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 21:10:50 -0400 Subject: [PATCH 108/124] remove some unnecessary debugging lines in xmodule_modifiers --- common/djangoapps/xmodule_modifiers.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py index 082c5f5122..221ad31116 100644 --- a/common/djangoapps/xmodule_modifiers.py +++ b/common/djangoapps/xmodule_modifiers.py @@ -91,13 +91,8 @@ def add_histogram(get_html, module): # TODO (ichuang): Remove after fall 2012 LMS migration done if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): filename = module.definition.get('filename','') - log.debug('filename = %s' % filename) data_dir = module.system.filestore.root_path.rsplit('/')[-1] edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filename) - log.debug('edit_link = %s' % edit_link) - log.debug('module = %s' % dir(module)) - log.debug('module type = %s' % type(module)) - log.debug('location = %s' % str(module.location)) else: edit_link = False From 9db88b0b52c23513b757b30bc47ef828e99bde8a Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 21:13:43 -0400 Subject: [PATCH 109/124] fix comment in dev.py --- lms/envs/dev.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/envs/dev.py b/lms/envs/dev.py index d61c2e8b39..50062e0513 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -60,13 +60,13 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' ################################ LMS Migration ################################# MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True -MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True +MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True # require that user be in the staff_* group to be able to enroll LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1'] ################################ OpenID Auth ################################# MITX_FEATURES['AUTH_USE_OPENID'] = True -MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True # require that user be in the staff_* group to be able to enroll +MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True INSTALLED_APPS += ('external_auth',) INSTALLED_APPS += ('django_openid_auth',) From 1ff49aa3f9da856c2ca213f108f11bc6eb3dd3fe Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 21:15:51 -0400 Subject: [PATCH 110/124] remove unnecessary comments from util-scripts/* --- utility-scripts/create_groups.py | 2 -- utility-scripts/create_user.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/utility-scripts/create_groups.py b/utility-scripts/create_groups.py index 063d2ae392..33c563127f 100644 --- a/utility-scripts/create_groups.py +++ b/utility-scripts/create_groups.py @@ -1,8 +1,6 @@ #!/usr/bin/python # # File: create_groups.py -# Date: 04-Aug-12 -# Author: I. Chuang # # Create all staff_* groups for classes in data directory. diff --git a/utility-scripts/create_user.py b/utility-scripts/create_user.py index c9708f537d..e5cb5aed2c 100644 --- a/utility-scripts/create_user.py +++ b/utility-scripts/create_user.py @@ -1,8 +1,6 @@ #!/usr/bin/python # # File: create_user.py -# Date: 04-Aug-12 -# Author: I. Chuang # # Create user. Prompt for groups and ExternalAuthMap From 3c23235885d3d78a2530bde95c4e7aa893456ef7 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 11:39:56 -0400 Subject: [PATCH 111/124] fix for some broken github edit links - avoids symlinks --- common/djangoapps/xmodule_modifiers.py | 9 ++++++--- common/lib/xmodule/xmodule/html_module.py | 4 ++-- common/lib/xmodule/xmodule/xml_module.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py index 221ad31116..843d2eaa38 100644 --- a/common/djangoapps/xmodule_modifiers.py +++ b/common/djangoapps/xmodule_modifiers.py @@ -90,9 +90,12 @@ def add_histogram(get_html, module): # TODO (ichuang): Remove after fall 2012 LMS migration done if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): - filename = module.definition.get('filename','') - data_dir = module.system.filestore.root_path.rsplit('/')[-1] - edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filename) + [filepath, filename] = module.definition.get('filename','') + osfs = module.system.filestore + if osfs.exists(filename): + filepath = filename # if original, unmangled filename exists then use it (github doesn't like symlinks) + data_dir = osfs.root_path.rsplit('/')[-1] + edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filepath) else: edit_link = False diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 8af1a9c5ba..260b84278b 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -98,8 +98,8 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): definition = {'data' : html} # TODO (ichuang): remove this after migration - # for Fall 2012 LMS migration: keep filename - definition['filename'] = filepath + # for Fall 2012 LMS migration: keep filename (and unmangled filename) + definition['filename'] = [ filepath, filename ] return definition diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index dee87921d9..fbb17fd236 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -141,8 +141,8 @@ class XmlDescriptor(XModuleDescriptor): definition = cls.definition_from_xml(definition_xml, system) # TODO (ichuang): remove this after migration - # for Fall 2012 LMS migration: keep filename - definition['filename'] = filepath + # for Fall 2012 LMS migration: keep filename (and unmangled filename) + definition['filename'] = [ filepath, filename ] return definition From 3ee224e3994fb66709dcab2a460d77329d3d4a6f Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 12:39:13 -0400 Subject: [PATCH 112/124] improve create_user script slightly, to auto-grab fullname for MIT users --- utility-scripts/create_user.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/utility-scripts/create_user.py b/utility-scripts/create_user.py index e5cb5aed2c..3ce9ce0ecf 100644 --- a/utility-scripts/create_user.py +++ b/utility-scripts/create_user.py @@ -56,17 +56,9 @@ while True: else: break -while True: - email = raw_input('email: ') - if User.objects.filter(email=email): - print "email %s already taken" % email - else: - break - -name = raw_input('Full name: ') - make_eamap = False if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y': + email = '%s@MIT.EDU' % uname if not email.endswith('@MIT.EDU'): print "Failed - email must be @MIT.EDU" sys.exit(-1) @@ -76,6 +68,13 @@ if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y': sys.exit(-1) make_eamap = True password = GenPasswd(12) + + # get name from kerberos + kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip() + name = raw_input('Full name: [%s] ' % kname).strip() + if name=='': + name = kname + print "name = %s" % name else: while True: password = getpass() @@ -84,6 +83,16 @@ else: break print "Oops, passwords do not match, please retry" + while True: + email = raw_input('email: ') + if User.objects.filter(email=email): + print "email %s already taken" % email + else: + break + + name = raw_input('Full name: ') + + user = User(username=uname, email=email, is_active=True) user.set_password(password) try: From c42960c172604472ae8a8a98349b2cdd986717ba Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 16:32:58 -0400 Subject: [PATCH 113/124] add feature ENABLE_SQL_TRACKING_LOGS and url view /event_logs --- common/djangoapps/track/models.py | 20 +++++++++++++++++++- common/djangoapps/track/views.py | 29 +++++++++++++++++++++++++++-- lms/envs/dev.py | 1 + lms/urls.py | 5 +++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/common/djangoapps/track/models.py b/common/djangoapps/track/models.py index 71a8362390..401fa2832f 100644 --- a/common/djangoapps/track/models.py +++ b/common/djangoapps/track/models.py @@ -1,3 +1,21 @@ from django.db import models -# Create your models here. +from django.db import models + +class TrackingLog(models.Model): + dtcreated = models.DateTimeField('creation date',auto_now_add=True) + username = models.CharField(max_length=32,blank=True) + ip = models.CharField(max_length=32,blank=True) + event_source = models.CharField(max_length=32) + event_type = models.CharField(max_length=32,blank=True) + event = models.TextField(blank=True) + agent = models.CharField(max_length=256,blank=True) + page = models.CharField(max_length=32,blank=True,null=True) + time = models.DateTimeField('event time') + + def __unicode__(self): + s = "[%s] %s@%s: %s | %s | %s | %s" % (self.time, self.username, self.ip, self.event_source, + self.event_type, self.page, self.event) + return s + + diff --git a/common/djangoapps/track/views.py b/common/djangoapps/track/views.py index a60d8bef28..1123b8d30a 100644 --- a/common/djangoapps/track/views.py +++ b/common/djangoapps/track/views.py @@ -2,19 +2,32 @@ import json import logging import os import datetime +import dateutil.parser -# Create your views here. +from django.contrib.auth.decorators import login_required from django.http import HttpResponse from django.http import Http404 +from django.shortcuts import redirect from django.conf import settings +from mitxmako.shortcuts import render_to_response + +from django_future.csrf import ensure_csrf_cookie +from track.models import TrackingLog log = logging.getLogger("tracking") +LOGFIELDS = ['username','ip','event_source','event_type','event','agent','page','time'] def log_event(event): event_str = json.dumps(event) log.info(event_str[:settings.TRACK_MAX_EVENT]) - + if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): + event['time'] = dateutil.parser.parse(event['time']) + tldat = TrackingLog(**dict([(x,event[x]) for x in LOGFIELDS])) + try: + tldat.save() + except Exception as err: + log.debug(err) def user_track(request): try: # TODO: Do the same for many of the optional META parameters @@ -70,4 +83,16 @@ def server_track(request, event_type, event, page=None): "page": page, "time": datetime.datetime.utcnow().isoformat(), } + + if event_type=="/event_logs" and request.user.is_staff: # don't log + return log_event(event) + +@login_required +@ensure_csrf_cookie +def view_tracking_log(request): + if not request.user.is_staff: + return redirect('/') + record_instances = TrackingLog.objects.all().order_by('-time')[0:100] + return render_to_response('tracking_log.html',{'records':record_instances}) + diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 50062e0513..204fcec04b 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -14,6 +14,7 @@ DEBUG = True TEMPLATE_DEBUG = True MITX_FEATURES['DISABLE_START_DATES'] = True +MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True WIKI_ENABLED = True diff --git a/lms/urls.py b/lms/urls.py index c74e92ea6e..9dc317039e 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -175,6 +175,11 @@ if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): url(r'^migrate/reload/(?P[^/]+)$', 'lms_migration.migrate.manage_modulestores'), ) +if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): + urlpatterns += ( + url(r'^event_logs$', 'track.views.view_tracking_log'), + ) + urlpatterns = patterns(*urlpatterns) if settings.DEBUG: From 190f1f8f892e352ad88a13f0f16aa1331906e014 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 16:35:32 -0400 Subject: [PATCH 114/124] tracking_log template --- lms/templates/tracking_log.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 lms/templates/tracking_log.html diff --git a/lms/templates/tracking_log.html b/lms/templates/tracking_log.html new file mode 100644 index 0000000000..66d375c2f3 --- /dev/null +++ b/lms/templates/tracking_log.html @@ -0,0 +1,14 @@ + +

            Tracking Log

            + +% for rec in records: + + + + + + + +% endfor +
            datetimeusernameipaddrsourcetype
            ${rec.time}${rec.username}${rec.ip}${rec.event_source}${rec.event_type}
            + \ No newline at end of file From 0347eb498c40efb45637045c3ec8e6e50deb1e99 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 20:24:00 -0400 Subject: [PATCH 115/124] add MITX_FEATURES flags to enable textbook and discussion, and modify course_navigation correspondingly --- lms/envs/common.py | 11 +++ lms/envs/dev_ike.py | 107 +-------------------------- lms/templates/course_navigation.html | 6 ++ 3 files changed, 21 insertions(+), 103 deletions(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index d89e6760a7..83a4bd4181 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -48,6 +48,17 @@ MITX_FEATURES = { ## DO NOT SET TO True IN THIS FILE ## Doing so will cause all courses to be released on production 'DISABLE_START_DATES': False, # When True, all courses will be active, regardless of start date + + 'ENABLE_TEXTBOOK' : True, + 'ENABLE_DISCUSSION' : True, + + 'ENABLE_SQL_TRACKING_LOGS': False, + 'ENABLE_LMS_MIGRATION': False, + + # extrernal access methods + 'ACCESS_REQUIRE_STAFF_FOR_COURSE': False, + 'AUTH_USE_OPENID': False, + 'AUTH_USE_MIT_CERTIFICATES' : False, } # Used for A/B testing diff --git a/lms/envs/dev_ike.py b/lms/envs/dev_ike.py index fb7d980550..b6cd67dfd8 100644 --- a/lms/envs/dev_ike.py +++ b/lms/envs/dev_ike.py @@ -9,108 +9,9 @@ sessions. Assumes structure: """ from .common import * from .logsettings import get_logger_config +from .dev import * -DEBUG = True -TEMPLATE_DEBUG = True +WIKI_ENABLED = False +MITX_FEATURES['ENABLE_TEXTBOOK'] = False +MITX_FEATURES['ENABLE_DISCUSSION'] = False -MITX_FEATURES['DISABLE_START_DATES'] = True - -WIKI_ENABLED = True - -LOGGING = get_logger_config(ENV_ROOT / "log", - logging_env="dev", - tracking_filename="tracking.log", - debug=True) - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ENV_ROOT / "db" / "mitx.db", - } -} - -CACHES = { - # This is the cache used for most things. Askbot will not work without a - # functioning cache -- it relies on caching to load its settings in places. - # In staging/prod envs, the sessions also live here. - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'mitx_loc_mem_cache', - 'KEY_FUNCTION': 'util.memcache.safe_key', - }, - - # The general cache is what you get if you use our util.cache. It's used for - # things like caching the course.xml file for different A/B test groups. - # We set it to be a DummyCache to force reloading of course.xml in dev. - # In staging environments, we would grab VERSION from data uploaded by the - # push process. - 'general': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - 'KEY_PREFIX': 'general', - 'VERSION': 4, - 'KEY_FUNCTION': 'util.memcache.safe_key', - } -} - -# Dummy secret key for dev -SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' - -################################ OpenID Auth ################################# -MITX_FEATURES['AUTH_USE_OPENID'] = True - -INSTALLED_APPS += ('external_auth',) -INSTALLED_APPS += ('django_openid_auth',) -#INSTALLED_APPS += ('ssl_auth',) - -#MIDDLEWARE_CLASSES += ( -# #'ssl_auth.ssl_auth.NginxProxyHeaderMiddleware', # ssl authentication behind nginx proxy -# ) - -#AUTHENTICATION_BACKENDS = ( -# 'django_openid_auth.auth.OpenIDBackend', -# 'django.contrib.auth.backends.ModelBackend', -# ) - -OPENID_CREATE_USERS = False -OPENID_UPDATE_DETAILS_FROM_SREG = True -OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id' -OPENID_USE_AS_ADMIN_LOGIN = False -#import external_auth.views as edXauth -#OPENID_RENDER_FAILURE = edXauth.edXauth_openid - -################################ DEBUG TOOLBAR ################################# -INSTALLED_APPS += ('debug_toolbar',) -MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) -INTERNAL_IPS = ('127.0.0.1',) - -DEBUG_TOOLBAR_PANELS = ( - 'debug_toolbar.panels.version.VersionDebugPanel', - 'debug_toolbar.panels.timer.TimerDebugPanel', - 'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel', - 'debug_toolbar.panels.headers.HeaderDebugPanel', - 'debug_toolbar.panels.request_vars.RequestVarsDebugPanel', - 'debug_toolbar.panels.sql.SQLDebugPanel', - 'debug_toolbar.panels.signals.SignalDebugPanel', - 'debug_toolbar.panels.logger.LoggingPanel', - -# Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and -# Django=1.3.1/1.4 where requests to views get duplicated (your method gets -# hit twice). So you can uncomment when you need to diagnose performance -# problems, but you shouldn't leave it on. -# 'debug_toolbar.panels.profiling.ProfilingDebugPanel', -) - -############################ FILE UPLOADS (ASKBOT) ############################# -DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' -MEDIA_ROOT = ENV_ROOT / "uploads" -MEDIA_URL = "/static/uploads/" -STATICFILES_DIRS.append(("uploads", MEDIA_ROOT)) -FILE_UPLOAD_TEMP_DIR = ENV_ROOT / "uploads" -FILE_UPLOAD_HANDLERS = ( - 'django.core.files.uploadhandler.MemoryFileUploadHandler', - 'django.core.files.uploadhandler.TemporaryFileUploadHandler', -) - -########################### PIPELINE ################################# - -PIPELINE_SASS_ARGUMENTS = '-r {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT) diff --git a/lms/templates/course_navigation.html b/lms/templates/course_navigation.html index 8bda22148d..84b0c04ca0 100644 --- a/lms/templates/course_navigation.html +++ b/lms/templates/course_navigation.html @@ -14,10 +14,16 @@ def url_class(url):
          • Courseware
          • Course Info
            • % if user.is_authenticated(): +% if settings.MITX_FEATURES.get('ENABLE_TEXTBOOK'):
          • Textbook
            • +% endif +% if settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
          • Discussion
            • +% endif % endif +% if settings.WIKI_ENABLED:
          • Wiki
            • +% endif % if user.is_authenticated():
          • Profile
            • % endif From 553f7046b470c1716619f6d359ca16a55d76b709 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 21:12:56 -0400 Subject: [PATCH 116/124] suggested username for ssl auth is conjoined name with no spaces --- common/djangoapps/external_auth/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py index d00a0a7182..0425f3e158 100644 --- a/common/djangoapps/external_auth/views.py +++ b/common/djangoapps/external_auth/views.py @@ -150,7 +150,7 @@ def edXauth_signup(request, eamap=None): context = {'has_extauth_info': True, 'show_signup_immediately' : True, 'extauth_email': eamap.external_email, - 'extauth_username' : eamap.external_name.split(' ')[0], + 'extauth_username' : eamap.external_name.replace(' ',''), # default - conjoin name, no spaces 'extauth_name': eamap.external_name, } From 76074442869a533ab90ea767db33018640226b8a Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 23:26:31 -0400 Subject: [PATCH 117/124] fix bug: course staff group based on dir_name, not course number --- lms/djangoapps/courseware/courses.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index e568f97f56..31ae3e7fda 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -123,9 +123,9 @@ def course_staff_group_name(course): if type(course)==str: coursename = course else: - coursename = course.metadata.get('course','') - if not coursename: # Fall 2012: not all course.xml have metadata correct yet coursename = course.metadata.get('data_dir','UnknownCourseName') + if not coursename: # Fall 2012: not all course.xml have metadata correct yet + coursename = course.metadata.get('course','') return 'staff_%s' % coursename def has_staff_access_to_course(user,course): @@ -138,8 +138,8 @@ def has_staff_access_to_course(user,course): if user.is_staff: return True user_groups = [x[1] for x in user.groups.values_list()] # note this is the Auth group, not UserTestGroup - log.debug('user is in groups %s' % user_groups) staff_group = course_staff_group_name(course) + log.debug('course %s user %s groups %s' % (staff_group, user, user_groups)) if staff_group in user_groups: return True return False From a46a37d1c0445bbdec154077b0a976119e818e77 Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 13:58:42 -0400 Subject: [PATCH 118/124] log.debug -> log.exception; revert log change in xml_module --- common/djangoapps/track/views.py | 2 +- common/lib/xmodule/xmodule/xml_module.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/djangoapps/track/views.py b/common/djangoapps/track/views.py index 1123b8d30a..31878bee26 100644 --- a/common/djangoapps/track/views.py +++ b/common/djangoapps/track/views.py @@ -27,7 +27,7 @@ def log_event(event): try: tldat.save() except Exception as err: - log.debug(err) + log.exception(err) def user_track(request): try: # TODO: Do the same for many of the optional META parameters diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index fbb17fd236..7a12ed869d 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -9,7 +9,7 @@ from fs.errors import ResourceNotFoundError import os import sys -log = logging.getLogger('mitx.' + __name__) +log = logging.getLogger(__name__) _AttrMapBase = namedtuple('_AttrMap', 'metadata_key to_metadata from_metadata') From 3484f5382cb593386f3871848060de7e29fab6ee Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:10:00 -0400 Subject: [PATCH 119/124] isinstance instead of type --- lms/djangoapps/courseware/courses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 31ae3e7fda..8193988d67 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -120,7 +120,7 @@ def course_staff_group_name(course): ''' course should be either a CourseDescriptor instance, or a string (the .course entry of a Location) ''' - if type(course)==str: + if isinstance(course,str): coursename = course else: coursename = course.metadata.get('data_dir','UnknownCourseName') From 871ed954be3638326c8cd472bd5f573973ef5790 Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:16:11 -0400 Subject: [PATCH 120/124] ACCESS_REQUIRE_STAFF_FOR_COURSE default False in lms.envs.dev --- lms/envs/dev.py | 2 +- lms/envs/dev_ike.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 204fcec04b..bc5b621b32 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -61,7 +61,7 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' ################################ LMS Migration ################################# MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True -MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True # require that user be in the staff_* group to be able to enroll +MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False # require that user be in the staff_* group to be able to enroll LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1'] diff --git a/lms/envs/dev_ike.py b/lms/envs/dev_ike.py index b6cd67dfd8..2256decb46 100644 --- a/lms/envs/dev_ike.py +++ b/lms/envs/dev_ike.py @@ -14,4 +14,5 @@ from .dev import * WIKI_ENABLED = False MITX_FEATURES['ENABLE_TEXTBOOK'] = False MITX_FEATURES['ENABLE_DISCUSSION'] = False +MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True # require that user be in the staff_* group to be able to enroll From 6f894c816cef3f7c6ee7541ffd1ffd31a77a8cce Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:19:45 -0400 Subject: [PATCH 121/124] use jquery for error msg in course_about --- lms/templates/portal/course_about.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html index bdea0a47d1..c2c1e3b747 100644 --- a/lms/templates/portal/course_about.html +++ b/lms/templates/portal/course_about.html @@ -20,7 +20,7 @@ if(json.success) { location.href="${reverse('dashboard')}"; }else{ - document.getElementById('register_message').innerHTML = "

            " + json.error + "

            "; + $('#register_message).html("

            " + json.error + "

            ") } }); })(this) From b1ddff838c15f85d3616582b94e2e8a8725ad466 Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:23:55 -0400 Subject: [PATCH 122/124] add comment about course start date logic --- lms/templates/portal/course_about.html | 1 + 1 file changed, 1 insertion(+) diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html index c2c1e3b747..a3bf8dd755 100644 --- a/lms/templates/portal/course_about.html +++ b/lms/templates/portal/course_about.html @@ -63,6 +63,7 @@ %if user.is_authenticated(): %if registered: <% + ## TODO: move this logic into a view if course.has_started() or settings.MITX_FEATURES['DISABLE_START_DATES']: course_target = reverse('info', args=[course.id]) else: From 8a1747770a1bae1d3274692aee856288425d1067 Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:30:53 -0400 Subject: [PATCH 123/124] redirect to course_about page if hit internal course page unregistered for --- lms/djangoapps/courseware/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 5db3bcf91a..f014e3fcb5 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -168,7 +168,7 @@ def index(request, course_id, chapter=None, section=None, registered = registered_for_course(course, request.user) if not registered: log.debug('User %s tried to view course %s but is not enrolled' % (request.user,course.location.url())) - return redirect('/') + return redirect(reverse('about_course', args=[course.id])) try: context = { From 9805ed89620f6567c5237ae2ab8b7688cf04903c Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:37:17 -0400 Subject: [PATCH 124/124] cleanup syntax, split long if into two lines --- common/djangoapps/track/views.py | 2 +- lms/djangoapps/courseware/module_render.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/djangoapps/track/views.py b/common/djangoapps/track/views.py index 31878bee26..b5f9c54665 100644 --- a/common/djangoapps/track/views.py +++ b/common/djangoapps/track/views.py @@ -23,7 +23,7 @@ def log_event(event): log.info(event_str[:settings.TRACK_MAX_EVENT]) if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): event['time'] = dateutil.parser.parse(event['time']) - tldat = TrackingLog(**dict([(x,event[x]) for x in LOGFIELDS])) + tldat = TrackingLog(**dict( (x,event[x]) for x in LOGFIELDS )) try: tldat.save() except Exception as err: diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index cdb9dc40f3..b6ba381a26 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -190,9 +190,9 @@ def get_module(user, request, location, student_module_cache, position=None): module.metadata['data_dir'] ) - if (settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and - (user.is_staff or has_staff_access_to_course(user, module.location.course))): - module.get_html = add_histogram(module.get_html, module) + if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF'): + if has_staff_access_to_course(user, module.location.course): + module.get_html = add_histogram(module.get_html, module) # If StudentModule for this instance wasn't already in the database, # and this isn't a guest user, create it.