Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge master to here.

Browse Source
This commit is contained in:
Ned Batchelder
2013-05-16 14:21:51 -04:00
parent bbd1d8d09e fca63f065f
commit e947218285
275 changed files with 8700 additions and 4441 deletions
Download Patch File Download Diff File Expand all files Collapse all files

456
scripts/create-dev-env.sh Executable file
View File

@@ -0,0 +1,456 @@
#!/usr/bin/env bash
set -e
# posix compliant sanity check
if [ -z $BASH ] || [ $BASH = "/bin/sh" ]; then
echo "Please use the bash interpreter to run this script"
exit 1
fi
trap "ouch" ERR
ouch() {
printf '\E[31m'
cat<<EOL
!! ERROR !!
The last command did not complete successfully,
For more details or trying running the
script again with the -v flag.
Output of the script is recorded in $LOG
EOL
printf '\E[0m'
}
error() {
printf '\E[31m'; echo "$@"; printf '\E[0m'
}
output() {
printf '\E[36m'; echo "$@"; printf '\E[0m'
}
usage() {
cat<<EO
Usage: $PROG [-c] [-v] [-h]
-c compile scipy and numpy
-s give access to global site-packages for virtualenv
-v set -x + spew
-h this
EO
info
}
info() {
cat<<EO
MITx base dir : $BASE
Python virtualenv dir : $PYTHON_DIR
Ruby RVM dir : $RUBY_DIR
Ruby ver : $RUBY_VER
EO
}
clone_repos() {
cd "$BASE"
if [[ -d "$BASE/mitx/.git" ]]; then
output "Pulling mitx"
cd "$BASE/mitx"
git pull
else
output "Cloning mitx"
if [[ -d "$BASE/mitx" ]]; then
mv "$BASE/mitx" "${BASE}/mitx.bak.$$"
fi
git clone git@github.com:MITx/mitx.git
fi
# By default, dev environments start with a copy of 6.002x
cd "$BASE"
mkdir -p "$BASE/data"
REPO="content-mit-6002x"
if [[ -d "$BASE/data/$REPO/.git" ]]; then
output "Pulling $REPO"
cd "$BASE/data/$REPO"
git pull
else
output "Cloning $REPO"
if [[ -d "$BASE/data/$REPO" ]]; then
mv "$BASE/data/$REPO" "${BASE}/data/$REPO.bak.$$"
fi
cd "$BASE/data"
git clone git@github.com:MITx/$REPO
fi
}
### START
PROG=${0##*/}
# Adjust this to wherever you'd like to place the codebase
BASE="${PROJECT_HOME:-$HOME}/mitx_all"
# Use a sensible default (~/.virtualenvs) for your Python virtualenvs
# unless you've already got one set up with virtualenvwrapper.
PYTHON_DIR=${WORKON_HOME:-"$HOME/.virtualenvs"}
# RVM defaults its install to ~/.rvm, but use the overridden rvm_path
# if that's what's preferred.
RUBY_DIR=${rvm_path:-"$HOME/.rvm"}
LOG="/var/tmp/install-$(date +%Y%m%d-%H%M%S).log"
# Make sure the user's not about to do anything dumb
if [[ $EUID -eq 0 ]]; then
error "This script should not be run using sudo or as the root user"
usage
exit 1
fi
# If in an existing virtualenv, bail
if [[ "x$VIRTUAL_ENV" != "x" ]]; then
envname=`basename $VIRTUAL_ENV`
error "Looks like you're already in the \"$envname\" virtual env."
error "Run \`deactivate\` and then re-run this script."
usage
exit 1
fi
# Read arguments
ARGS=$(getopt "cvhs" "$*")
if [[ $? != 0 ]]; then
usage
exit 1
fi
eval set -- "$ARGS"
while true; do
case $1 in
-c)
compile=true
shift
;;
-s)
systempkgs=true
shift
;;
-v)
set -x
verbose=true
shift
;;
-h)
usage
exit 0
;;
--)
shift
break
;;
esac
done
cat<<EO
This script will setup a local MITx environment, this
includes
* Django
* A local copy of Python and library dependencies
* A local copy of Ruby and library dependencies
It will also attempt to install operating system dependencies
with apt(debian) or brew(OSx).
To compile scipy and numpy from source use the -c option
!!! Do not run this script from an existing virtualenv !!!
If you are in a ruby/python virtualenv please start a new
shell.
EO
info
output "Press return to begin or control-C to abort"
read dummy
# Log all stdout and stderr
exec > >(tee $LOG)
exec 2>&1
# Install basic system requirements
mkdir -p $BASE
case `uname -s` in
[Ll]inux)
command -v lsb_release &>/dev/null || {
error "Please install lsb-release."
exit 1
}
distro=`lsb_release -cs`
case $distro in
maya|lisa|natty|oneiric|precise|quantal)
sudo apt-get install git
;;
*)
error "Unsupported distribution - $distro"
exit 1
;;
esac
;;
Darwin)
if [[ ! -w /usr/local ]]; then
cat<<EO
You need to be able to write to /usr/local for
the installation of brew and brew packages.
Either make sure the group you are in (most likely 'staff')
can write to that directory or simply execute the following
and re-run the script:
$ sudo chown -R $USER /usr/local
EO
exit 1
fi
command -v brew &>/dev/null || {
output "Installing brew"
/usr/bin/ruby <(curl -fsSkL raw.github.com/mxcl/homebrew/go)
}
command -v git &>/dev/null || {
output "Installing git"
brew install git
}
;;
*)
error "Unsupported platform"
exit 1
;;
esac
# Clone MITx repositories
clone_repos
# Install system-level dependencies
bash $BASE/mitx/install-system-req.sh
output "Installing RVM, Ruby, and required gems"
# If we're not installing RVM in the default location, then we'll do some
# funky stuff to make sure that we load in the RVM stuff properly on login.
if [ "$HOME/.rvm" != $RUBY_DIR ]; then
if ! grep -q "export rvm_path=$RUBY_DIR" ~/.rvmrc; then
if [[ -f $HOME/.rvmrc ]]; then
output "Copying existing .rvmrc to .rvmrc.bak"
cp $HOME/.rvmrc $HOME/.rvmrc.bak
fi
output "Creating $HOME/.rvmrc so rvm uses $RUBY_DIR"
echo "export rvm_path=$RUBY_DIR" > $HOME/.rvmrc
fi
fi
curl -sL get.rvm.io | bash -s -- --version 1.15.7
# Ensure we have RVM available as a shell function so that it can mess
# with the environment and set everything up properly. The RVM install
# process adds this line to login scripts, so this shouldn't be necessary
# for the user to do each time.
if [[ `type -t rvm` != "function" ]]; then
source $RUBY_DIR/scripts/rvm
fi
# Ruby doesn't like to build with clang, which is the default on OS X, so
# use gcc instead. This may not work, since if your gcc was installed with
# XCode 4.2 or greater, you have an LLVM-based gcc, which also doesn't
# always play nicely with Ruby, though it seems to be better than clang.
# You may have to install apple-gcc42 using Homebrew if this doesn't work.
# See `rvm requirements` for more information.
case `uname -s` in
Darwin)
export CC=gcc
;;
esac
# Let the repo override the version of Ruby to install
if [[ -r $BASE/mitx/.ruby-version ]]; then
RUBY_VER=`cat $BASE/mitx/.ruby-version`
fi
# Current stable version of RVM (1.19.0) requires the following to build Ruby:
#
# autoconf automake libtool pkg-config libyaml libxml2 libxslt libksba openssl
#
# If we decide to upgrade from the current version (1.15.7), can run
#
# LESS="-E" rvm install $RUBY_VER --autolibs=3 --with-readline
#
# to have RVM look for a package manager like Homebrew and install any missing
# libs automatically. RVM's --autolibs flag defaults to 2, which will fail if
# any required libs are missing.
LESS="-E" rvm install $RUBY_VER --with-readline
# Create the "mitx" gemset
rvm use "$RUBY_VER@mitx" --create
output "Installing gem bundler"
gem install bundler
output "Installing ruby packages"
bundle install --gemfile $BASE/mitx/Gemfile
# Install Python virtualenv
output "Installing python virtualenv"
case `uname -s` in
Darwin)
# Add brew's path
PATH=/usr/local/share/python:/usr/local/bin:$PATH
;;
esac
# virtualenvwrapper uses the $WORKON_HOME env var to determine where to place
# virtualenv directories. Make sure it matches the selected $PYTHON_DIR.
export WORKON_HOME=$PYTHON_DIR
# Load in the mkvirtualenv function if needed
if [[ `type -t mkvirtualenv` != "function" ]]; then
source `which virtualenvwrapper.sh`
fi
# Create MITx virtualenv and link it to repo
# virtualenvwrapper automatically sources the activation script
if [[ $systempkgs ]]; then
mkvirtualenv -a "$BASE/mitx" --system-site-packages mitx || {
error "mkvirtualenv exited with a non-zero error"
return 1
}
else
# default behavior for virtualenv>1.7 is
# --no-site-packages
mkvirtualenv -a "$BASE/mitx" mitx || {
error "mkvirtualenv exited with a non-zero error"
return 1
}
fi
# compile numpy and scipy if requested
NUMPY_VER="1.6.2"
SCIPY_VER="0.10.1"
if [[ -n $compile ]]; then
output "Downloading numpy and scipy"
curl -sL -o numpy.tar.gz http://downloads.sourceforge.net/project/numpy/NumPy/${NUMPY_VER}/numpy-${NUMPY_VER}.tar.gz
curl -sL -o scipy.tar.gz http://downloads.sourceforge.net/project/scipy/scipy/${SCIPY_VER}/scipy-${SCIPY_VER}.tar.gz
tar xf numpy.tar.gz
tar xf scipy.tar.gz
rm -f numpy.tar.gz scipy.tar.gz
output "Compiling numpy"
cd "$BASE/numpy-${NUMPY_VER}"
python setup.py install
output "Compiling scipy"
cd "$BASE/scipy-${SCIPY_VER}"
python setup.py install
cd "$BASE"
rm -rf numpy-${NUMPY_VER} scipy-${SCIPY_VER}
fi
case `uname -s` in
Darwin)
# on mac os x get the latest distribute and pip
curl http://python-distribute.org/distribute_setup.py | python
pip install -U pip
# need latest pytz before compiling numpy and scipy
pip install -U pytz
pip install numpy
# scipy needs cython
pip install cython
# fixes problem with scipy on 10.8
pip install -e git+https://github.com/scipy/scipy#egg=scipy-dev
;;
esac
output "Installing MITx pre-requirements"
pip install -r $BASE/mitx/pre-requirements.txt
output "Installing MITx requirements"
# Need to be in the mitx dir to get the paths to local modules right
cd $BASE/mitx
pip install -r requirements.txt
mkdir "$BASE/log" || true
mkdir "$BASE/db" || true
# Configure Git
output "Fixing your git default settings"
git config --global push.default current
### DONE
cat<<END
Success!!
To start using Django you will need to activate the local Python
and Ruby environments. Ensure the following lines are added to your
login script, and source your login script if needed:
source `which virtualenvwrapper.sh`
source $RUBY_DIR/scripts/rvm
Then, every time you're ready to work on the project, just run
$ workon mitx
To initialize Django
$ rake django-admin[syncdb]
$ rake django-admin[migrate]
To start the Django on port 8000
$ rake lms
Or to start Django on a different <port#>
$ rake django-admin[runserver,lms,dev,<port#>]
If the Django development server starts properly you
should see:
Development server is running at http://127.0.0.1:<port#>/
Quit the server with CONTROL-C.
Connect your browser to http://127.0.0.1:<port#> to
view the Django site.
END
exit 0

108
scripts/install-system-req.sh Executable file
View File

@@ -0,0 +1,108 @@
#!/usr/bin/env bash
# posix compliant sanity check
if [ -z $BASH ] || [ $BASH = "/bin/sh" ]; then
echo "Please use the bash interpreter to run this script"
exit 1
fi
error() {
printf '\E[31m'; echo "$@"; printf '\E[0m'
}
output() {
printf '\E[36m'; echo "$@"; printf '\E[0m'
}
### START
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
BREW_FILE=$DIR/"brew-formulas.txt"
APT_REPOS_FILE=$DIR/"apt-repos.txt"
APT_PKGS_FILE=$DIR/"apt-packages.txt"
case `uname -s` in
[Ll]inux)
command -v lsb_release &>/dev/null || {
error "Please install lsb-release."
exit 1
}
distro=`lsb_release -cs`
case $distro in
maya|lisa|natty|oneiric|precise|quantal)
output "Installing Ubuntu requirements"
# DEBIAN_FRONTEND=noninteractive is required for silent mysql-server installation
export DEBIAN_FRONTEND=noninteractive
# add repositories
cat $APT_REPOS_FILE | xargs -n 1 sudo add-apt-repository -y
sudo apt-get -y update
# install packages listed in APT_PKGS_FILE
cat $APT_PKGS_FILE | xargs sudo apt-get -y install
;;
*)
error "Unsupported distribution - $distro"
exit 1
;;
esac
;;
Darwin)
if [[ ! -w /usr/local ]]; then
cat<<EO
You need to be able to write to /usr/local for
the installation of brew and brew packages.
Either make sure the group you are in (most likely 'staff')
can write to that directory or simply execute the following
and re-run the script:
$ sudo chown -R $USER /usr/local
EO
exit 1
fi
output "Installing OSX requirements"
if [[ ! -r $BREW_FILE ]]; then
error "$BREW_FILE does not exist, needed to install brew"
exit 1
fi
# brew errors if the package is already installed
for pkg in $(cat $BREW_FILE); do
grep $pkg <(brew list) &>/dev/null || {
output "Installing $pkg"
brew install $pkg
}
done
# paths where brew likes to install python scripts
PATH=/usr/local/share/python:/usr/local/bin:$PATH
command -v pip &>/dev/null || {
output "Installing pip"
easy_install pip
}
if ! grep -Eq ^1.7 <(virtualenv --version 2>/dev/null); then
output "Installing virtualenv >1.7"
pip install 'virtualenv>1.7' virtualenvwrapper
fi
command -v coffee &>/dev/null || {
output "Installing coffee script"
curl --insecure https://npmjs.org/install.sh | sh
npm install -g coffee-script
}
;;
*)
error "Unsupported platform"
exit 1
;;
esac

1
scripts/run.sh Executable file
View File

@@ -0,0 +1 @@
rake lms

50
scripts/run_watch_data.py Executable file
View File

@@ -0,0 +1,50 @@
#! /usr/bin/env python
# This script requires that you have watchdog installed. You can install
# watchdog via 'pip install watchdog'
import sys
import time
import logging
import os
from subprocess import Popen
from signal import SIGTERM
from watchdog.observers import Observer
from watchdog.events import LoggingEventHandler, FileSystemEventHandler
# To watch more (or more specific) directories, change WATCH_DIRS to include the
# directories you want to watch. Note that this is recursive. If you want to
# watch fewer or more extensions, you can change EXTENSIONS. To watch all
# extensions, add "*" to EXTENSIONS.
WATCH_DIRS = ["../data", "common/lib/xmodule/xmodule/js", "common/lib/xmodule/xmodule/css"]
EXTENSIONS = ["*", "xml", "js", "css", "coffee", "scss", "html"]
WATCH_DIRS = [os.path.abspath(os.path.normpath(dir)) for dir in WATCH_DIRS]
class DjangoEventHandler(FileSystemEventHandler):
def __init__(self, process):
FileSystemEventHandler.__init__(self)
self.process = process
def on_any_event(self, event):
for extension in EXTENSIONS:
if event.src_path.endswith(extension) or extension == "*":
print "%s changed: restarting server." % event.src_path
os.system("touch lms/__init__.py")
break
if __name__ == "__main__":
event_handler = DjangoEventHandler(Popen(['rake', 'lms']))
observer = Observer()
for dir in WATCH_DIRS:
observer.schedule(event_handler, dir, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()

74
scripts/runone.py Executable file
View File

@@ -0,0 +1,74 @@
#!/usr/bin/env python
import argparse
import os
import sys
# I want this:
# ERROR: test_update_and_fetch (mitx.cms.djangoapps.contentstore.tests.test_course_settings.CourseDetailsViewTest)
# to become:
# test --settings=cms.envs.test --pythonpath=. -s cms/djangoapps/contentstore/tests/test_course_settings.py:CourseDetailsViewTest.test_update_and_fetch
def find_full_path(path_to_file):
"""Find the full path where we only have a relative path from somewhere in the tree."""
for subdir, dirs, files in os.walk("."):
full = os.path.relpath(os.path.join(subdir, path_to_file))
if os.path.exists(full):
return full
def main(argv):
parser = argparse.ArgumentParser(description="Run just one test")
parser.add_argument('--nocapture', '-s', action='store_true', help="Don't capture stdout (any stdout output will be printed immediately)")
parser.add_argument('words', metavar="WORDS", nargs='+', help="The description of a test failure, like 'ERROR: test_set_missing_field (courseware.tests.test_model_data.TestStudentModuleStorage)'")
args = parser.parse_args(argv)
words = []
# Collect all the words, ignoring what was quoted together, and get rid of parens.
for argword in args.words:
words.extend(w.strip("()") for w in argword.split())
# If it starts with "ERROR:" or "FAIL:", just ignore that.
if words[0].endswith(':'):
del words[0]
test_method = words[0]
test_path = words[1].split('.')
if test_path[0] == 'mitx':
del test_path[0]
test_class = test_path[-1]
del test_path[-1]
test_py_path = "%s.py" % ("/".join(test_path))
test_py_path = find_full_path(test_py_path)
test_spec = "%s:%s.%s" % (test_py_path, test_class, test_method)
settings = None
if test_py_path.startswith('cms'):
settings = 'cms.envs.test'
elif test_py_path.startswith('lms'):
settings = 'lms.envs.test'
if settings:
# Run as a django test suite
from django.core import management
django_args = ["django-admin.py", "test", "--pythonpath=."]
django_args.append("--settings=%s" % settings)
if args.nocapture:
django_args.append("-s")
django_args.append(test_spec)
print " ".join(django_args)
management.execute_from_command_line(django_args)
else:
# Run as a nose test suite
import nose.core
nose_args = ["nosetests"]
if args.nocapture:
nose_args.append("-s")
nose_args.append(test_spec)
print " ".join(nose_args)
nose.core.main(argv=nose_args)
if __name__ == "__main__":
main(sys.argv[1:])

46
scripts/setup-test-dirs.sh Executable file
View File

@@ -0,0 +1,46 @@
#!/usr/bin/env bash
# Create symlinks from ~/mitx_all/data or $ROOT/data, with root passed as first arg
# to all the test courses in mitx/common/test/data/
# posix compliant sanity check
if [ -z $BASH ] || [ $BASH = "/bin/sh" ]; then
echo "Please use the bash interpreter to run this script"
exit 1
fi
ROOT="${1:-$HOME/mitx_all}"
if [[ ! -d "$ROOT" ]]; then
echo "'$ROOT' is not a directory"
exit 1
fi
if [[ ! -d "$ROOT/mitx" ]]; then
echo "'$ROOT' is not the root mitx_all directory"
exit 1
fi
if [[ ! -d "$ROOT/data" ]]; then
echo "'$ROOT' is not the root mitx_all directory"
exit 1
fi
echo "ROOT is $ROOT"
cd $ROOT/data
for course in $(/bin/ls ../mitx/common/test/data/)
do
# Get rid of the symlink if it already exists
if [[ -L "$course" ]]; then
echo "Removing link to '$course'"
rm -f $course
fi
echo "Make link to '$course'"
# Create it
ln -s "../mitx/common/test/data/$course"
done
# go back to where we came from
cd -