From 1736eb212532479f290223db4ab02b61fb5abd62 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Fri, 25 May 2012 16:15:12 -0400 Subject: [PATCH 01/10] first commit --- create-dev-env.sh | 221 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100755 create-dev-env.sh diff --git a/create-dev-env.sh b/create-dev-env.sh new file mode 100755 index 0000000000..35653c38ba --- /dev/null +++ b/create-dev-env.sh @@ -0,0 +1,221 @@ +#!/bin/bash +set -e + +error() { + printf '\E[36m'; echo "$@"; printf '\E[0m' +} +output() { + printf '\E[36m'; echo "$@"; printf '\E[0m' +} +usage() { + cat< $HOME/.rvmrc +fi +mkdir -p $BASE +rm -f $LOG +case `uname -s` in + [Ll]inux) + command -v lsb_release &>/dev/null || { + error "Please install lsb-release." + exit 1 + } + distro=`lsb_release -cs` + case $distro in + lisa|natty|oneiric|precise) + output "Installing ubuntu requirements" + sudo apt-get -y update + sudo apt-get -y install $APT_PKGS + ;; + *) + error "Unsupported distribution - $distro" + exit 1 + ;; + esac + ;; + Darwin) + command -v brew &>/dev/null || { + output "Installing brew" + /usr/bin/ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)" >>$LOG + } + output "Installing OSX requirements" + # brew errors if the package is already installed + for pkg in $BREW_PKGS; do + grep $pkg <(brew list) &>/dev/null || { + output "Installing $pkg" + brew install $pkg >>$LOG + } + done + command -v pip &>/dev/null || { + output "Installing pip" + sudo easy_install pip >>$LOG + } + command -v virtualenv &>/dev/null || { + output "Installing virtualenv" + sudo pip install virtualenv virtualenvwrapper >> $LOG + } + ;; + *) + error "Unsupported platform" + exit 1 + ;; +esac +output "Installing rvm and ruby" +curl -sL get.rvm.io | bash -s stable +source $RUBY_DIR/scripts/rvm +rvm install $RUBY_VER +virtualenv "$PYTHON_DIR" +source $PYTHON_DIR/bin/activate +output "Installing ruby packages - $GEM_PKGS" +gem install $GEM_PKGS +cd "$BASE" +output "Cloning mitx, askbot and data repos" +if [[ -d "$BASE/mitx" ]]; then + mv "$BASE/mitx" "${BASE}/mitx.bak.$$" +fi +git clone git@github.com:MITx/mitx.git >>$LOG +if [[ -d "$BASE/askbot-devel" ]]; then + mv "$BASE/askbot-devel" "${BASE}/askbot-devel.bak.$$" +fi +git clone git@github.com:MITx/askbot-devel >>$LOG +if [[ -d "$BASE/data" ]]; then + mv "$BASE/data" "${BASE}/data.bak.$$" +fi +hg clone ssh://hg-content@gp.mitx.mit.edu/data >>$LOG + +if [[ -n $compile ]]; then + output "Downloading numpy and scipy" + curl -sL -o numpy.tar.gz http://downloads.sourceforge.net/project/numpy/NumPy/${NUMPY_VER}/numpy-${NUMPY_VER}.tar.gz + curl -sL -o scipy.tar.gz http://downloads.sourceforge.net/project/scipy/scipy/${SCIPY_VER}/scipy-${SCIPY_VER}.tar.gz + tar xf numpy.tar.gz + tar xf scipy.tar.gz + rm -f numpy.tar.gz scipy.tar.gz + output "Compiling numpy" + cd "$BASE/numpy-${NUMPY_VER}" + python setup.py install >>$LOG 2>&1 + output "Compiling scipy" + cd "$BASE/scipy-${SCIPY_VER}" + python setup.py install >>$LOG 2>&1 + cd "$BASE" + rm -rf numpy-${NUMPY_VER} scipy-${SCIPY_VER} +fi + +output "Installing askbot requirements" +pip install -r askbot-devel/askbot_requirements.txt >>$LOG +pip install -r askbot-devel/askbot_requirements_dev.txt >>$LOG +output "Installing MITx requirements" +pip install -r mitx/pre-requirements.txt >> $LOG +pip install -r mitx/requirements.txt >>$LOG + +mkdir "$BASE/log" +mkdir "$BASE/db" + +cat< Date: Fri, 25 May 2012 16:22:09 -0400 Subject: [PATCH 02/10] mkdirs fail silently --- create-dev-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index 35653c38ba..1f2ddd4376 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -195,8 +195,8 @@ output "Installing MITx requirements" pip install -r mitx/pre-requirements.txt >> $LOG pip install -r mitx/requirements.txt >>$LOG -mkdir "$BASE/log" -mkdir "$BASE/db" +mkdir "$BASE/log" || true +mkdir "$BASE/db" || true cat< Date: Fri, 25 May 2012 16:37:22 -0400 Subject: [PATCH 03/10] updating gem install list --- create-dev-env.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index 1f2ddd4376..6c4d1cae8e 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -26,9 +26,7 @@ info() { MITx base dir : $BASE Python dir : $PYTHON_DIR Ruby dir : $RUBY_DIR - Ruby ver : $RUBY_VER - Ruby libraries : $GEM_PKGS EO @@ -45,7 +43,6 @@ SCIPY_VER="0.10.1" LOG="/var/tmp/install.log" BREW_PKGS="readline sqlite gdbm pkg-config gfortran mercurial python yuicompressor " APT_PKGS="curl git mercurial python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor" -GEM_PKGS="sass bourbon" ARGS=$(getopt "cvh" "$*") if [[ $? != 0 ]]; then @@ -154,8 +151,10 @@ source $RUBY_DIR/scripts/rvm rvm install $RUBY_VER virtualenv "$PYTHON_DIR" source $PYTHON_DIR/bin/activate -output "Installing ruby packages - $GEM_PKGS" -gem install $GEM_PKGS +output "Installing ruby packages" +gem install --version '0.8.3' rake +gem install --version '3.1.15' sass +gem install --version '1.3.6' bourbon cd "$BASE" output "Cloning mitx, askbot and data repos" if [[ -d "$BASE/mitx" ]]; then From 3b3cf5b7a08e7caeca1f1269754016dfa62b53f2 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 29 May 2012 09:33:42 -0400 Subject: [PATCH 04/10] added node/coffeescript to dependency list --- create-dev-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index 6c4d1cae8e..b207dbdeb3 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -41,8 +41,8 @@ RUBY_VER="1.9.3" NUMPY_VER="1.6.2" SCIPY_VER="0.10.1" LOG="/var/tmp/install.log" -BREW_PKGS="readline sqlite gdbm pkg-config gfortran mercurial python yuicompressor " -APT_PKGS="curl git mercurial python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor" +BREW_PKGS="readline sqlite gdbm pkg-config gfortran mercurial python yuicompressor node" +APT_PKGS="curl git mercurial python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript" ARGS=$(getopt "cvh" "$*") if [[ $? != 0 ]]; then From d011ab99ada2ddcba4f9189010c84a9c616ccc68 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 30 May 2012 11:43:33 -0400 Subject: [PATCH 05/10] added root user check --- create-dev-env.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index b207dbdeb3..0a3d7258eb 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -2,7 +2,7 @@ set -e error() { - printf '\E[36m'; echo "$@"; printf '\E[0m' + printf '\E[31m'; echo "$@"; printf '\E[0m' } output() { printf '\E[36m'; echo "$@"; printf '\E[0m' @@ -29,10 +29,10 @@ info() { Ruby ver : $RUBY_VER EO - - - } + + + PROG=${0##*/} BASE="$HOME/mitx_all" PYTHON_DIR="$BASE/python" @@ -44,6 +44,12 @@ LOG="/var/tmp/install.log" BREW_PKGS="readline sqlite gdbm pkg-config gfortran mercurial python yuicompressor node" APT_PKGS="curl git mercurial python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript" +if [[ $EUID -eq 0 ]]; then + error "This script should not be run using sudo or as the root user" + usage + exit 1 +fi + ARGS=$(getopt "cvh" "$*") if [[ $? != 0 ]]; then usage From 908e9c0427b0c14ca166ca81fe0acea3990d1f75 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 31 May 2012 10:41:45 -0400 Subject: [PATCH 06/10] no longer redirecting STDOUT when running the brew installer --- create-dev-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create-dev-env.sh b/create-dev-env.sh index 0a3d7258eb..1b3b932dd5 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -127,7 +127,7 @@ case `uname -s` in Darwin) command -v brew &>/dev/null || { output "Installing brew" - /usr/bin/ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)" >>$LOG + /usr/bin/ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)" } output "Installing OSX requirements" # brew errors if the package is already installed From 26ce972cea05bf75e21d1c7443b87754e7a174e2 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 31 May 2012 11:39:55 -0400 Subject: [PATCH 07/10] adding brew-formulas, updating dev env script --- brew-formulas.txt | 9 +++++++++ create-dev-env.sh | 34 ++++++++++++++++++++++------------ 2 files changed, 31 insertions(+), 12 deletions(-) create mode 100644 brew-formulas.txt diff --git a/brew-formulas.txt b/brew-formulas.txt new file mode 100644 index 0000000000..5f76861eda --- /dev/null +++ b/brew-formulas.txt @@ -0,0 +1,9 @@ +readline +sqlite +gdbm +pkg-config +gfortran +mercurial +python +yuicompressor +node diff --git a/create-dev-env.sh b/create-dev-env.sh index 1b3b932dd5..1b3c272bb3 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -41,7 +41,7 @@ RUBY_VER="1.9.3" NUMPY_VER="1.6.2" SCIPY_VER="0.10.1" LOG="/var/tmp/install.log" -BREW_PKGS="readline sqlite gdbm pkg-config gfortran mercurial python yuicompressor node" +BREW_PKGS="$BASE/mit/brew-formulas.txt" APT_PKGS="curl git mercurial python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript" if [[ $EUID -eq 0 ]]; then @@ -92,6 +92,10 @@ cat</dev/null || { output "Installing $pkg" brew install $pkg >>$LOG @@ -151,16 +155,7 @@ case `uname -s` in exit 1 ;; esac -output "Installing rvm and ruby" -curl -sL get.rvm.io | bash -s stable -source $RUBY_DIR/scripts/rvm -rvm install $RUBY_VER -virtualenv "$PYTHON_DIR" -source $PYTHON_DIR/bin/activate -output "Installing ruby packages" -gem install --version '0.8.3' rake -gem install --version '3.1.15' sass -gem install --version '1.3.6' bourbon + cd "$BASE" output "Cloning mitx, askbot and data repos" if [[ -d "$BASE/mitx" ]]; then @@ -176,6 +171,21 @@ if [[ -d "$BASE/data" ]]; then fi hg clone ssh://hg-content@gp.mitx.mit.edu/data >>$LOG + +output "Installing rvm and ruby" +curl -sL get.rvm.io | bash -s stable +source $RUBY_DIR/scripts/rvm +rvm install $RUBY_VER +virtualenv "$PYTHON_DIR" +source $PYTHON_DIR/bin/activate +output "Installing gem bundler" +gem install bundler +output "Installing ruby packages" +cd $BASE/mitx +bundle install + +cd $BASE + if [[ -n $compile ]]; then output "Downloading numpy and scipy" curl -sL -o numpy.tar.gz http://downloads.sourceforge.net/project/numpy/NumPy/${NUMPY_VER}/numpy-${NUMPY_VER}.tar.gz From a385dc4d2e5a53ecb79ad4b4bf612745a35d09d3 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 31 May 2012 14:34:50 -0400 Subject: [PATCH 08/10] added install.txt, updated env script for better error handling --- install.txt | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 install.txt diff --git a/install.txt b/install.txt new file mode 100644 index 0000000000..70c8655306 --- /dev/null +++ b/install.txt @@ -0,0 +1,74 @@ +This document describes how to set up the MITx development environment +for both Linux (Ubuntu) and MacOS (OSX Lion). + +There is also a script "create-dev-env.sh" that automates these steps. + +1) Make an mitx_all directory and clone the repos + (download and install git and mercurial if you don't have them already) + + mkdir ~/mitx_all + cd ~/mitx_all + git clone git@github.com:MITx/mitx.git + git clone git@github.com:MITx/askbot-devel + hg clone ssh://hg-content@gp.mitx.mit.edu/data + +2) Install OSX dependencies (Mac users only) + + a) Install the brew utility if necessary + /usr/bin/ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)" + + b) Install the brew package list + cat ~/mitx_all/mitx/brew-formulas.txt | xargs brew install + + c) Install python pip if necessary + sudo easy_install pip + + d) Install python virtualenv if necessary + sudo pip install virtualenv virtualenvwrapper + +3) Install Ubuntu dependencies (Linux users only) + + sudo apt-get install curl python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript + + +4) Install rvm, ruby, and libraries + + echo "export rvm_path=$HOME/mitx_all/ruby" > $HOME/.rvmrc + curl -sL get.rvm.io | bash -s stable + source ~/mitx_all/ruby/scripts/rvm + rvm install 1.9.3 + gem install bundler + cd ~/mitx_all/mitx + bundle install + +5) Install python libraries + + source ~/mitx_all/python/bin/activate + cd ~/mitx_all + pip install -r askbot-devel/askbot_requirements.txt + pip install -r askbot-devel/askbot_requirements_dev.txt + pip install -r mitx/pre-requirements.txt + pip install -r mitx/requirements.txt + +6) Create log and db dirs + + mkdir ~/mitx_all/log + mkdir ~/mitx_all/db + +7) Start the dev server + + To start using Django you will need + to activate the local Python and Ruby + environment: + + $ source ~/mitx_all/ruby/scripts/rvm + $ source ~/mitx_all/python/bin/activate + + To initialize and start a local instance of Django: + + $ cd ~/mitx_all/mitx + $ django-admin.py syncdb --settings=envs.dev --pythonpath=. + $ django-admin.py migrate --settings=envs.dev --pythonpath=. + $ django-admin.py runserver --settings=envs.dev --pythonpath=. + + From 178cdc877e3d83b107d11f72956761c42995096d Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 31 May 2012 14:36:23 -0400 Subject: [PATCH 09/10] removing mercurial from brew install list --- brew-formulas.txt | 1 - create-dev-env.sh | 76 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/brew-formulas.txt b/brew-formulas.txt index 5f76861eda..61e3d33e3a 100644 --- a/brew-formulas.txt +++ b/brew-formulas.txt @@ -3,7 +3,6 @@ sqlite gdbm pkg-config gfortran -mercurial python yuicompressor node diff --git a/create-dev-env.sh b/create-dev-env.sh index 1b3c272bb3..7190d81a73 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -1,6 +1,21 @@ #!/bin/bash set -e +trap "ouch" ERR +ouch() { + printf '\E[31m' + + cat<>$LOG + output "Cloning askbot-devel" + if [[ -d "$BASE/askbot-devel" ]]; then + mv "$BASE/askbot-devel" "${BASE}/askbot-devel.bak.$$" + fi + git clone git@github.com:MITx/askbot-devel >>$LOG + output "Cloning data" + if [[ -d "$BASE/data" ]]; then + mv "$BASE/data" "${BASE}/data.bak.$$" + fi + hg clone ssh://hg-content@gp.mitx.mit.edu/data >>$LOG +} PROG=${0##*/} BASE="$HOME/mitx_all" @@ -40,8 +72,8 @@ RUBY_DIR="$BASE/ruby" RUBY_VER="1.9.3" NUMPY_VER="1.6.2" SCIPY_VER="0.10.1" +BREW_FILE="$BASE/mitx/brew-formulas.txt" LOG="/var/tmp/install.log" -BREW_PKGS="$BASE/mit/brew-formulas.txt" APT_PKGS="curl git mercurial python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript" if [[ $EUID -eq 0 ]]; then @@ -49,7 +81,6 @@ if [[ $EUID -eq 0 ]]; then usage exit 1 fi - ARGS=$(getopt "cvh" "$*") if [[ $? != 0 ]]; then usage @@ -121,11 +152,12 @@ case `uname -s` in output "Installing ubuntu requirements" sudo apt-get -y update sudo apt-get -y install $APT_PKGS + clone_repos ;; *) error "Unsupported distribution - $distro" exit 1 - ;; + ;; esac ;; Darwin) @@ -133,9 +165,24 @@ case `uname -s` in output "Installing brew" /usr/bin/ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)" } + command -v git &>/dev/null || { + output "Installing git" + brew install git >> $LOG + } + command -v hg &>/dev/null || { + output "Installaing mercurial" + brew install mercurial >> $LOG + } + + clone_repos + output "Installing OSX requirements" + if [[ ! -r $BREW_FILE ]]; then + error "$BREW_FILE does not exist, needed to install brew deps" + exit 1 + fi # brew errors if the package is already installed - for pkg in $(cat $BREW_PKGS); do + for pkg in $(cat $BREW_FILE); do grep $pkg <(brew list) &>/dev/null || { output "Installing $pkg" brew install $pkg >>$LOG @@ -156,22 +203,6 @@ case `uname -s` in ;; esac -cd "$BASE" -output "Cloning mitx, askbot and data repos" -if [[ -d "$BASE/mitx" ]]; then - mv "$BASE/mitx" "${BASE}/mitx.bak.$$" -fi -git clone git@github.com:MITx/mitx.git >>$LOG -if [[ -d "$BASE/askbot-devel" ]]; then - mv "$BASE/askbot-devel" "${BASE}/askbot-devel.bak.$$" -fi -git clone git@github.com:MITx/askbot-devel >>$LOG -if [[ -d "$BASE/data" ]]; then - mv "$BASE/data" "${BASE}/data.bak.$$" -fi -hg clone ssh://hg-content@gp.mitx.mit.edu/data >>$LOG - - output "Installing rvm and ruby" curl -sL get.rvm.io | bash -s stable source $RUBY_DIR/scripts/rvm @@ -181,7 +212,8 @@ source $PYTHON_DIR/bin/activate output "Installing gem bundler" gem install bundler output "Installing ruby packages" -cd $BASE/mitx +# hack :( +cd $BASE/mitx || true bundle install cd $BASE From 6c4a3cf618155ecc18673bb53ddccfb6a81c9297 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 31 May 2012 14:54:38 -0400 Subject: [PATCH 10/10] updating for coffeescript --- create-dev-env.sh | 5 +++++ install.txt | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/create-dev-env.sh b/create-dev-env.sh index 7190d81a73..a45fc0dba0 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -196,6 +196,11 @@ case `uname -s` in output "Installing virtualenv" sudo pip install virtualenv virtualenvwrapper >> $LOG } + command -v coffee &>/dev/null || { + output "Installing coffee script" + curl http://npmjs.org/install.sh | sh + npm install -g coffee-script + } ;; *) error "Unsupported platform" diff --git a/install.txt b/install.txt index 70c8655306..fa82b11a5c 100644 --- a/install.txt +++ b/install.txt @@ -26,6 +26,10 @@ There is also a script "create-dev-env.sh" that automates these steps. d) Install python virtualenv if necessary sudo pip install virtualenv virtualenvwrapper + e) Install coffee script + curl http://npmjs.org/install.sh | sh + npm install -g coffee-script + 3) Install Ubuntu dependencies (Linux users only) sudo apt-get install curl python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript