Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: Overhaul Dockerfile for sandboxes (#31232)

Browse Source 
* chore: Overhaul Dockerfile for sandboxes using Docker best practices
* Reduce image size by installing and removing prerequisite packages
in the same layer.
* Rearrange stages to use docker-production settings for non-dev
  targets. docker-production settings already inherit production
  settings and can be used to override configuration specific to
  containers e.g. logging.

* chore: write improved Dockerfile2 just for testing

* chore: update development stage to not run as app user

* fix: wrap settings configuration in if statement

* chore: update Dockerfile.
* Moved code COPY command down in the base stage.
* Added comments.

Co-authored-by: Alie Langston <alangsto@wellesley.edu>
This commit is contained in:
Saad Ali
2022-12-06 13:42:13 +00:00
committed by GitHub
parent bc94071c90
commit d0fad59306
2 changed files with 124 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

270
Dockerfile
View File

@@ -1,4 +1,4 @@
FROM ubuntu:focal as base
FROM ubuntu:focal as minimal-system
# Warning: This file is experimental.
#
@@ -9,53 +9,16 @@ FROM ubuntu:focal as base
# * Related to ^, use no Ansible or Paver.
# Long-term goal:
# * Be a suitable base for production LMS and CMS images (THIS IS NOT YET THE CASE!).
#
# Install system requirements.
# We update, upgrade, and delete lists all in one layer
# in order to reduce total image size.
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install --yes \
# Global requirements
build-essential \
curl \
# If we don't need gcc, we should remove it.
g++ \
gcc \
git \
git-core \
language-pack-en \
libfreetype6-dev \
libmysqlclient-dev \
libssl-dev \
libxml2-dev \
libxmlsec1-dev \
libxslt1-dev \
swig \
# openedx requirements
gettext \
gfortran \
graphviz \
libffi-dev \
libfreetype6-dev \
libgeos-dev \
libgraphviz-dev \
libjpeg8-dev \
liblapack-dev \
libpng-dev \
libsqlite3-dev \
libxml2-dev \
libxmlsec1-dev \
libxslt1-dev \
# lynx: Required by https://github.com/openedx/edx-platform/blob/b489a4ecb122/openedx/core/lib/html_to_text.py#L16
lynx \
ntp \
pkg-config \
python3-dev \
python3-venv && \
rm -rf /var/lib/apt/lists/*
# Set locale.
RUN locale-gen en_US.UTF-8
ARG DEBIAN_FRONTEND=noninteractive
ARG SERVICE_VARIANT
ARG SERVICE_PORT
# Env vars: paver
# We intentionally don't use paver in this Dockerfile, but Devstack may invoke paver commands
# during provisioning. Enabling NO_PREREQ_INSTALL tells paver not to re-install Python
# requirements for every paver command, potentially saving a lot of developer time.
ARG NO_PREREQ_INSTALL='1'
# Env vars: locale
ENV LANG='en_US.UTF-8'
@@ -66,146 +29,153 @@ ENV LC_ALL='en_US.UTF-8'
ENV CONFIG_ROOT='/edx/etc'
ENV LMS_CFG="$CONFIG_ROOT/lms.yml"
ENV CMS_CFG="$CONFIG_ROOT/cms.yml"
ENV EDX_PLATFORM_SETTINGS='production'
# Env vars: path
ENV VIRTUAL_ENV='/edx/app/edxapp/venvs/edxapp'
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
ENV VIRTUAL_ENV="/edx/app/edxapp/venvs/edxapp"
ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
ENV PATH="/edx/app/edxapp/edx-platform/node_modules/.bin:${PATH}"
ENV PATH="/edx/app/edxapp/edx-platform/bin:${PATH}"
ENV PATH="/edx/app/edxapp/nodeenv/bin:${PATH}"
# Create config directory. Create, define, and switch to working directory.
RUN mkdir -p "$CONFIG_ROOT"
WORKDIR /edx/app/edxapp/edx-platform
# Env vars: paver
# We intentionally don't use paver in this Dockerfile, but Devstack may invoke paver commands
# during provisioning. Enabling NO_PREREQ_INSTALL tells paver not to re-install Python
# requirements for every paver command, potentially saving a lot of developer time.
ENV NO_PREREQ_INSTALL='1'
# Create user before assigning any directory ownership to it.
RUN useradd -m --shell /bin/false app
# Set up a Python virtual environment.
# It is already 'activated' because $VIRTUAL_ENV/bin was put on $PATH.
RUN python3.8 -m venv "$VIRTUAL_ENV"
# Use debconf to set locales to be generated when the locales apt package is installed later.
RUN echo "locales locales/default_environment_locale select en_US.UTF-8" | debconf-set-selections
RUN echo "locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8" | debconf-set-selections
# Install Python requirements.
# Requires copying over requirements files, but not entire repository.
# Install requirements that are absolutely necessary
RUN apt-get update && \
apt-get -y dist-upgrade && \
apt-get -y install --no-install-recommends \
python3 \
python3-venv \
python3.8 \
python3.8-minimal \
libpython3.8 \
libpython3.8-stdlib \
libmysqlclient21 \
libssl1.1 \
libxmlsec1-openssl \
# lynx: Required by https://github.com/openedx/edx-platform/blob/b489a4ecb122/openedx/core/lib/html_to_text.py#L16
lynx \
ntp \
gettext \
gfortran \
graphviz \
locales \
swig \
&& \
apt-get clean all && \
rm -rf /var/lib/apt/*
RUN mkdir -p /edx/var/edxapp
RUN mkdir -p /edx/etc
RUN chown app:app /edx/var/edxapp
# The builder-production stage is a temporary stage that installs required packages and builds the python virtualenv,
# installs nodejs and node_modules.
# The built artifacts from this stage are then copied to the base stage.
FROM minimal-system as builder-production
RUN apt-get update && \
apt-get -y install --no-install-recommends \
curl \
git \
git-core \
pkg-config \
build-essential \
libmysqlclient-dev \
libssl-dev \
libxml2-dev \
libxmlsec1-dev \
libxslt1-dev \
python3-dev \
libffi-dev \
libfreetype6-dev \
libgeos-dev \
libgraphviz-dev \
libjpeg8-dev \
liblapack-dev \
libpng-dev \
libsqlite3-dev \
libxml2-dev \
libxmlsec1-dev \
libxslt1-dev
# Setup python virtual environment
# It is already 'activated' because $VIRTUAL_ENV/bin was put on $PATH
RUN python3.8 -m venv "${VIRTUAL_ENV}"
# Install python requirements
# Requires copying over requirements files, but not entire repository
COPY requirements requirements
RUN pip install -r requirements/pip.txt
RUN pip install -r requirements/edx/base.txt
# Set up a Node environment and install Node requirements.
# Must be done after Python requirements, since nodeenv is installed
# via pip.
# The node environment is already 'activated' because its .../bin was put on $PATH.
# Install node and node modules
RUN nodeenv /edx/app/edxapp/nodeenv --node=16.14.0 --prebuilt
RUN npm install -g npm@8.5.x
COPY package.json package.json
COPY package-lock.json package-lock.json
RUN npm set progress=false && npm install
# Copy over remaining parts of repository (including all code).
# The builder-development stage is a temporary stage that installs python modules required for development purposes
# The built artifacts from this stage are then copied to the development stage.
FROM builder-production as builder-development
RUN pip install -r requirements/edx/development.txt
# base stage
FROM minimal-system as base
# Copy python virtual environment, nodejs and node_modules
COPY --from=builder-production /edx/app/edxapp/venvs/edxapp /edx/app/edxapp/venvs/edxapp
COPY --from=builder-production /edx/app/edxapp/nodeenv /edx/app/edxapp/nodeenv
COPY --from=builder-production /edx/app/edxapp/edx-platform/node_modules /edx/app/edxapp/edx-platform/node_modules
# Copy over remaining parts of repository (including all code)
COPY . .
# Install Python requirements again in order to capture local projects
RUN pip install -e .
RUN useradd -m --shell /bin/false app
USER app
##################################################
# Define LMS docker-based non-dev target.
FROM base as lms-docker
ENV SERVICE_VARIANT lms
ARG LMS_CFG_OVERRIDE
RUN echo "$LMS_CFG_OVERRIDE"
ENV LMS_CFG="${LMS_CFG_OVERRIDE:-$LMS_CFG}"
RUN echo "$LMS_CFG"
# Production target
FROM base as production
ENV EDX_PLATFORM_SETTINGS='docker-production'
ENV DJANGO_SETTINGS_MODULE="lms.envs.$EDX_PLATFORM_SETTINGS"
EXPOSE 8000
ENV SERVICE_VARIANT "${SERVICE_VARIANT}"
ENV SERVICE_PORT "${SERVICE_PORT}"
ENV DJANGO_SETTINGS_MODULE="${SERVICE_VARIANT}.envs.$EDX_PLATFORM_SETTINGS"
EXPOSE ${SERVICE_PORT}
CMD gunicorn \
-c /edx/app/edxapp/edx-platform/lms/docker_lms_gunicorn.py \
--name lms \
--bind=0.0.0.0:8000 \
-c /edx/app/edxapp/edx-platform/${SERVICE_VARIANT}/docker_${SERVICE_VARIANT}_gunicorn.py \
--name ${SERVICE_VARIANT} \
--bind=0.0.0.0:${SERVICE_PORT} \
--max-requests=1000 \
--access-logfile \
- lms.wsgi:application
- ${SERVICE_VARIANT}.wsgi:application
##################################################
# Define LMS non-dev target.
FROM base as lms
ENV LMS_CFG="$CONFIG_ROOT/lms.yml"
ENV SERVICE_VARIANT lms
ENV DJANGO_SETTINGS_MODULE="lms.envs.$EDX_PLATFORM_SETTINGS"
EXPOSE 8000
CMD gunicorn \
-c /edx/app/edxapp/edx-platform/lms/docker_lms_gunicorn.py \
--name lms \
--bind=0.0.0.0:8000 \
--max-requests=1000 \
--access-logfile \
- lms.wsgi:application
# Development target
FROM base as development
COPY --from=builder-development /edx/app/edxapp/venvs/edxapp /edx/app/edxapp/venvs/edxapp
##################################################
# Define CMS non-dev target.
FROM base as cms
ENV SERVICE_VARIANT cms
ENV EDX_PLATFORM_SETTINGS='production'
ENV DJANGO_SETTINGS_MODULE="cms.envs.$EDX_PLATFORM_SETTINGS"
EXPOSE 8010
CMD gunicorn \
-c /edx/app/edxapp/edx-platform/cms/docker_cms_gunicorn.py \
--name cms \
--bind=0.0.0.0:8010 \
--max-requests=1000 \
--access-logfile \
- cms.wsgi:application
##################################################
# Define intermediate dev target for LMS/CMS.
#
# Although it might seem more logical to forego the `dev` stage
# and instead base `lms-dev` and `cms-dev` off of `lms` and
# `cms`, respectively, we choose to have this `dev` stage
# so that the installed development requirements are contained
# in a single layer, shared between `lms-dev` and `cms-dev`.
FROM base as dev
USER root
RUN pip install -r requirements/edx/development.txt
# Link configuration YAMLs and set EDX_PLATFORM_SE1TTINGS.
ENV EDX_PLATFORM_SETTINGS='devstack_docker'
RUN ln -s "$(pwd)/lms/envs/devstack-experimental.yml" "$LMS_CFG"
RUN ln -s "$(pwd)/cms/envs/devstack-experimental.yml" "$CMS_CFG"
# Temporary compatibility hack while devstack is supporting
# both the old `edxops/edxapp` image and this image:
# Add in a dummy ../edxapp_env file.
# The edxapp_env file was originally needed for sourcing to get
# environment variables like LMS_CFG, but now we just set
# those variables right in the Dockerfile.
# Temporary compatibility hack while devstack is supporting both the old `edxops/edxapp` image and this image.
# * Add in a dummy ../edxapp_env file
# * devstack sets /edx/etc/studio.yml as CMS_CFG.
RUN ln -s "$(pwd)/cms/envs/devstack-experimental.yml" "/edx/etc/studio.yml"
RUN touch ../edxapp_env
USER app
##################################################
# Define LMS dev target.
FROM dev as lms-dev
ENV SERVICE_VARIANT lms
ENV DJANGO_SETTINGS_MODULE="lms.envs.$EDX_PLATFORM_SETTINGS"
EXPOSE 18000
CMD while true; do python ./manage.py lms runserver 0.0.0.0:18000; sleep 2; done
##################################################
# Define CMS dev target.
FROM dev as cms-dev
ENV SERVICE_VARIANT cms
ENV DJANGO_SETTINGS_MODULE="cms.envs.$EDX_PLATFORM_SETTINGS"
EXPOSE 18010
CMD while true; do python ./manage.py cms runserver 0.0.0.0:18010; sleep 2; done
ENV EDX_PLATFORM_SETTINGS='devstack_docker'
ENV SERVICE_VARIANT "${SERVICE_VARIANT}"
ENV DJANGO_SETTINGS_MODULE="${SERVICE_VARIANT}.envs.$EDX_PLATFORM_SETTINGS"
EXPOSE ${SERVICE_PORT}
CMD ./manage.py ${SERVICE_VARIANT} runserver 0.0.0.0:${SERVICE_PORT}

7
xmodule/static_content.py
View File

@@ -297,9 +297,10 @@ def main():
installed_apps += ('edxval',)
except ImportError:
pass
settings.configure(
INSTALLED_APPS=installed_apps,
)
if not settings.configured:
settings.configure(
INSTALLED_APPS=installed_apps,
)
django.setup()
args = docopt(main.__doc__)