From b16b4eb4e724d8c15ec0365fbbb11ccaf3d6217e Mon Sep 17 00:00:00 2001 From: Ahmed Jazzar Date: Wed, 3 Oct 2018 22:28:14 -0700 Subject: [PATCH 1/2] Upgrade bleach to 2.1.4 Update ORA2 repo version Fix Python tests Upgrade Recommender XBlock version Remove unnecessary requirements and update bleacher usage Keep html5lib unpinned Upgrading requirements Ignore python-interpolate-html for bleached html --- common/lib/capa/capa/util.py | 6 ++---- requirements/edx/base.in | 4 ++-- requirements/edx/base.txt | 8 ++++---- requirements/edx/development.txt | 8 ++++---- requirements/edx/github.in | 4 ++-- requirements/edx/testing.txt | 8 ++++---- 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/common/lib/capa/capa/util.py b/common/lib/capa/capa/util.py index c700d07387..7bd44b7039 100644 --- a/common/lib/capa/capa/util.py +++ b/common/lib/capa/capa/util.py @@ -171,10 +171,6 @@ def sanitize_html(html_code): Used to sanitize XQueue responses from Matlab. """ attributes = bleach.ALLOWED_ATTRIBUTES.copy() - # Yuck! but bleach does not offer the option of passing in allowed_protocols, - # and matlab uses data urls for images - if u'data' not in bleach.BleachSanitizer.allowed_protocols: - bleach.BleachSanitizer.allowed_protocols.append(u'data') attributes.update({ '*': ['class', 'style', 'id'], 'audio': ['controls', 'autobuffer', 'autoplay', 'src'], @@ -182,6 +178,7 @@ def sanitize_html(html_code): }) output = bleach.clean( html_code, + protocols=bleach.ALLOWED_PROTOCOLS + ['data'], tags=bleach.ALLOWED_TAGS + ['div', 'p', 'audio', 'pre', 'img', 'span'], styles=['white-space'], attributes=attributes @@ -197,6 +194,7 @@ def get_inner_html_from_xpath(xpath_node): # returns string from xpath node html = etree.tostring(xpath_node).strip() # strips outer tag from html string + # xss-lint: disable=python-interpolate-html inner_html = re.sub('(?ms)<%s[^>]*>(.*)' % (xpath_node.tag, xpath_node.tag), '\\1', html) return inner_html.strip() diff --git a/requirements/edx/base.in b/requirements/edx/base.in index d12161627d..5908000506 100644 --- a/requirements/edx/base.in +++ b/requirements/edx/base.in @@ -27,7 +27,7 @@ analytics-python==1.1.0 # Used for Segment analytics aniso8601==3.0.2 # via tincan attrs # Reduces boilerplate code involving class attributes Babel==1.3 # Internationalization utilities, used for date formatting in a few places -bleach==1.4 # Allowed-list-based HTML sanitizing library that escapes or strips markup and attributes; used for capa and LTI +bleach==2.1.4 # Allowed-list-based HTML sanitizing library that escapes or strips markup and attributes; used for capa and LTI boto==2.39.0 # Deprecated version of the AWS SDK; we should stop using this boto3==1.4.8 # Amazon Web Services SDK for Python botocore==1.8.17 # via boto3, s3transfer @@ -95,7 +95,7 @@ futures ; python_version == "2.7" # via django-pipeline, python-swift-client, glob2==0.3 # Enhanced glob module, used in openedx.core.lib.rooted_paths gunicorn==19.0 help-tokens -html5lib==0.999 # HTML parser, used for capa problems +html5lib # HTML parser, used for capa problems ipaddr==2.1.11 # Ip network support for Embargo feature jsonfield # Django model field for validated JSON; used in several apps mailsnake==1.6.2 # Needed for mailchimp (mailing djangoapp) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 48fb98c40a..81a7f9e725 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -25,12 +25,12 @@ git+https://github.com/mitodl/edx-sga.git@3828ba9e413080a81b907a3381e5ffa05e063f git+https://github.com/edx/xblock-lti-consumer.git@v1.1.8#egg=lti_consumer-xblock==1.1.8 git+https://github.com/edx/MongoDBProxy.git@25b99097615bda06bd7cdfe5669ed80dc2a7fed0#egg=MongoDBProxy==0.1.0 -e . -git+https://github.com/edx/edx-ora2.git@2.1.18#egg=ora2==2.1.18 +git+https://github.com/edx/edx-ora2.git@2.2.0#egg=ora2==2.2.0 -e git+https://github.com/dgrtwo/ParsePy.git@7949b9f754d1445eff8e8f20d0e967b9a6420639#egg=parse_rest -e git+https://github.com/appliedsec/pygeoip.git@95e69341cebf5a6a9fbf7c4f5439d458898bdc3b#egg=pygeoip -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev -e git+https://github.com/edx/RateXBlock.git@367e19c0f6eac8a5f002fd0f1559555f8e74bfff#egg=rate-xblock -git+https://github.com/edx/RecommenderXBlock.git@1.3.3#egg=recommender-xblock==1.3.3 +git+https://github.com/edx/RecommenderXBlock.git@1.4.0#egg=recommender-xblock==1.4.0 -e common/lib/safe_lxml -e common/lib/sandbox-packages -e common/lib/symmath @@ -51,7 +51,7 @@ attrs==17.4.0 babel==1.3 beautifulsoup4==4.6.3 # via pynliner billiard==3.3.0.23 # via celery -bleach==1.4 +bleach==2.1.4 boto3==1.4.8 boto==2.39.0 botocore==1.8.17 @@ -144,7 +144,7 @@ glob2==0.3 gunicorn==19.0 hash-ring==1.3.1 # via django-memcached-hashring help-tokens==1.0.3 -html5lib==0.999 +html5lib==1.0.1 httplib2==0.11.3 # via oauth2, zendesk idna==2.7 ipaddr==2.1.11 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 536cd1bdab..3a799f1864 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -28,12 +28,12 @@ git+https://github.com/edx/lettuce.git@7a04591c78ac56dac3eb3e91ca94b15cce844133# git+https://github.com/edx/xblock-lti-consumer.git@v1.1.8#egg=lti_consumer-xblock==1.1.8 git+https://github.com/edx/MongoDBProxy.git@25b99097615bda06bd7cdfe5669ed80dc2a7fed0#egg=MongoDBProxy==0.1.0 -e . -git+https://github.com/edx/edx-ora2.git@2.1.18#egg=ora2==2.1.18 +git+https://github.com/edx/edx-ora2.git@2.2.0#egg=ora2==2.2.0 -e git+https://github.com/dgrtwo/ParsePy.git@7949b9f754d1445eff8e8f20d0e967b9a6420639#egg=parse_rest -e git+https://github.com/appliedsec/pygeoip.git@95e69341cebf5a6a9fbf7c4f5439d458898bdc3b#egg=pygeoip -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev -e git+https://github.com/edx/RateXBlock.git@367e19c0f6eac8a5f002fd0f1559555f8e74bfff#egg=rate-xblock -git+https://github.com/edx/RecommenderXBlock.git@1.3.3#egg=recommender-xblock==1.3.3 +git+https://github.com/edx/RecommenderXBlock.git@1.4.0#egg=recommender-xblock==1.4.0 -e common/lib/safe_lxml -e common/lib/sandbox-packages -e common/lib/symmath @@ -60,7 +60,7 @@ backports.functools-lru-cache==1.5 beautifulsoup4==4.6.3 before-after==1.0.1 billiard==3.3.0.23 -bleach==1.4 +bleach==2.1.4 bok-choy==0.9.0 boto3==1.4.8 boto==2.39.0 @@ -178,7 +178,7 @@ glob2==0.3 gunicorn==19.0 hash-ring==1.3.1 help-tokens==1.0.3 -html5lib==0.999 +html5lib==1.0.1 httplib2==0.11.3 httpretty==0.9.5 idna==2.7 diff --git a/requirements/edx/github.in b/requirements/edx/github.in index a5c5404589..e0116ff6cb 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -89,8 +89,8 @@ # Our libraries: -e git+https://github.com/edx/codejail.git@a320d43ce6b9c93b17636b2491f724d9e433be47#egg=codejail -e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock --e git+https://github.com/edx/edx-ora2.git@2.1.18#egg=ora2==2.1.18 --e git+https://github.com/edx/RecommenderXBlock.git@1.3.3#egg=recommender-xblock==1.3.3 +-e git+https://github.com/edx/edx-ora2.git@2.2.0#egg=ora2==2.2.0 +-e git+https://github.com/edx/RecommenderXBlock.git@1.4.0#egg=recommender-xblock==1.4.0 -e git+https://github.com/solashirai/crowdsourcehinter.git@518605f0a95190949fe77bd39158450639e2e1dc#egg=crowdsourcehinter-xblock==0.1 -e git+https://github.com/edx/RateXBlock.git@367e19c0f6eac8a5f002fd0f1559555f8e74bfff#egg=rate-xblock -e git+https://github.com/edx/DoneXBlock.git@01a14f3bd80ae47dd08cdbbe2f88f3eb88d00fba#egg=done-xblock diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index ac45be3a93..de52476fa6 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -26,12 +26,12 @@ git+https://github.com/edx/lettuce.git@7a04591c78ac56dac3eb3e91ca94b15cce844133# git+https://github.com/edx/xblock-lti-consumer.git@v1.1.8#egg=lti_consumer-xblock==1.1.8 git+https://github.com/edx/MongoDBProxy.git@25b99097615bda06bd7cdfe5669ed80dc2a7fed0#egg=MongoDBProxy==0.1.0 -e . -git+https://github.com/edx/edx-ora2.git@2.1.18#egg=ora2==2.1.18 +git+https://github.com/edx/edx-ora2.git@2.2.0#egg=ora2==2.2.0 -e git+https://github.com/dgrtwo/ParsePy.git@7949b9f754d1445eff8e8f20d0e967b9a6420639#egg=parse_rest -e git+https://github.com/appliedsec/pygeoip.git@95e69341cebf5a6a9fbf7c4f5439d458898bdc3b#egg=pygeoip -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev -e git+https://github.com/edx/RateXBlock.git@367e19c0f6eac8a5f002fd0f1559555f8e74bfff#egg=rate-xblock -git+https://github.com/edx/RecommenderXBlock.git@1.3.3#egg=recommender-xblock==1.3.3 +git+https://github.com/edx/RecommenderXBlock.git@1.4.0#egg=recommender-xblock==1.4.0 -e common/lib/safe_lxml -e common/lib/sandbox-packages -e common/lib/symmath @@ -57,7 +57,7 @@ backports.functools-lru-cache==1.5 # via astroid, pylint beautifulsoup4==4.6.3 before-after==1.0.1 billiard==3.3.0.23 -bleach==1.4 +bleach==2.1.4 bok-choy==0.9.0 boto3==1.4.8 boto==2.39.0 @@ -172,7 +172,7 @@ glob2==0.3 gunicorn==19.0 hash-ring==1.3.1 help-tokens==1.0.3 -html5lib==0.999 +html5lib==1.0.1 httplib2==0.11.3 httpretty==0.9.5 idna==2.7 From abb33ce56d531e744f9f6de0ad39fe574bfa2c71 Mon Sep 17 00:00:00 2001 From: Ahmed Jazzar Date: Wed, 31 Oct 2018 14:12:31 -0700 Subject: [PATCH 2/2] Upgrade requirements --- requirements/edx/base.txt | 1 + requirements/edx/development.txt | 3 ++- requirements/edx/testing.txt | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 81a7f9e725..824608a8c3 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -239,6 +239,7 @@ user-util==0.1.5 voluptuous==0.11.5 watchdog==0.9.0 web-fragments==0.2.2 +webencodings==0.5.1 # via html5lib webob==1.8.3 # via xblock wrapt==1.10.5 xblock-review==1.1.5 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 3a799f1864..cbb2a5d896 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -338,12 +338,13 @@ uritemplate==3.0.0 urllib3==1.23 urlobject==2.4.3 user-util==0.1.5 -virtualenv==16.0.0 +virtualenv==16.1.0 voluptuous==0.11.5 vulture==1.0 w3lib==1.19.0 watchdog==0.9.0 web-fragments==0.2.2 +webencodings==0.5.1 webob==1.8.3 werkzeug==0.14.1 wrapt==1.10.5 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index de52476fa6..62161c8999 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -322,11 +322,12 @@ uritemplate==3.0.0 urllib3==1.23 urlobject==2.4.3 # via pa11ycrawler user-util==0.1.5 -virtualenv==16.0.0 # via tox +virtualenv==16.1.0 # via tox voluptuous==0.11.5 w3lib==1.19.0 # via parsel, scrapy watchdog==0.9.0 web-fragments==0.2.2 +webencodings==0.5.1 webob==1.8.3 werkzeug==0.14.1 # via flask wrapt==1.10.5