Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Make upgrade requirement files. (#24078)

Browse Source 
Remove pandas requirement.
Replace pandas functionality with itertools.group_by.
This commit is contained in:
ericfab179
2020-06-01 08:08:09 -05:00
committed by GitHub
parent 740550491d
commit 01f56ba530
7 changed files with 35 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
openedx/core/process_warnings.py
View File

@@ -6,10 +6,10 @@ from __future__ import print_function
import json
import os
import io
import itertools
import re
import argparse
from collections import Counter
import pandas as pd
from write_to_html import (
HtmlOutlineWriter,
@@ -156,16 +156,29 @@ def process_warnings_json(dir_path):
return compressed_warnings_data
def group_and_sort_by_sumof(dataframe, group, sort_by):
groups_by = dataframe.groupby(group)
temp_list_to_sort = [(key, value, value[sort_by].sum()) for key, value in groups_by]
def group_and_sort_by_sumof(data, group, sort_by):
"""
Group and sort data.
Return
List of tuples. Each tuple has:
- Group key
- Iterable of warnings that belongs to that group
- Count of warnings that belong to that group
"""
sorted_data = sorted(data, key=lambda x: x[columns.index(group)])
groups_by = itertools.groupby(sorted_data, lambda x: x[columns_index_dict[group]])
temp_list_to_sort = []
for key, generator in groups_by:
value = list(generator)
temp_list_to_sort.append((key, value, sum([item[columns_index_dict[sort_by]] for item in value])))
# sort by count
return sorted(temp_list_to_sort, key=lambda x: -x[2])
def write_html_report(warnings_dataframe, html_path):
def write_html_report(warnings_data, html_path):
"""
converts from panda dataframe to our html
converts from list of lists data to our html
"""
html_path = os.path.expanduser(html_path)
if "/" in html_path:
@@ -175,7 +188,7 @@ def write_html_report(warnings_dataframe, html_path):
with io.open(html_path, "w") as fout:
html_writer = HtmlOutlineWriter(fout)
category_sorted_by_count = group_and_sort_by_sumof(
warnings_dataframe, "category", "num"
warnings_data, "category", "num"
)
for category, group_in_category, category_count in category_sorted_by_count:
# xss-lint: disable=python-wrap-html
@@ -211,20 +224,20 @@ def write_html_report(warnings_dataframe, html_path):
)
html_writer.start_section(html, klass=u"warning_text")
# warnings_object[location][warning_text] is a list
for _, warning in message_group.iterrows():
for warning in message_group:
# xss-lint: disable=python-wrap-html
html = u'<span class="count">{warning_file_path}</span> '.format(
warning_file_path=warning["filename"]
warning_file_path=warning[columns_index_dict["filename"]]
)
html_writer.start_section(html, klass=u"warning")
# xss-lint: disable=python-wrap-html
html = u'<p class="lineno">lineno: {lineno}</p> '.format(
lineno=warning["lineno"]
lineno=warning[columns_index_dict["lineno"]]
)
html_writer.write(html)
# xss-lint: disable=python-wrap-html
html = u'<p class="num">num_occur: {num}</p> '.format(
num=warning["num"]
num=warning[columns_index_dict["num"]]
)
html_writer.write(html)
@@ -242,5 +255,4 @@ if __name__ == "__main__":
parser.add_argument("--html-path", default="test_html.html")
args = parser.parse_args()
data_output = process_warnings_json(args.dir_path)
data_dataframe = pd.DataFrame(data=data_output, columns=columns)
write_html_report(data_dataframe, args.html_path)
write_html_report(data_output, args.html_path)

3
requirements/constraints.txt
View File

@@ -78,9 +78,6 @@ mock<4.0.0
# oauthlib>3.0.1 causes test failures
oauthlib==3.0.1
# Version 0.23.0 requires python-dateutil>=2.5.0
pandas==0.22.0
# path 13.2.0 drops support for Python 3.5
path<13.2.0

2
requirements/edx/base.txt
View File

@@ -50,7 +50,7 @@ cryptography==2.9.2 # via -r requirements/edx/../edx-sandbox/shared.txt, d
cssutils==1.0.2 # via pynliner
ddt==1.3.1 # via -c requirements/edx/../constraints.txt, xblock-drag-and-drop-v2, xblock-poll
decorator==4.4.2 # via pycontracts
defusedxml==0.6.0 # via -r requirements/edx/base.in, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core
defusedxml==0.7.0rc1 # via -r requirements/edx/base.in, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core
git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/github.in, django-statici18n
git+https://github.com/edx/django-babel-underscore.git@37705f7377a4d0a4e673f1431895ce28a8860cd7#egg=django-babel-underscore==0.6.0 # via -r requirements/edx/github.in
git+https://github.com/Zegocover/enmerkar.git@dbc113798aa4beabdfa2d00e6fef48248eb0f185#egg=django-babel==0.6.3.dev0 # via -r requirements/edx/github.in

1
requirements/edx/coverage.in
View File

@@ -14,4 +14,3 @@
coverage # Code coverage testing for Python
diff-cover # Automatically find diff lines that need test coverage
pandas # Used to process warnings generated by pytest

6
requirements/edx/coverage.txt
View File

@@ -12,11 +12,7 @@ jinja2-pluralize==0.3.0 # via diff-cover
jinja2==2.11.2 # via diff-cover, jinja2-pluralize
markupsafe==1.1.1 # via jinja2
more-itertools==8.3.0 # via zipp
numpy==1.18.4 # via pandas
pandas==0.22.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/coverage.in
pluggy==0.13.1 # via diff-cover
pygments==2.6.1 # via diff-cover
python-dateutil==2.4.0 # via -c requirements/edx/../constraints.txt, pandas
pytz==2020.1 # via pandas
six==1.15.0 # via diff-cover, python-dateutil
six==1.15.0 # via diff-cover
zipp==1.0.0 # via -c requirements/edx/../constraints.txt, importlib-metadata

9
requirements/edx/development.txt
View File

@@ -59,7 +59,7 @@ cssselect==1.1.0 # via -r requirements/edx/testing.txt, pyquery
cssutils==1.0.2 # via -r requirements/edx/testing.txt, pynliner
ddt==1.3.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, xblock-drag-and-drop-v2, xblock-poll
decorator==4.4.2 # via -r requirements/edx/testing.txt, pycontracts
defusedxml==0.6.0 # via -r requirements/edx/testing.txt, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core
defusedxml==0.7.0rc1 # via -r requirements/edx/testing.txt, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core
diff-cover==2.6.1 # via -r requirements/edx/testing.txt
distlib==0.3.0 # via -r requirements/edx/testing.txt, virtualenv
git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/testing.txt, django-statici18n
@@ -198,12 +198,11 @@ mysqlclient==1.4.6 # via -r requirements/edx/testing.txt
newrelic==5.12.1.141 # via -r requirements/edx/testing.txt, edx-django-utils
nltk==3.5 # via -r requirements/edx/testing.txt, chem
nodeenv==1.3.5 # via -r requirements/edx/testing.txt
numpy==1.18.4 # via -r requirements/edx/testing.txt, chem, openedx-calc, pandas, scipy
numpy==1.18.4 # via -r requirements/edx/testing.txt, chem, openedx-calc, scipy
oauthlib==3.0.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, django-oauth-toolkit, lti-consumer-xblock, requests-oauthlib, social-auth-core
openedx-calc==1.0.9 # via -r requirements/edx/testing.txt
git+https://github.com/edx/edx-ora2.git@2.7.7#egg=ora2==2.7.7 # via -r requirements/edx/testing.txt
packaging==20.4 # via -r requirements/edx/testing.txt, bleach, drf-yasg, pytest, sphinx, tox
pandas==0.22.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt
path.py==12.4.0 # via -r requirements/edx/testing.txt, edx-enterprise, edx-i18n-tools, ora2, xmodule
path==13.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, path.py
pathlib2==2.3.5 # via -r requirements/edx/testing.txt, pytest
@@ -248,14 +247,14 @@ pytest-metadata==1.8.0 # via -r requirements/edx/testing.txt, pytest-json-rep
pytest-randomly==3.4.0 # via -r requirements/edx/testing.txt
pytest-xdist==1.32.0 # via -r requirements/edx/testing.txt
pytest==5.3.5 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, pytest-attrib, pytest-cov, pytest-django, pytest-forked, pytest-json-report, pytest-metadata, pytest-randomly, pytest-xdist
python-dateutil==2.4.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, analytics-python, botocore, edx-ace, edx-drf-extensions, edx-enterprise, edx-proctoring, faker, freezegun, icalendar, ora2, pandas, xblock
python-dateutil==2.4.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, analytics-python, botocore, edx-ace, edx-drf-extensions, edx-enterprise, edx-proctoring, faker, freezegun, icalendar, ora2, xblock
python-levenshtein==0.12.0 # via -r requirements/edx/testing.txt
python-memcached==1.59 # via -r requirements/edx/testing.txt
python-slugify==4.0.0 # via -r requirements/edx/testing.txt, code-annotations, transifex-client
python-swiftclient==3.9.0 # via -r requirements/edx/testing.txt, ora2
python3-openid==3.1.0 ; python_version >= "3" # via -r requirements/edx/testing.txt, social-auth-core
python3-saml==1.9.0 # via -r requirements/edx/testing.txt
pytz==2020.1 # via -r requirements/edx/testing.txt, babel, capa, celery, django, django-ses, edx-completion, edx-enterprise, edx-proctoring, edx-submissions, edx-tincan-py35, event-tracking, fs, icalendar, ora2, pandas, xblock
pytz==2020.1 # via -r requirements/edx/testing.txt, babel, capa, celery, django, django-ses, edx-completion, edx-enterprise, edx-proctoring, edx-submissions, edx-tincan-py35, event-tracking, fs, icalendar, ora2, xblock
pyuca==1.2 # via -r requirements/edx/testing.txt
pywatchman==1.4.1 # via -r requirements/edx/development.in
pyyaml==5.3.1 # via -r requirements/edx/testing.txt, code-annotations, edx-django-release-util, edx-i18n-tools, sphinxcontrib-openapi, xblock

9
requirements/edx/testing.txt
View File

@@ -58,7 +58,7 @@ cssselect==1.1.0 # via -r requirements/edx/testing.in, pyquery
cssutils==1.0.2 # via -r requirements/edx/base.txt, pynliner
ddt==1.3.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, -r requirements/edx/testing.in, xblock-drag-and-drop-v2, xblock-poll
decorator==4.4.2 # via -r requirements/edx/base.txt, pycontracts
defusedxml==0.6.0 # via -r requirements/edx/base.txt, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core
defusedxml==0.7.0rc1 # via -r requirements/edx/base.txt, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core
diff-cover==2.6.1 # via -r requirements/edx/coverage.txt
distlib==0.3.0 # via virtualenv
git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/base.txt, django-statici18n
@@ -190,12 +190,11 @@ mysqlclient==1.4.6 # via -r requirements/edx/base.txt
newrelic==5.12.1.141 # via -r requirements/edx/base.txt, edx-django-utils
nltk==3.5 # via -r requirements/edx/base.txt, chem
nodeenv==1.3.5 # via -r requirements/edx/base.txt
numpy==1.18.4 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, chem, openedx-calc, pandas, scipy
numpy==1.18.4 # via -r requirements/edx/base.txt, chem, openedx-calc, scipy
oauthlib==3.0.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, django-oauth-toolkit, lti-consumer-xblock, requests-oauthlib, social-auth-core
openedx-calc==1.0.9 # via -r requirements/edx/base.txt
git+https://github.com/edx/edx-ora2.git@2.7.7#egg=ora2==2.7.7 # via -r requirements/edx/base.txt
packaging==20.4 # via -r requirements/edx/base.txt, bleach, drf-yasg, pytest, tox
pandas==0.22.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/coverage.txt
path.py==12.4.0 # via -r requirements/edx/base.txt, edx-enterprise, edx-i18n-tools, ora2, xmodule
path==13.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, path.py
pathlib2==2.3.5 # via pytest
@@ -238,14 +237,14 @@ pytest-metadata==1.8.0 # via -r requirements/edx/testing.in, pytest-json-repo
pytest-randomly==3.4.0 # via -r requirements/edx/testing.in
pytest-xdist==1.32.0 # via -r requirements/edx/testing.in
pytest==5.3.5 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.in, pytest-attrib, pytest-cov, pytest-django, pytest-forked, pytest-json-report, pytest-metadata, pytest-randomly, pytest-xdist
python-dateutil==2.4.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, analytics-python, botocore, edx-ace, edx-drf-extensions, edx-enterprise, edx-proctoring, faker, freezegun, icalendar, ora2, pandas, xblock
python-dateutil==2.4.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, analytics-python, botocore, edx-ace, edx-drf-extensions, edx-enterprise, edx-proctoring, faker, freezegun, icalendar, ora2, xblock
python-levenshtein==0.12.0 # via -r requirements/edx/base.txt
python-memcached==1.59 # via -r requirements/edx/base.txt
python-slugify==4.0.0 # via -r requirements/edx/base.txt, code-annotations, transifex-client
python-swiftclient==3.9.0 # via -r requirements/edx/base.txt, ora2
python3-openid==3.1.0 ; python_version >= "3" # via -r requirements/edx/base.txt, social-auth-core
python3-saml==1.9.0 # via -r requirements/edx/base.txt
pytz==2020.1 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, babel, capa, celery, django, django-ses, edx-completion, edx-enterprise, edx-proctoring, edx-submissions, edx-tincan-py35, event-tracking, fs, icalendar, ora2, pandas, xblock
pytz==2020.1 # via -r requirements/edx/base.txt, babel, capa, celery, django, django-ses, edx-completion, edx-enterprise, edx-proctoring, edx-submissions, edx-tincan-py35, event-tracking, fs, icalendar, ora2, xblock
pyuca==1.2 # via -r requirements/edx/base.txt
pyyaml==5.3.1 # via -r requirements/edx/base.txt, code-annotations, edx-django-release-util, edx-i18n-tools, xblock
radon==4.1.0 # via -r requirements/edx/testing.in