diff --git a/openedx/core/process_warnings.py b/openedx/core/process_warnings.py index 6c749e76f1..e0b0fd37f8 100644 --- a/openedx/core/process_warnings.py +++ b/openedx/core/process_warnings.py @@ -6,10 +6,10 @@ from __future__ import print_function import json import os import io +import itertools import re import argparse from collections import Counter -import pandas as pd from write_to_html import ( HtmlOutlineWriter, @@ -156,16 +156,29 @@ def process_warnings_json(dir_path): return compressed_warnings_data -def group_and_sort_by_sumof(dataframe, group, sort_by): - groups_by = dataframe.groupby(group) - temp_list_to_sort = [(key, value, value[sort_by].sum()) for key, value in groups_by] +def group_and_sort_by_sumof(data, group, sort_by): + """ + Group and sort data. + + Return + List of tuples. Each tuple has: + - Group key + - Iterable of warnings that belongs to that group + - Count of warnings that belong to that group + """ + sorted_data = sorted(data, key=lambda x: x[columns.index(group)]) + groups_by = itertools.groupby(sorted_data, lambda x: x[columns_index_dict[group]]) + temp_list_to_sort = [] + for key, generator in groups_by: + value = list(generator) + temp_list_to_sort.append((key, value, sum([item[columns_index_dict[sort_by]] for item in value]))) # sort by count return sorted(temp_list_to_sort, key=lambda x: -x[2]) -def write_html_report(warnings_dataframe, html_path): +def write_html_report(warnings_data, html_path): """ - converts from panda dataframe to our html + converts from list of lists data to our html """ html_path = os.path.expanduser(html_path) if "/" in html_path: @@ -175,7 +188,7 @@ def write_html_report(warnings_dataframe, html_path): with io.open(html_path, "w") as fout: html_writer = HtmlOutlineWriter(fout) category_sorted_by_count = group_and_sort_by_sumof( - warnings_dataframe, "category", "num" + warnings_data, "category", "num" ) for category, group_in_category, category_count in category_sorted_by_count: # xss-lint: disable=python-wrap-html @@ -211,20 +224,20 @@ def write_html_report(warnings_dataframe, html_path): ) html_writer.start_section(html, klass=u"warning_text") # warnings_object[location][warning_text] is a list - for _, warning in message_group.iterrows(): + for warning in message_group: # xss-lint: disable=python-wrap-html html = u'{warning_file_path} '.format( - warning_file_path=warning["filename"] + warning_file_path=warning[columns_index_dict["filename"]] ) html_writer.start_section(html, klass=u"warning") # xss-lint: disable=python-wrap-html html = u'

lineno: {lineno}

'.format( - lineno=warning["lineno"] + lineno=warning[columns_index_dict["lineno"]] ) html_writer.write(html) # xss-lint: disable=python-wrap-html html = u'

num_occur: {num}

'.format( - num=warning["num"] + num=warning[columns_index_dict["num"]] ) html_writer.write(html) @@ -242,5 +255,4 @@ if __name__ == "__main__": parser.add_argument("--html-path", default="test_html.html") args = parser.parse_args() data_output = process_warnings_json(args.dir_path) - data_dataframe = pd.DataFrame(data=data_output, columns=columns) - write_html_report(data_dataframe, args.html_path) + write_html_report(data_output, args.html_path) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 4dcd7a4492..39cf905eb2 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -78,9 +78,6 @@ mock<4.0.0 # oauthlib>3.0.1 causes test failures oauthlib==3.0.1 -# Version 0.23.0 requires python-dateutil>=2.5.0 -pandas==0.22.0 - # path 13.2.0 drops support for Python 3.5 path<13.2.0 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 0fd1341b40..f0c5dddeec 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -50,7 +50,7 @@ cryptography==2.9.2 # via -r requirements/edx/../edx-sandbox/shared.txt, d cssutils==1.0.2 # via pynliner ddt==1.3.1 # via -c requirements/edx/../constraints.txt, xblock-drag-and-drop-v2, xblock-poll decorator==4.4.2 # via pycontracts -defusedxml==0.6.0 # via -r requirements/edx/base.in, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core +defusedxml==0.7.0rc1 # via -r requirements/edx/base.in, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/github.in, django-statici18n git+https://github.com/edx/django-babel-underscore.git@37705f7377a4d0a4e673f1431895ce28a8860cd7#egg=django-babel-underscore==0.6.0 # via -r requirements/edx/github.in git+https://github.com/Zegocover/enmerkar.git@dbc113798aa4beabdfa2d00e6fef48248eb0f185#egg=django-babel==0.6.3.dev0 # via -r requirements/edx/github.in diff --git a/requirements/edx/coverage.in b/requirements/edx/coverage.in index dd0104c1dc..e52a152f87 100644 --- a/requirements/edx/coverage.in +++ b/requirements/edx/coverage.in @@ -14,4 +14,3 @@ coverage # Code coverage testing for Python diff-cover # Automatically find diff lines that need test coverage -pandas # Used to process warnings generated by pytest diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index 48a2328b15..21ef45bbf2 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -12,11 +12,7 @@ jinja2-pluralize==0.3.0 # via diff-cover jinja2==2.11.2 # via diff-cover, jinja2-pluralize markupsafe==1.1.1 # via jinja2 more-itertools==8.3.0 # via zipp -numpy==1.18.4 # via pandas -pandas==0.22.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/coverage.in pluggy==0.13.1 # via diff-cover pygments==2.6.1 # via diff-cover -python-dateutil==2.4.0 # via -c requirements/edx/../constraints.txt, pandas -pytz==2020.1 # via pandas -six==1.15.0 # via diff-cover, python-dateutil +six==1.15.0 # via diff-cover zipp==1.0.0 # via -c requirements/edx/../constraints.txt, importlib-metadata diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index cc55d66e53..a6976b3008 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -59,7 +59,7 @@ cssselect==1.1.0 # via -r requirements/edx/testing.txt, pyquery cssutils==1.0.2 # via -r requirements/edx/testing.txt, pynliner ddt==1.3.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, xblock-drag-and-drop-v2, xblock-poll decorator==4.4.2 # via -r requirements/edx/testing.txt, pycontracts -defusedxml==0.6.0 # via -r requirements/edx/testing.txt, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core +defusedxml==0.7.0rc1 # via -r requirements/edx/testing.txt, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core diff-cover==2.6.1 # via -r requirements/edx/testing.txt distlib==0.3.0 # via -r requirements/edx/testing.txt, virtualenv git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/testing.txt, django-statici18n @@ -198,12 +198,11 @@ mysqlclient==1.4.6 # via -r requirements/edx/testing.txt newrelic==5.12.1.141 # via -r requirements/edx/testing.txt, edx-django-utils nltk==3.5 # via -r requirements/edx/testing.txt, chem nodeenv==1.3.5 # via -r requirements/edx/testing.txt -numpy==1.18.4 # via -r requirements/edx/testing.txt, chem, openedx-calc, pandas, scipy +numpy==1.18.4 # via -r requirements/edx/testing.txt, chem, openedx-calc, scipy oauthlib==3.0.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, django-oauth-toolkit, lti-consumer-xblock, requests-oauthlib, social-auth-core openedx-calc==1.0.9 # via -r requirements/edx/testing.txt git+https://github.com/edx/edx-ora2.git@2.7.7#egg=ora2==2.7.7 # via -r requirements/edx/testing.txt packaging==20.4 # via -r requirements/edx/testing.txt, bleach, drf-yasg, pytest, sphinx, tox -pandas==0.22.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt path.py==12.4.0 # via -r requirements/edx/testing.txt, edx-enterprise, edx-i18n-tools, ora2, xmodule path==13.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, path.py pathlib2==2.3.5 # via -r requirements/edx/testing.txt, pytest @@ -248,14 +247,14 @@ pytest-metadata==1.8.0 # via -r requirements/edx/testing.txt, pytest-json-rep pytest-randomly==3.4.0 # via -r requirements/edx/testing.txt pytest-xdist==1.32.0 # via -r requirements/edx/testing.txt pytest==5.3.5 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, pytest-attrib, pytest-cov, pytest-django, pytest-forked, pytest-json-report, pytest-metadata, pytest-randomly, pytest-xdist -python-dateutil==2.4.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, analytics-python, botocore, edx-ace, edx-drf-extensions, edx-enterprise, edx-proctoring, faker, freezegun, icalendar, ora2, pandas, xblock +python-dateutil==2.4.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, analytics-python, botocore, edx-ace, edx-drf-extensions, edx-enterprise, edx-proctoring, faker, freezegun, icalendar, ora2, xblock python-levenshtein==0.12.0 # via -r requirements/edx/testing.txt python-memcached==1.59 # via -r requirements/edx/testing.txt python-slugify==4.0.0 # via -r requirements/edx/testing.txt, code-annotations, transifex-client python-swiftclient==3.9.0 # via -r requirements/edx/testing.txt, ora2 python3-openid==3.1.0 ; python_version >= "3" # via -r requirements/edx/testing.txt, social-auth-core python3-saml==1.9.0 # via -r requirements/edx/testing.txt -pytz==2020.1 # via -r requirements/edx/testing.txt, babel, capa, celery, django, django-ses, edx-completion, edx-enterprise, edx-proctoring, edx-submissions, edx-tincan-py35, event-tracking, fs, icalendar, ora2, pandas, xblock +pytz==2020.1 # via -r requirements/edx/testing.txt, babel, capa, celery, django, django-ses, edx-completion, edx-enterprise, edx-proctoring, edx-submissions, edx-tincan-py35, event-tracking, fs, icalendar, ora2, xblock pyuca==1.2 # via -r requirements/edx/testing.txt pywatchman==1.4.1 # via -r requirements/edx/development.in pyyaml==5.3.1 # via -r requirements/edx/testing.txt, code-annotations, edx-django-release-util, edx-i18n-tools, sphinxcontrib-openapi, xblock diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 3dd2b6e13d..83fb7db184 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -58,7 +58,7 @@ cssselect==1.1.0 # via -r requirements/edx/testing.in, pyquery cssutils==1.0.2 # via -r requirements/edx/base.txt, pynliner ddt==1.3.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, -r requirements/edx/testing.in, xblock-drag-and-drop-v2, xblock-poll decorator==4.4.2 # via -r requirements/edx/base.txt, pycontracts -defusedxml==0.6.0 # via -r requirements/edx/base.txt, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core +defusedxml==0.7.0rc1 # via -r requirements/edx/base.txt, djangorestframework-xml, ora2, python3-openid, python3-saml, safe-lxml, social-auth-core diff-cover==2.6.1 # via -r requirements/edx/coverage.txt distlib==0.3.0 # via virtualenv git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/base.txt, django-statici18n @@ -190,12 +190,11 @@ mysqlclient==1.4.6 # via -r requirements/edx/base.txt newrelic==5.12.1.141 # via -r requirements/edx/base.txt, edx-django-utils nltk==3.5 # via -r requirements/edx/base.txt, chem nodeenv==1.3.5 # via -r requirements/edx/base.txt -numpy==1.18.4 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, chem, openedx-calc, pandas, scipy +numpy==1.18.4 # via -r requirements/edx/base.txt, chem, openedx-calc, scipy oauthlib==3.0.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, django-oauth-toolkit, lti-consumer-xblock, requests-oauthlib, social-auth-core openedx-calc==1.0.9 # via -r requirements/edx/base.txt git+https://github.com/edx/edx-ora2.git@2.7.7#egg=ora2==2.7.7 # via -r requirements/edx/base.txt packaging==20.4 # via -r requirements/edx/base.txt, bleach, drf-yasg, pytest, tox -pandas==0.22.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/coverage.txt path.py==12.4.0 # via -r requirements/edx/base.txt, edx-enterprise, edx-i18n-tools, ora2, xmodule path==13.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, path.py pathlib2==2.3.5 # via pytest @@ -238,14 +237,14 @@ pytest-metadata==1.8.0 # via -r requirements/edx/testing.in, pytest-json-repo pytest-randomly==3.4.0 # via -r requirements/edx/testing.in pytest-xdist==1.32.0 # via -r requirements/edx/testing.in pytest==5.3.5 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.in, pytest-attrib, pytest-cov, pytest-django, pytest-forked, pytest-json-report, pytest-metadata, pytest-randomly, pytest-xdist -python-dateutil==2.4.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, analytics-python, botocore, edx-ace, edx-drf-extensions, edx-enterprise, edx-proctoring, faker, freezegun, icalendar, ora2, pandas, xblock +python-dateutil==2.4.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, analytics-python, botocore, edx-ace, edx-drf-extensions, edx-enterprise, edx-proctoring, faker, freezegun, icalendar, ora2, xblock python-levenshtein==0.12.0 # via -r requirements/edx/base.txt python-memcached==1.59 # via -r requirements/edx/base.txt python-slugify==4.0.0 # via -r requirements/edx/base.txt, code-annotations, transifex-client python-swiftclient==3.9.0 # via -r requirements/edx/base.txt, ora2 python3-openid==3.1.0 ; python_version >= "3" # via -r requirements/edx/base.txt, social-auth-core python3-saml==1.9.0 # via -r requirements/edx/base.txt -pytz==2020.1 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, babel, capa, celery, django, django-ses, edx-completion, edx-enterprise, edx-proctoring, edx-submissions, edx-tincan-py35, event-tracking, fs, icalendar, ora2, pandas, xblock +pytz==2020.1 # via -r requirements/edx/base.txt, babel, capa, celery, django, django-ses, edx-completion, edx-enterprise, edx-proctoring, edx-submissions, edx-tincan-py35, event-tracking, fs, icalendar, ora2, xblock pyuca==1.2 # via -r requirements/edx/base.txt pyyaml==5.3.1 # via -r requirements/edx/base.txt, code-annotations, edx-django-release-util, edx-i18n-tools, xblock radon==4.1.0 # via -r requirements/edx/testing.in