From cd47b28339ede15125bf22e38c024c6d92e38e03 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 25 Jun 2018 18:03:33 -0400 Subject: [PATCH] Get rid of all code related to the CDN Experiments. These were very early experiments by the old performance team on using CDNs for videos and logging performance data. No one is looking at this data and it's just extra cruft on every page. --- .../xmodule/video_module/video_module.py | 16 +-- .../courseware/tests/test_video_mongo.py | 57 -------- lms/envs/aws.py | 4 - lms/envs/common.py | 13 -- lms/templates/main.html | 30 ---- lms/urls.py | 3 - .../core/djangoapps/performance/__init__.py | 0 .../djangoapps/performance/tests/__init__.py | 0 .../djangoapps/performance/tests/test_logs.py | 135 ------------------ openedx/core/djangoapps/performance/urls.py | 11 -- openedx/core/djangoapps/performance/utils.py | 44 ------ .../djangoapps/performance/views/__init__.py | 57 -------- 12 files changed, 2 insertions(+), 368 deletions(-) delete mode 100644 openedx/core/djangoapps/performance/__init__.py delete mode 100644 openedx/core/djangoapps/performance/tests/__init__.py delete mode 100644 openedx/core/djangoapps/performance/tests/test_logs.py delete mode 100644 openedx/core/djangoapps/performance/urls.py delete mode 100644 openedx/core/djangoapps/performance/utils.py delete mode 100644 openedx/core/djangoapps/performance/views/__init__.py diff --git a/common/lib/xmodule/xmodule/video_module/video_module.py b/common/lib/xmodule/xmodule/video_module/video_module.py index ee5e29be1e..9b821012e7 100644 --- a/common/lib/xmodule/xmodule/video_module/video_module.py +++ b/common/lib/xmodule/xmodule/video_module/video_module.py @@ -306,20 +306,8 @@ class VideoModule(VideoFields, VideoTranscriptsMixin, VideoStudentViewHandlers, transcripts = self.get_transcripts_info() track_url, transcript_language, sorted_languages = self.get_transcripts_for_student(transcripts=transcripts) - # CDN_VIDEO_URLS is only to be used here and will be deleted - # TODO(ali@edx.org): Delete this after the CDN experiment has completed. - html_id = self.location.html_id() - if self.system.user_location == 'CN' and \ - settings.FEATURES.get('ENABLE_VIDEO_BEACON', False) and \ - html_id in getattr(settings, 'CDN_VIDEO_URLS', {}).keys(): - cdn_urls = getattr(settings, 'CDN_VIDEO_URLS', {})[html_id] - cdn_exp_group, new_source = random.choice(zip(range(len(cdn_urls)), cdn_urls)) - if cdn_exp_group > 0: - sources[0] = new_source - cdn_eval = True - else: - cdn_eval = False - cdn_exp_group = None + cdn_eval = False + cdn_exp_group = None self.youtube_streams = youtube_streams or create_youtube_string(self) # pylint: disable=W0201 diff --git a/lms/djangoapps/courseware/tests/test_video_mongo.py b/lms/djangoapps/courseware/tests/test_video_mongo.py index ca6450e1a9..37eb51e758 100644 --- a/lms/djangoapps/courseware/tests/test_video_mongo.py +++ b/lms/djangoapps/courseware/tests/test_video_mongo.py @@ -987,63 +987,6 @@ class TestGetHtmlMethod(BaseTestXmodule): self.assertIn("\'poster\': \'null\'", context) -@attr(shard=7) -class TestVideoCDNRewriting(BaseTestXmodule): - """ - Tests for Video CDN. - """ - - def setUp(self, *args, **kwargs): - super(TestVideoCDNRewriting, self).setUp(*args, **kwargs) - self.original_video_file = "original_video.mp4" - self.original_video_url = "http://www.originalvideo.com/" + self.original_video_file - - @patch.dict("django.conf.settings.CDN_VIDEO_URLS", - {"CN": "https://chinacdn.cn/"}) - def test_rewrite_video_url_success(self): - """ - Test successful CDN request. - """ - cdn_response_video_url = settings.CDN_VIDEO_URLS["CN"] + self.original_video_file - - self.assertEqual( - rewrite_video_url(settings.CDN_VIDEO_URLS["CN"], self.original_video_url), - cdn_response_video_url - ) - - @patch.dict("django.conf.settings.CDN_VIDEO_URLS", - {"CN": "https://chinacdn.cn/"}) - def test_rewrite_url_concat(self): - """ - Test that written URLs are returned clean despite input - """ - cdn_response_video_url = settings.CDN_VIDEO_URLS["CN"] + "original_video.mp4" - - self.assertEqual( - rewrite_video_url(settings.CDN_VIDEO_URLS["CN"] + "///", self.original_video_url), - cdn_response_video_url - ) - - def test_rewrite_video_url_invalid_url(self): - """ - Test if no alternative video in CDN exists. - """ - invalid_cdn_url = 'http://http://fakecdn.com/' - self.assertIsNone(rewrite_video_url(invalid_cdn_url, self.original_video_url)) - - def test_none_args(self): - """ - Ensure None args return None - """ - self.assertIsNone(rewrite_video_url(None, None)) - - def test_emptystring_args(self): - """ - Ensure emptyrstring args return None - """ - self.assertIsNone(rewrite_video_url("", "")) - - @attr(shard=7) @ddt.ddt class TestVideoDescriptorInitialization(BaseTestXmodule): diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 2d4d79606c..a497f31c90 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -808,10 +808,6 @@ VIDEO_IMAGE_SETTINGS = ENV_TOKENS.get('VIDEO_IMAGE_SETTINGS', VIDEO_IMAGE_SETTIN ##### VIDEO TRANSCRIPTS STORAGE ##### VIDEO_TRANSCRIPTS_SETTINGS = ENV_TOKENS.get('VIDEO_TRANSCRIPTS_SETTINGS', VIDEO_TRANSCRIPTS_SETTINGS) -##### CDN EXPERIMENT/MONITORING FLAGS ##### -CDN_VIDEO_URLS = ENV_TOKENS.get('CDN_VIDEO_URLS', CDN_VIDEO_URLS) -ONLOAD_BEACON_SAMPLE_RATE = ENV_TOKENS.get('ONLOAD_BEACON_SAMPLE_RATE', ONLOAD_BEACON_SAMPLE_RATE) - ##### ECOMMERCE API CONFIGURATION SETTINGS ##### ECOMMERCE_PUBLIC_URL_ROOT = ENV_TOKENS.get('ECOMMERCE_PUBLIC_URL_ROOT', ECOMMERCE_PUBLIC_URL_ROOT) ECOMMERCE_API_URL = ENV_TOKENS.get('ECOMMERCE_API_URL', ECOMMERCE_API_URL) diff --git a/lms/envs/common.py b/lms/envs/common.py index 2946023bd0..faba36a698 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -301,12 +301,6 @@ FEATURES = { # log all information from cybersource callbacks 'LOG_POSTPAY_CALLBACKS': True, - # enable beacons for video timing statistics - 'ENABLE_VIDEO_BEACON': False, - - # enable beacons for lms onload event statistics - 'ENABLE_ONLOAD_BEACON': False, - # Toggle platform-wide course licensing 'LICENSING': False, @@ -2996,13 +2990,6 @@ SEARCH_FILTER_GENERATOR = "lms.lib.courseware_search.lms_filter_generator.LmsSea # Override to skip enrollment start date filtering in course search SEARCH_SKIP_ENROLLMENT_START_DATE_FILTERING = False -### PERFORMANCE EXPERIMENT SETTINGS ### -# CDN experiment/monitoring flags -CDN_VIDEO_URLS = {} - -# Page onload event sampling rate (min 0.0, max 1.0) -ONLOAD_BEACON_SAMPLE_RATE = 0.0 - # The configuration visibility of account fields. ACCOUNT_VISIBILITY_CONFIGURATION = { # Default visibility level for accounts without a specified value diff --git a/lms/templates/main.html b/lms/templates/main.html index 019d838435..e147af4807 100644 --- a/lms/templates/main.html +++ b/lms/templates/main.html @@ -203,33 +203,3 @@ from pipeline_mako import render_require_js_path_overrides next=urlquote_plus(login_redirect_url if login_redirect_url else request.path) ) if (login_redirect_url or request) else "" } - - -% if settings.FEATURES.get('ENABLE_ONLOAD_BEACON', False): - -% endif diff --git a/lms/urls.py b/lms/urls.py index bd5bad78e7..0f184ae166 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -74,9 +74,6 @@ urlpatterns = [ # Event tracking endpoints url(r'', include('track.urls')), - # Performance endpoints - url(r'', include('openedx.core.djangoapps.performance.urls')), - # Static template view endpoints like blog, faq, etc. url(r'', include('static_template_view.urls')), diff --git a/openedx/core/djangoapps/performance/__init__.py b/openedx/core/djangoapps/performance/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/openedx/core/djangoapps/performance/tests/__init__.py b/openedx/core/djangoapps/performance/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/openedx/core/djangoapps/performance/tests/test_logs.py b/openedx/core/djangoapps/performance/tests/test_logs.py deleted file mode 100644 index f53938aa2c..0000000000 --- a/openedx/core/djangoapps/performance/tests/test_logs.py +++ /dev/null @@ -1,135 +0,0 @@ -# pylint: disable=no-member -""" -Tests that performance data is successfully logged. -""" -import datetime -import dateutil -import json - -import logging -from StringIO import StringIO - -from django.test import TestCase -from django.test.client import RequestFactory -from openedx.core.djangoapps.performance.views import performance_log - - -class PerformanceTrackingTest(TestCase): - """ - Tests that performance logs correctly handle events - """ - - def setUp(self): - super(PerformanceTrackingTest, self).setUp() - self.request_factory = RequestFactory() - self.stream = StringIO() - self.handler = logging.StreamHandler(self.stream) - self.log = logging.getLogger() - self.log.setLevel(logging.INFO) - for handler in self.log.handlers: - self.log.removeHandler(handler) - self.log.addHandler(self.handler) - self.addCleanup(self.log.removeHandler, self.handler) - self.addCleanup(self.handler.close) - - def test_empty_get(self): - request = self.request_factory.get('/performance') - pre_time = datetime.datetime.utcnow() - performance_log(request) - post_time = datetime.datetime.utcnow() - self.handler.flush() - logged_value = json.loads(self.stream.getvalue().strip()) - self.assertEqual(logged_value['accept_language'], '') - self.assertEqual(logged_value['agent'], '') - self.assertEqual(logged_value['event'], '') - self.assertEqual(logged_value['event_source'], 'browser') - self.assertEqual(logged_value['expgroup'], '') - self.assertEqual(logged_value['id'], '') - self.assertEqual(logged_value['page'], '') - self.assertEqual(logged_value['referer'], '') - self.assertEqual(logged_value['value'], '') - logged_time = dateutil.parser.parse(logged_value['time']).replace(tzinfo=None) - self.assertLessEqual(pre_time, logged_time) - self.assertGreaterEqual(post_time, logged_time) - - def test_empty_post(self): - request = self.request_factory.post('/performance') - pre_time = datetime.datetime.utcnow() - performance_log(request) - post_time = datetime.datetime.utcnow() - self.handler.flush() - logged_value = json.loads(self.stream.getvalue().strip()) - self.assertEqual(logged_value['accept_language'], '') - self.assertEqual(logged_value['agent'], '') - self.assertEqual(logged_value['event'], '') - self.assertEqual(logged_value['event_source'], 'browser') - self.assertEqual(logged_value['expgroup'], '') - self.assertEqual(logged_value['id'], '') - self.assertEqual(logged_value['page'], '') - self.assertEqual(logged_value['referer'], '') - self.assertEqual(logged_value['value'], '') - logged_time = dateutil.parser.parse(logged_value['time']).replace(tzinfo=None) - self.assertLessEqual(pre_time, logged_time) - self.assertGreaterEqual(post_time, logged_time) - - def test_populated_get(self): - request = self.request_factory.get('/performance', - {'event': "a_great_event", - 'id': "12345012345", - 'expgroup': "17", 'page': "atestpage", - 'value': "100234"}) - request.META['HTTP_ACCEPT_LANGUAGE'] = "en" - request.META['HTTP_REFERER'] = "https://www.edx.org/evilpage" - request.META['HTTP_USER_AGENT'] = "Mozilla/5.0" - request.META['REMOTE_ADDR'] = "18.19.20.21" - request.META['SERVER_NAME'] = "some-aws-server" - pre_time = datetime.datetime.utcnow() - performance_log(request) - post_time = datetime.datetime.utcnow() - self.handler.flush() - logged_value = json.loads(self.stream.getvalue().strip()) - self.assertEqual(logged_value['accept_language'], 'en') - self.assertEqual(logged_value['agent'], 'Mozilla/5.0') - self.assertEqual(logged_value['event'], 'a_great_event') - self.assertEqual(logged_value['event_source'], 'browser') - self.assertEqual(logged_value['expgroup'], '17') - self.assertEqual(logged_value['host'], 'some-aws-server') - self.assertEqual(logged_value['id'], '12345012345') - self.assertEqual(logged_value['ip'], '18.19.20.21') - self.assertEqual(logged_value['page'], 'atestpage') - self.assertEqual(logged_value['referer'], 'https://www.edx.org/evilpage') - self.assertEqual(logged_value['value'], '100234') - logged_time = dateutil.parser.parse(logged_value['time']).replace(tzinfo=None) - self.assertLessEqual(pre_time, logged_time) - self.assertGreaterEqual(post_time, logged_time) - - def test_populated_post(self): - request = self.request_factory.post('/performance', - {'event': "a_great_event", - 'id': "12345012345", - 'expgroup': "17", 'page': "atestpage", - 'value': "100234"}) - request.META['HTTP_ACCEPT_LANGUAGE'] = "en" - request.META['HTTP_REFERER'] = "https://www.edx.org/evilpage" - request.META['HTTP_USER_AGENT'] = "Mozilla/5.0" - request.META['REMOTE_ADDR'] = "18.19.20.21" - request.META['SERVER_NAME'] = "some-aws-server" - pre_time = datetime.datetime.utcnow() - performance_log(request) - post_time = datetime.datetime.utcnow() - self.handler.flush() - logged_value = json.loads(self.stream.getvalue().strip()) - self.assertEqual(logged_value['accept_language'], 'en') - self.assertEqual(logged_value['agent'], 'Mozilla/5.0') - self.assertEqual(logged_value['event'], 'a_great_event') - self.assertEqual(logged_value['event_source'], 'browser') - self.assertEqual(logged_value['expgroup'], '17') - self.assertEqual(logged_value['host'], 'some-aws-server') - self.assertEqual(logged_value['id'], '12345012345') - self.assertEqual(logged_value['ip'], '18.19.20.21') - self.assertEqual(logged_value['page'], 'atestpage') - self.assertEqual(logged_value['referer'], 'https://www.edx.org/evilpage') - self.assertEqual(logged_value['value'], '100234') - logged_time = dateutil.parser.parse(logged_value['time']).replace(tzinfo=None) - self.assertLessEqual(pre_time, logged_time) - self.assertGreaterEqual(post_time, logged_time) diff --git a/openedx/core/djangoapps/performance/urls.py b/openedx/core/djangoapps/performance/urls.py deleted file mode 100644 index 41cefb995a..0000000000 --- a/openedx/core/djangoapps/performance/urls.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -URLs for performance app -""" - -from django.conf.urls import url - -from openedx.core.djangoapps.performance.views import performance_log - -urlpatterns = [ - url(r'^performance$', performance_log), -] diff --git a/openedx/core/djangoapps/performance/utils.py b/openedx/core/djangoapps/performance/utils.py deleted file mode 100644 index 12f4664b96..0000000000 --- a/openedx/core/djangoapps/performance/utils.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Common utilities for performance testing. -""" -from contextlib import contextmanager - - -def collect_profile_func(file_prefix, enabled=False): - """ - Method decorator for collecting profile. - """ - import functools - - def _outer(func): - """ - Outer function decorator. - """ - @functools.wraps(func) - def _inner(self, *args, **kwargs): - """ - Inner wrapper function. - """ - if enabled: - with collect_profile(file_prefix): - return func(self, *args, **kwargs) - else: - return func(self, *args, **kwargs) - return _inner - return _outer - - -@contextmanager -def collect_profile(file_prefix): - """ - Context manager to collect profile information. - """ - import cProfile - import uuid - profiler = cProfile.Profile() - profiler.enable() - try: - yield - finally: - profiler.disable() - profiler.dump_stats("{0}_{1}_master.profile".format(file_prefix, uuid.uuid4())) diff --git a/openedx/core/djangoapps/performance/views/__init__.py b/openedx/core/djangoapps/performance/views/__init__.py deleted file mode 100644 index f82bd24d56..0000000000 --- a/openedx/core/djangoapps/performance/views/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Views for logging performance data. -""" -import datetime -import json -import logging - -from django.http import HttpResponse - -from track.utils import DateTimeJSONEncoder - - -log = logging.getLogger("perflog") - - -def _get_request_header(request, header_name, default=''): - """Helper method to get header values from a request's META dict, if present.""" - if request is not None and hasattr(request, 'META') and header_name in request.META: - return request.META[header_name] - else: - return default - - -def _get_request_value(request, value_name, default=''): - """Helper method to get header values from a request's GET or POST dicts, if present.""" - if request is not None and hasattr(request, 'GET') and value_name in request.GET: - return request.GET[value_name] - elif request is not None and hasattr(request, 'POST') and value_name in request.POST: - return request.POST[value_name] - else: - return default - - -def performance_log(request): - """ - Log when POST call to "performance" URL is made by a user. - Request should provide "event" and "page" arguments. - """ - - event = { - "ip": _get_request_header(request, 'REMOTE_ADDR'), - "referer": _get_request_header(request, 'HTTP_REFERER'), - "accept_language": _get_request_header(request, 'HTTP_ACCEPT_LANGUAGE'), - "event_source": "browser", - "event": _get_request_value(request, 'event'), - "agent": _get_request_header(request, 'HTTP_USER_AGENT'), - "page": _get_request_value(request, 'page'), - "id": _get_request_value(request, 'id'), - "expgroup": _get_request_value(request, 'expgroup'), - "value": _get_request_value(request, 'value'), - "time": datetime.datetime.utcnow(), - "host": _get_request_header(request, 'SERVER_NAME'), - } - - log.info(json.dumps(event, cls=DateTimeJSONEncoder)) - - return HttpResponse(status=204)