Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Get rid of all code related to the CDN Experiments.

Browse Source 
These were very early experiments by the old performance team
on using CDNs for videos and logging performance data.  No one
is looking at this data and it's just extra cruft on every page.
This commit is contained in:
Feanil Patel
2018-06-25 18:03:33 -04:00
parent 5f6069eeaa
commit cd47b28339
12 changed files with 2 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
common/lib/xmodule/xmodule/video_module/video_module.py
View File

@@ -306,20 +306,8 @@ class VideoModule(VideoFields, VideoTranscriptsMixin, VideoStudentViewHandlers,
transcripts = self.get_transcripts_info()
track_url, transcript_language, sorted_languages = self.get_transcripts_for_student(transcripts=transcripts)
# CDN_VIDEO_URLS is only to be used here and will be deleted
# TODO(ali@edx.org): Delete this after the CDN experiment has completed.
html_id = self.location.html_id()
if self.system.user_location == 'CN' and \
settings.FEATURES.get('ENABLE_VIDEO_BEACON', False) and \
html_id in getattr(settings, 'CDN_VIDEO_URLS', {}).keys():
cdn_urls = getattr(settings, 'CDN_VIDEO_URLS', {})[html_id]
cdn_exp_group, new_source = random.choice(zip(range(len(cdn_urls)), cdn_urls))
if cdn_exp_group > 0:
sources[0] = new_source
cdn_eval = True
else:
cdn_eval = False
cdn_exp_group = None
cdn_eval = False
cdn_exp_group = None
self.youtube_streams = youtube_streams or create_youtube_string(self) # pylint: disable=W0201

57
lms/djangoapps/courseware/tests/test_video_mongo.py
View File

@@ -987,63 +987,6 @@ class TestGetHtmlMethod(BaseTestXmodule):
self.assertIn("\'poster\': \'null\'", context)
@attr(shard=7)
class TestVideoCDNRewriting(BaseTestXmodule):
"""
Tests for Video CDN.
"""
def setUp(self, *args, **kwargs):
super(TestVideoCDNRewriting, self).setUp(*args, **kwargs)
self.original_video_file = "original_video.mp4"
self.original_video_url = "http://www.originalvideo.com/" + self.original_video_file
@patch.dict("django.conf.settings.CDN_VIDEO_URLS",
{"CN": "https://chinacdn.cn/"})
def test_rewrite_video_url_success(self):
"""
Test successful CDN request.
"""
cdn_response_video_url = settings.CDN_VIDEO_URLS["CN"] + self.original_video_file
self.assertEqual(
rewrite_video_url(settings.CDN_VIDEO_URLS["CN"], self.original_video_url),
cdn_response_video_url
)
@patch.dict("django.conf.settings.CDN_VIDEO_URLS",
{"CN": "https://chinacdn.cn/"})
def test_rewrite_url_concat(self):
"""
Test that written URLs are returned clean despite input
"""
cdn_response_video_url = settings.CDN_VIDEO_URLS["CN"] + "original_video.mp4"
self.assertEqual(
rewrite_video_url(settings.CDN_VIDEO_URLS["CN"] + "///", self.original_video_url),
cdn_response_video_url
)
def test_rewrite_video_url_invalid_url(self):
"""
Test if no alternative video in CDN exists.
"""
invalid_cdn_url = 'http://http://fakecdn.com/'
self.assertIsNone(rewrite_video_url(invalid_cdn_url, self.original_video_url))
def test_none_args(self):
"""
Ensure None args return None
"""
self.assertIsNone(rewrite_video_url(None, None))
def test_emptystring_args(self):
"""
Ensure emptyrstring args return None
"""
self.assertIsNone(rewrite_video_url("", ""))
@attr(shard=7)
@ddt.ddt
class TestVideoDescriptorInitialization(BaseTestXmodule):

4
lms/envs/aws.py
View File

@@ -808,10 +808,6 @@ VIDEO_IMAGE_SETTINGS = ENV_TOKENS.get('VIDEO_IMAGE_SETTINGS', VIDEO_IMAGE_SETTIN
##### VIDEO TRANSCRIPTS STORAGE #####
VIDEO_TRANSCRIPTS_SETTINGS = ENV_TOKENS.get('VIDEO_TRANSCRIPTS_SETTINGS', VIDEO_TRANSCRIPTS_SETTINGS)
##### CDN EXPERIMENT/MONITORING FLAGS #####
CDN_VIDEO_URLS = ENV_TOKENS.get('CDN_VIDEO_URLS', CDN_VIDEO_URLS)
ONLOAD_BEACON_SAMPLE_RATE = ENV_TOKENS.get('ONLOAD_BEACON_SAMPLE_RATE', ONLOAD_BEACON_SAMPLE_RATE)
##### ECOMMERCE API CONFIGURATION SETTINGS #####
ECOMMERCE_PUBLIC_URL_ROOT = ENV_TOKENS.get('ECOMMERCE_PUBLIC_URL_ROOT', ECOMMERCE_PUBLIC_URL_ROOT)
ECOMMERCE_API_URL = ENV_TOKENS.get('ECOMMERCE_API_URL', ECOMMERCE_API_URL)

13
lms/envs/common.py
View File

@@ -301,12 +301,6 @@ FEATURES = {
# log all information from cybersource callbacks
'LOG_POSTPAY_CALLBACKS': True,
# enable beacons for video timing statistics
'ENABLE_VIDEO_BEACON': False,
# enable beacons for lms onload event statistics
'ENABLE_ONLOAD_BEACON': False,
# Toggle platform-wide course licensing
'LICENSING': False,
@@ -2996,13 +2990,6 @@ SEARCH_FILTER_GENERATOR = "lms.lib.courseware_search.lms_filter_generator.LmsSea
# Override to skip enrollment start date filtering in course search
SEARCH_SKIP_ENROLLMENT_START_DATE_FILTERING = False
### PERFORMANCE EXPERIMENT SETTINGS ###
# CDN experiment/monitoring flags
CDN_VIDEO_URLS = {}
# Page onload event sampling rate (min 0.0, max 1.0)
ONLOAD_BEACON_SAMPLE_RATE = 0.0
# The configuration visibility of account fields.
ACCOUNT_VISIBILITY_CONFIGURATION = {
# Default visibility level for accounts without a specified value

30
lms/templates/main.html
View File

@@ -203,33 +203,3 @@ from pipeline_mako import render_require_js_path_overrides
next=urlquote_plus(login_redirect_url if login_redirect_url else request.path)
) if (login_redirect_url or request) else ""
}</%def>
<!-- Performance beacon for onload times -->
% if settings.FEATURES.get('ENABLE_ONLOAD_BEACON', False):
<script>
(function () {
var sample_rate = ${settings.ONLOAD_BEACON_SAMPLE_RATE | n, dump_js_escaped_json};
var roll = Math.floor(Math.random() * 100)/100;
var onloadBeaconSent = false;
if(roll < sample_rate){
$(window).load(function() {
setTimeout(function(){
var t = window.performance.timing;
var data = {
event: "onload",
value: t.loadEventEnd - t.navigationStart,
page: window.location.href,
};
if (!onloadBeaconSent) {
$.ajax({method: "POST", url: "/performance", data: data});
}
onloadBeaconSent = true;
}, 0);
});
}
}());
</script>
% endif

3
lms/urls.py
View File

@@ -74,9 +74,6 @@ urlpatterns = [
# Event tracking endpoints
url(r'', include('track.urls')),
# Performance endpoints
url(r'', include('openedx.core.djangoapps.performance.urls')),
# Static template view endpoints like blog, faq, etc.
url(r'', include('static_template_view.urls')),

0
openedx/core/djangoapps/performance/__init__.py
View File

0
openedx/core/djangoapps/performance/tests/__init__.py
View File

135
openedx/core/djangoapps/performance/tests/test_logs.py
View File

@@ -1,135 +0,0 @@
# pylint: disable=no-member
"""
Tests that performance data is successfully logged.
"""
import datetime
import dateutil
import json
import logging
from StringIO import StringIO
from django.test import TestCase
from django.test.client import RequestFactory
from openedx.core.djangoapps.performance.views import performance_log
class PerformanceTrackingTest(TestCase):
"""
Tests that performance logs correctly handle events
"""
def setUp(self):
super(PerformanceTrackingTest, self).setUp()
self.request_factory = RequestFactory()
self.stream = StringIO()
self.handler = logging.StreamHandler(self.stream)
self.log = logging.getLogger()
self.log.setLevel(logging.INFO)
for handler in self.log.handlers:
self.log.removeHandler(handler)
self.log.addHandler(self.handler)
self.addCleanup(self.log.removeHandler, self.handler)
self.addCleanup(self.handler.close)
def test_empty_get(self):
request = self.request_factory.get('/performance')
pre_time = datetime.datetime.utcnow()
performance_log(request)
post_time = datetime.datetime.utcnow()
self.handler.flush()
logged_value = json.loads(self.stream.getvalue().strip())
self.assertEqual(logged_value['accept_language'], '')
self.assertEqual(logged_value['agent'], '')
self.assertEqual(logged_value['event'], '')
self.assertEqual(logged_value['event_source'], 'browser')
self.assertEqual(logged_value['expgroup'], '')
self.assertEqual(logged_value['id'], '')
self.assertEqual(logged_value['page'], '')
self.assertEqual(logged_value['referer'], '')
self.assertEqual(logged_value['value'], '')
logged_time = dateutil.parser.parse(logged_value['time']).replace(tzinfo=None)
self.assertLessEqual(pre_time, logged_time)
self.assertGreaterEqual(post_time, logged_time)
def test_empty_post(self):
request = self.request_factory.post('/performance')
pre_time = datetime.datetime.utcnow()
performance_log(request)
post_time = datetime.datetime.utcnow()
self.handler.flush()
logged_value = json.loads(self.stream.getvalue().strip())
self.assertEqual(logged_value['accept_language'], '')
self.assertEqual(logged_value['agent'], '')
self.assertEqual(logged_value['event'], '')
self.assertEqual(logged_value['event_source'], 'browser')
self.assertEqual(logged_value['expgroup'], '')
self.assertEqual(logged_value['id'], '')
self.assertEqual(logged_value['page'], '')
self.assertEqual(logged_value['referer'], '')
self.assertEqual(logged_value['value'], '')
logged_time = dateutil.parser.parse(logged_value['time']).replace(tzinfo=None)
self.assertLessEqual(pre_time, logged_time)
self.assertGreaterEqual(post_time, logged_time)
def test_populated_get(self):
request = self.request_factory.get('/performance',
{'event': "a_great_event",
'id': "12345012345",
'expgroup': "17", 'page': "atestpage",
'value': "100234"})
request.META['HTTP_ACCEPT_LANGUAGE'] = "en"
request.META['HTTP_REFERER'] = "https://www.edx.org/evilpage"
request.META['HTTP_USER_AGENT'] = "Mozilla/5.0"
request.META['REMOTE_ADDR'] = "18.19.20.21"
request.META['SERVER_NAME'] = "some-aws-server"
pre_time = datetime.datetime.utcnow()
performance_log(request)
post_time = datetime.datetime.utcnow()
self.handler.flush()
logged_value = json.loads(self.stream.getvalue().strip())
self.assertEqual(logged_value['accept_language'], 'en')
self.assertEqual(logged_value['agent'], 'Mozilla/5.0')
self.assertEqual(logged_value['event'], 'a_great_event')
self.assertEqual(logged_value['event_source'], 'browser')
self.assertEqual(logged_value['expgroup'], '17')
self.assertEqual(logged_value['host'], 'some-aws-server')
self.assertEqual(logged_value['id'], '12345012345')
self.assertEqual(logged_value['ip'], '18.19.20.21')
self.assertEqual(logged_value['page'], 'atestpage')
self.assertEqual(logged_value['referer'], 'https://www.edx.org/evilpage')
self.assertEqual(logged_value['value'], '100234')
logged_time = dateutil.parser.parse(logged_value['time']).replace(tzinfo=None)
self.assertLessEqual(pre_time, logged_time)
self.assertGreaterEqual(post_time, logged_time)
def test_populated_post(self):
request = self.request_factory.post('/performance',
{'event': "a_great_event",
'id': "12345012345",
'expgroup': "17", 'page': "atestpage",
'value': "100234"})
request.META['HTTP_ACCEPT_LANGUAGE'] = "en"
request.META['HTTP_REFERER'] = "https://www.edx.org/evilpage"
request.META['HTTP_USER_AGENT'] = "Mozilla/5.0"
request.META['REMOTE_ADDR'] = "18.19.20.21"
request.META['SERVER_NAME'] = "some-aws-server"
pre_time = datetime.datetime.utcnow()
performance_log(request)
post_time = datetime.datetime.utcnow()
self.handler.flush()
logged_value = json.loads(self.stream.getvalue().strip())
self.assertEqual(logged_value['accept_language'], 'en')
self.assertEqual(logged_value['agent'], 'Mozilla/5.0')
self.assertEqual(logged_value['event'], 'a_great_event')
self.assertEqual(logged_value['event_source'], 'browser')
self.assertEqual(logged_value['expgroup'], '17')
self.assertEqual(logged_value['host'], 'some-aws-server')
self.assertEqual(logged_value['id'], '12345012345')
self.assertEqual(logged_value['ip'], '18.19.20.21')
self.assertEqual(logged_value['page'], 'atestpage')
self.assertEqual(logged_value['referer'], 'https://www.edx.org/evilpage')
self.assertEqual(logged_value['value'], '100234')
logged_time = dateutil.parser.parse(logged_value['time']).replace(tzinfo=None)
self.assertLessEqual(pre_time, logged_time)
self.assertGreaterEqual(post_time, logged_time)

11
openedx/core/djangoapps/performance/urls.py
View File

@@ -1,11 +0,0 @@
"""
URLs for performance app
"""
from django.conf.urls import url
from openedx.core.djangoapps.performance.views import performance_log
urlpatterns = [
url(r'^performance$', performance_log),
]

44
openedx/core/djangoapps/performance/utils.py
View File

@@ -1,44 +0,0 @@
"""
Common utilities for performance testing.
"""
from contextlib import contextmanager
def collect_profile_func(file_prefix, enabled=False):
"""
Method decorator for collecting profile.
"""
import functools
def _outer(func):
"""
Outer function decorator.
"""
@functools.wraps(func)
def _inner(self, *args, **kwargs):
"""
Inner wrapper function.
"""
if enabled:
with collect_profile(file_prefix):
return func(self, *args, **kwargs)
else:
return func(self, *args, **kwargs)
return _inner
return _outer
@contextmanager
def collect_profile(file_prefix):
"""
Context manager to collect profile information.
"""
import cProfile
import uuid
profiler = cProfile.Profile()
profiler.enable()
try:
yield
finally:
profiler.disable()
profiler.dump_stats("{0}_{1}_master.profile".format(file_prefix, uuid.uuid4()))

57
openedx/core/djangoapps/performance/views/__init__.py
View File

@@ -1,57 +0,0 @@
"""
Views for logging performance data.
"""
import datetime
import json
import logging
from django.http import HttpResponse
from track.utils import DateTimeJSONEncoder
log = logging.getLogger("perflog")
def _get_request_header(request, header_name, default=''):
"""Helper method to get header values from a request's META dict, if present."""
if request is not None and hasattr(request, 'META') and header_name in request.META:
return request.META[header_name]
else:
return default
def _get_request_value(request, value_name, default=''):
"""Helper method to get header values from a request's GET or POST dicts, if present."""
if request is not None and hasattr(request, 'GET') and value_name in request.GET:
return request.GET[value_name]
elif request is not None and hasattr(request, 'POST') and value_name in request.POST:
return request.POST[value_name]
else:
return default
def performance_log(request):
"""
Log when POST call to "performance" URL is made by a user.
Request should provide "event" and "page" arguments.
"""
event = {
"ip": _get_request_header(request, 'REMOTE_ADDR'),
"referer": _get_request_header(request, 'HTTP_REFERER'),
"accept_language": _get_request_header(request, 'HTTP_ACCEPT_LANGUAGE'),
"event_source": "browser",
"event": _get_request_value(request, 'event'),
"agent": _get_request_header(request, 'HTTP_USER_AGENT'),
"page": _get_request_value(request, 'page'),
"id": _get_request_value(request, 'id'),
"expgroup": _get_request_value(request, 'expgroup'),
"value": _get_request_value(request, 'value'),
"time": datetime.datetime.utcnow(),
"host": _get_request_header(request, 'SERVER_NAME'),
}
log.info(json.dumps(event, cls=DateTimeJSONEncoder))
return HttpResponse(status=204)