From 7f90b5d3e1829bee7af025f57d5617c23e0af6fc Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Tue, 21 Feb 2023 10:51:36 -0500 Subject: [PATCH] feat: add SurveyReportUpload and add send report method (#31431) * feat: add SurveyReportUpload and add send report method * docs: Update openedx/features/survey_report/management/commands/generate_report.py Co-authored-by: Maria Grimaldi * docs: Update openedx/features/survey_report/models.py Co-authored-by: Maria Grimaldi * refactor: Update openedx/features/survey_report/models.py Co-authored-by: Maria Grimaldi * style: Update openedx/features/survey_report/api.py Co-authored-by: Maria Grimaldi * feat: add migratio file and update status field name * refactor: rename send report method * test: fix test errors * test: add command options * refactor: simple conditional instead of ok method * fix: remove useless imports * fix: use status code instead of status * feat: add zapier endpoint * style: solve pylint issues * feat: add id field to send report data * refactor: regenerate migration with correct history * feat: add anonymous site id model * feat: update zapier url --------- Co-authored-by: Maria Grimaldi Co-authored-by: Alejandro Cardenas --- lms/envs/production.py | 4 +- lms/envs/test.py | 2 + openedx/features/survey_report/api.py | 62 ++++++++++++++++++- .../management/commands/generate_report.py | 25 ++++++-- .../commands/tests/test_generate_report.py | 2 +- .../migrations/0004_surveyreportupload.py | 24 +++++++ .../0005_surveyreportanonymoussiteid.py | 20 ++++++ openedx/features/survey_report/models.py | 38 ++++++++++++ 8 files changed, 167 insertions(+), 10 deletions(-) create mode 100644 openedx/features/survey_report/migrations/0004_surveyreportupload.py create mode 100644 openedx/features/survey_report/migrations/0005_surveyreportanonymoussiteid.py diff --git a/lms/envs/production.py b/lms/envs/production.py index 9352be3bb0..c76ca3e859 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -1083,6 +1083,8 @@ COURSE_LIVE_GLOBAL_CREDENTIALS["BIG_BLUE_BUTTON"] = { ############## Settings for survey report ############## SURVEY_REPORT_EXTRA_DATA = ENV_TOKENS.get('SURVEY_REPORT_EXTRA_DATA', {}) - +SURVEY_REPORT_ENDPOINT = ENV_TOKENS.get('SURVEY_REPORT_ENDPOINT', + 'https://hooks.zapier.com/hooks/catch/11595998/3ouwv7m/') +ANONYMOUS_SURVEY_REPORT = False AVAILABLE_DISCUSSION_TOURS = ENV_TOKENS.get('AVAILABLE_DISCUSSION_TOURS', []) diff --git a/lms/envs/test.py b/lms/envs/test.py index 634c2bb627..b1fb5f8b24 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -674,3 +674,5 @@ MFE_CONFIG_OVERRIDES = { ############## Settings for survey report ############## SURVEY_REPORT_EXTRA_DATA = {} +SURVEY_REPORT_ENDPOINT = "https://example.com/survey_report" +ANONYMOUS_SURVEY_REPORT = False diff --git a/openedx/features/survey_report/api.py b/openedx/features/survey_report/api.py index 7cd8633b2f..bce26b19b0 100644 --- a/openedx/features/survey_report/api.py +++ b/openedx/features/survey_report/api.py @@ -1,10 +1,18 @@ """ Contains the logic to manage survey report model. """ +import requests from django.conf import settings +from django.forms.models import model_to_dict -from openedx.features.survey_report.models import SurveyReport +from openedx.features.survey_report.models import ( + SurveyReport, + SurveyReportUpload, + SurveyReportAnonymousSiteID, + SURVEY_REPORT_ERROR, + SURVEY_REPORT_GENERATED +) from openedx.features.survey_report.queries import ( get_course_enrollments, get_recently_active_users, @@ -12,7 +20,6 @@ from openedx.features.survey_report.queries import ( get_registered_learners, get_unique_courses_offered ) -from .models import SURVEY_REPORT_ERROR, SURVEY_REPORT_GENERATED MAX_WEEKS_SINCE_LAST_LOGIN: int = 4 @@ -49,6 +56,57 @@ def generate_report() -> None: except (Exception, ) as update_report_error: update_report(survey_report.id, {"state": SURVEY_REPORT_ERROR}) raise Exception(update_report_error) from update_report_error + return survey_report.id + + +def get_id() -> str: + """ Generate id for the survey report.""" + if not settings.ANONYMOUS_SURVEY_REPORT: + return settings.LMS_BASE + return str(SurveyReportAnonymousSiteID.objects.get_or_create()[0].id) + + +def send_report_to_external_api(report_id: int) -> None: + """ + Send a report to Openedx endpoint and save the response in the SurveyReportUpload model. + + endpoint: The value of the setting SURVEY_REPORT_ENDPOINT + + content_type: JSON + + payload: + - courses_offered: Total number of active unique courses. + - learner: Recently active users with login in some weeks. + - registered_learners: Total number of users ever registered in the platform. + - enrollments: Total number of active enrollments in the platform. + - generated_certificates: Total number of generated certificates. + - extra_data: Extra information that will be saved in the report, E.g: site_name, openedx-release. + - created_at: Date when the report was generated, this date will send with format '%m-%d-%Y %H:%M:%S' + """ + report = SurveyReport.objects.get(id=report_id) + + fields = [ + "courses_offered", + "learners", + "registered_learners", + "generated_certificates", + "enrollments", + ] + + data = model_to_dict(report, fields=fields) + data["id"] = get_id() + data["extra_data"] = report.extra_data + data["created_at"] = report.created_at.strftime("%m-%d-%Y %H:%M:%S") + + request = requests.post(settings.SURVEY_REPORT_ENDPOINT, json=data) + + request.raise_for_status() + + SurveyReportUpload.objects.create( + report=report, + status_code=request.status_code, + request_details=request.content + ) def update_report(survey_report_id: int, data: dict) -> None: diff --git a/openedx/features/survey_report/management/commands/generate_report.py b/openedx/features/survey_report/management/commands/generate_report.py index 5c8978e4c2..1904ac9f1b 100644 --- a/openedx/features/survey_report/management/commands/generate_report.py +++ b/openedx/features/survey_report/management/commands/generate_report.py @@ -4,7 +4,7 @@ CLI command to generate survey report. from django.core.management.base import BaseCommand, CommandError -from openedx.features.survey_report.api import generate_report +from openedx.features.survey_report.api import generate_report, send_report_to_external_api class Command(BaseCommand): @@ -22,12 +22,25 @@ class Command(BaseCommand): learners ever registered, and generated certificates. """ - def handle(self, *_args, **_options): + def add_arguments(self, parser): + parser.add_argument( + '--no-send', + action='store_true', + help='Do not send the report after generated.' + ) + + def handle(self, *_args, **options): try: - generate_report() + report = generate_report() + self.stdout.write(self.style.SUCCESS('Survey report has been generated successfully.')) except Exception as error: raise CommandError(f'An error has occurred while survey report was generating. {error}') from error - self.stdout.write( - self.style.SUCCESS('Survey report has been generated successfully.') - ) + if not options['no_send']: + try: + send_report_to_external_api(report_id=report) + self.stdout.write(self.style.SUCCESS('Survey report has been sent successfully.')) + except Exception as send_error: + raise CommandError( + f'An error has occurred while survey report was sending. {send_error}' + ) from send_error diff --git a/openedx/features/survey_report/management/commands/tests/test_generate_report.py b/openedx/features/survey_report/management/commands/tests/test_generate_report.py index a2156fb019..74204981d5 100644 --- a/openedx/features/survey_report/management/commands/tests/test_generate_report.py +++ b/openedx/features/survey_report/management/commands/tests/test_generate_report.py @@ -31,7 +31,7 @@ class GenerateReportTest(TestCase): } mock_get_report_data.return_value = report_test_data out = StringIO() - call_command('generate_report', stdout=out) + call_command('generate_report', no_send=True, stdout=out) survey_report = SurveyReport.objects.last() diff --git a/openedx/features/survey_report/migrations/0004_surveyreportupload.py b/openedx/features/survey_report/migrations/0004_surveyreportupload.py new file mode 100644 index 0000000000..cc77eaef71 --- /dev/null +++ b/openedx/features/survey_report/migrations/0004_surveyreportupload.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.16 on 2023-02-01 15:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('survey_report', '0003_add_state_field_and_add_default_values_to_fields'), + ] + + operations = [ + migrations.CreateModel( + name='SurveyReportUpload', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('sent_at', models.DateTimeField(auto_now=True, help_text='Date when the report was sent to external api.')), + ('status_code', models.IntegerField(help_text='Request status code.')), + ('request_details', models.CharField(blank=True, help_text='Information about the send request.', max_length=255, null=True)), + ('report', models.ForeignKey(help_text='The report that was sent.', on_delete=django.db.models.deletion.CASCADE, to='survey_report.surveyreport')), + ], + ), + ] diff --git a/openedx/features/survey_report/migrations/0005_surveyreportanonymoussiteid.py b/openedx/features/survey_report/migrations/0005_surveyreportanonymoussiteid.py new file mode 100644 index 0000000000..ccf7d059a2 --- /dev/null +++ b/openedx/features/survey_report/migrations/0005_surveyreportanonymoussiteid.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.16 on 2023-02-10 15:45 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('survey_report', '0004_surveyreportupload'), + ] + + operations = [ + migrations.CreateModel( + name='SurveyReportAnonymousSiteID', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ], + ), + ] diff --git a/openedx/features/survey_report/models.py b/openedx/features/survey_report/models.py index ebcf4195c8..a753da620b 100644 --- a/openedx/features/survey_report/models.py +++ b/openedx/features/survey_report/models.py @@ -2,6 +2,8 @@ Survey Report models. """ +import uuid + from django.db import models from jsonfield import JSONField @@ -58,3 +60,39 @@ class SurveyReport(models.Model): class Meta: ordering = ["-created_at"] get_latest_by = 'created_at' + + +class SurveyReportUpload(models.Model): + """ + This model stores the result of the POST request made to an external service after generating a survey report. + + .. no_pii: + + fields: + - sent_at: Date when the report was sent. + - report: The report that was sent. + - status: Request status code. + - request_details: Information about the send request. + """ + sent_at = models.DateTimeField(auto_now=True, help_text="Date when the report was sent to external api.") + report = models.ForeignKey(SurveyReport, on_delete=models.CASCADE, help_text="The report that was sent.") + status_code = models.IntegerField(help_text="Request status code.") + request_details = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Information about the send request." + ) + + def is_uploaded(self) -> bool: + return 200 <= self.status_code < 300 + + +class SurveyReportAnonymousSiteID(models.Model): + """ + This model is just to save the identification which will be send to the external API when + the settings ANONYMOUS_SURVEY_REPORT is defined. + + .. no_pii: + """ + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)