diff --git a/.annotation_safe_list.yml b/.annotation_safe_list.yml new file mode 100644 index 0000000000..e5acad5bb3 --- /dev/null +++ b/.annotation_safe_list.yml @@ -0,0 +1,320 @@ +# This is a Code Annotations automatically-generated Django model safelist file. +# These models must be annotated as follows in order to be counted in the coverage report. +# See https://code-annotations.readthedocs.io/en/latest/safelist.html for more information. +# +# fake_app_1.FakeModelName: +# ".. no_pii::": "This model has no PII" +# fake_app_2.FakeModel2: +# ".. choice_annotation::": foo, bar, baz + +# Via Django +auth.Group: + ".. no_pii:" : "No PII" +auth.Permission: + ".. no_pii:" : "No PII" +auth.User: + ".. pii:": "Contains username, password, and email address, retired in AccountRetirementView" + ".. pii_types:" : username, email_address, password + ".. pii_retirement:" : local_api +contenttypes.ContentType: + ".. no_pii:": "No PII" +admin.LogEntry: + ".. no_pii:": "No PII" +redirects.Redirect: + ".. no_pii:": "No PII" +sessions.Session: + ".. no_pii:": "No PII" +sites.Site: + ".. no_pii:": "No PII" + +# Automatically generated models in edx-enterprise that can't be annotated there +consent.HistoricalDataSharingConsent: + ".. pii:": "The username field inherited from Consent contains PII." + ".. pii_types:": username + ".. pii_retirement:": consumer_api +degreed.HistoricalDegreedEnterpriseCustomerConfiguration: + ".. no_pii:": "No PII" +enterprise.HistoricalEnrollmentNotificationEmailTemplate: + ".. no_pii:": "No PII" +enterprise.HistoricalEnterpriseCourseEnrollment: + ".. no_pii:": "No PII" +enterprise.HistoricalEnterpriseCustomer: + ".. no_pii:": "No PII" +enterprise.HistoricalEnterpriseCustomerCatalog: + ".. no_pii:": "No PII" +enterprise.HistoricalEnterpriseCustomerEntitlement: + ".. no_pii:": "No PII" + +# Via ORA2 +assessment.Assessment: + ".. no_pii:": "No PII" +assessment.AssessmentFeedback: + ".. no_pii:": "No PII" +assessment.AssessmentFeedbackOption: + ".. no_pii:": "No PII" +assessment.AssessmentPart: + ".. no_pii:": "No PII" +assessment.Criterion: + ".. no_pii:": "No PII" +assessment.CriterionOption: + ".. no_pii:": "No PII" +assessment.PeerWorkflow: + ".. no_pii:": "No PII" +assessment.PeerWorkflowItem: + ".. no_pii:": "No PII" +assessment.Rubric: + ".. no_pii:": "No PII" +assessment.StaffWorkflow: + ".. no_pii:": "No PII" +assessment.StudentTrainingWorkflow: + ".. no_pii:": "No PII" +assessment.StudentTrainingWorkflowItem: + ".. no_pii:": "No PII" +assessment.TrainingExample: + ".. no_pii:": "No PII" +workflow.AssessmentWorkflow: + ".. no_pii:": "No PII" +workflow.AssessmentWorkflowCancellation: + ".. no_pii:": "No PII" +workflow.AssessmentWorkflowStep: + ".. no_pii:": "No PII" + +# Via edx-celeryutils +celery_utils.ChordData: + ".. no_pii:": "No PII" +celery_utils.FailedTask: + ".. no_pii:": "No PII" + +# Via completion XBlock +completion.BlockCompletion: + ".. no_pii:": "No PII" + +# Via django_notify (required / installed by wiki) +django_notify.Notification: + ".. no_pii:": "No PII" +django_notify.NotificationType: + ".. no_pii:": "No PII" +django_notify.Settings: + ".. no_pii:": "No PII" +django_notify.Subscription: + ".. no_pii:": "No PII" + +# Via django-openid-auth https://github.com/edx/django-openid-auth +django_openid_auth.Association: + ".. no_pii:": "No PII" +django_openid_auth.Nonce: + ".. no_pii:": "No PII" +django_openid_auth.UserOpenID: + ".. pii:": "User OpenID associations. Not used and empty on edx.org, therefore not retired." + ".. pii_types:": external_service, password + ".. pii_retirement:": retained + +# Via django-celery +djcelery.CrontabSchedule: + ".. no_pii:": "No PII" +djcelery.IntervalSchedule: + ".. no_pii:": "No PII" +djcelery.PeriodicTask: + ".. no_pii:": "No PII" +djcelery.PeriodicTasks: + ".. no_pii:": "No PII" +djcelery.TaskMeta: + ".. no_pii:": "No PII" +djcelery.TaskSetMeta: + ".. no_pii:": "No PII" +djcelery.TaskState: + ".. no_pii:": "No PII" +djcelery.WorkerState: + ".. no_pii:": "No PII" + +# Via edx-oauth2-provider https://github.com/edx/edx-oauth2-provider +edx_oauth2_provider.TrustedClient: + ".. no_pii:": "No PII" + +# Via Proctoring +edx_proctoring.ProctoredExam: + ".. no_pii:": "No PII" +edx_proctoring.ProctoredExamReviewPolicy: + ".. no_pii:": "No PII" +edx_proctoring.ProctoredExamReviewPolicyHistory: + ".. no_pii:": "No PII" +edx_proctoring.ProctoredExamSoftwareSecureComment: + ".. no_pii:": "No PII" +edx_proctoring.ProctoredExamSoftwareSecureReview: + ".. pii:": "Proctored exam review feedback from Software Secure, contains video_url. Retained for record keeping." + ".. pii_types:": video + ".. pii_retirement:": retained +edx_proctoring.ProctoredExamSoftwareSecureReviewHistory: + ".. pii:": "Proctored exam review feedback from Software Secure, contains video_url. Retained for record keeping." + ".. pii_types:": video + ".. pii_retirement:": retained +edx_proctoring.ProctoredExamStudentAllowance: + ".. no_pii:": "No PII" +edx_proctoring.ProctoredExamStudentAllowanceHistory: + ".. no_pii:": "No PII" +edx_proctoring.ProctoredExamStudentAttempt: + ".. pii:": "Tracks attempts by a user to take a proctored exam. Contains student_name. Retained for record keeping." + ".. pii_types:": name + ".. pii_retirement:": retained +edx_proctoring.ProctoredExamStudentAttemptHistory: + ".. pii:": "Tracks attempts by a user to take a proctored exam. Contains student_name. Retained for record keeping." + ".. pii_types:": name + ".. pii_retirement:": retained + +# Via VAL +edxval.CourseVideo: + ".. no_pii:": "No PII" +edxval.EncodedVideo: + ".. no_pii:": "No PII" +edxval.Profile: + ".. no_pii:": "No PII" +edxval.ThirdPartyTranscriptCredentialsState: + ".. no_pii:": "No PII" +edxval.TranscriptPreference: + ".. no_pii:": "No PII" +edxval.Video: + ".. no_pii:": "No PII" +edxval.VideoImage: + ".. no_pii:": "No PII" +edxval.VideoTranscript: + ".. no_pii:": "No PII" + +# Via Milestones +milestones.CourseContentMilestone: + ".. no_pii:": "No PII" +milestones.CourseMilestone: + ".. no_pii:": "No PII" +milestones.Milestone: + ".. no_pii:": "No PII" +milestones.MilestoneRelationshipType: + ".. no_pii:": "No PII" +milestones.UserMilestone: + ".. no_pii:": "No PII" + +# Via Django OAuth2 Provider https://github.com/edx/django-oauth2-provider +oauth2.Client: + ".. no_pii:": "No PII" +oauth2.AccessToken: + ".. pii:": "Contains 3rd party authentication secrets. Retired in DeactivateLogoutView." + ".. pii_types:": password, other + ".. pii_retirement:": local_api +oauth2.Grant: + ".. pii:": "Contains 3rd party authentication secrets. Retired in DeactivateLogoutView." + ".. pii_types:": password, other + ".. pii_retirement:": local_api +oauth2.RefreshToken: + ".. pii:": "Contains 3rd party authentication secrets. Retired in DeactivateLogoutView." + ".. pii_types:": password, other + ".. pii_retirement:": local_api + +# Via Django OAuth Toolkit https://github.com/evonove/django-oauth-toolkit +oauth2_provider.AccessToken: + ".. pii:": "Contains 3rd party authentication secrets. Retired in DeactivateLogoutView." + ".. pii_types:": password, other + ".. pii_retirement:": local_api +oauth2_provider.Application: + ".. pii:": "Contains 3rd party authentication secrets. Retired in DeactivateLogoutView." + ".. pii_types:": password, other + ".. pii_retirement:": local_api +oauth2_provider.Grant: + ".. pii:": "Contains 3rd party authentication secrets. Retired in DeactivateLogoutView." + ".. pii_types:": password, other + ".. pii_retirement:": local_api +oauth2_provider.RefreshToken: + ".. pii:": "Contains 3rd party authentication secrets. Retired in DeactivateLogoutView." + ".. pii_types:": password, other + ".. pii_retirement:": local_api + +# Via Django OAuth Plus https://bitbucket.org/david/django-oauth-plus +oauth_provider.Consumer: + ".. no_pii:": "No PII, unused and empty in edx.org" +oauth_provider.Nonce: + ".. no_pii:": "No PII, unused and empty in edx.org" +oauth_provider.Scope: + ".. no_pii:": "No PII, unused and empty in edx.org" +oauth_provider.Token: + ".. pii:": "User OAuth associations. Not used and empty on edx.org, therefore not retired." + ".. pii_types:": external_service, password + ".. pii_retirement:": retained + +# Via edx-organizations +organizations.Organization: + ".. no_pii:": "No PII" +organizations.OrganizationCourse: + ".. no_pii:": "No PII" + +# Via Problem Builder XBlock +problem_builder.Answer: + ".. no_pii:": "No PII" +problem_builder.Share: + ".. no_pii:": "No PII" + +# Via Social Django https://github.com/python-social-auth/social-app-django +social_django.Association: + ".. no_pii:": "No PII" +social_django.Code: + ".. pii:": "Transient - email address stored with email authentication code, removed automatically so not retired" + ".. pii_types:": email_address + ".. pii_retirement:": local_api +social_django.Nonce: + ".. no_pii:": "No PII" +social_django.Partial: + ".. no_pii:": "No PII" +social_django.UserSocialAuth: + ".. pii:": "3rd party authentication data, retired in DeactivateLogoutView" + ".. pii_types:": external_service + ".. pii_retirement:": local_api + +# Via Splash https://github.com/edx/django-splash +splash.SplashConfig: + ".. no_pii:": "No PII" + +# Via edx-submissions +submissions.Score: + ".. no_pii:": "No PII" +submissions.ScoreAnnotation: + ".. no_pii:": "No PII" +submissions.ScoreSummary: + ".. no_pii:": "No PII" +submissions.StudentItem: + ".. no_pii:": "No PII" +submissions.Submission: + ".. no_pii:": "No PII" + +# Via sorl-thumbnail https://github.com/jazzband/sorl-thumbnail +thumbnail.KVStore: + ".. no_pii:": "No PII" + +# Via django-user-tasks +user_tasks.UserTaskArtifact: + ".. no_pii:": "No PII" +user_tasks.UserTaskStatus: + ".. no_pii:": "No PII" + +# Via waffle +waffle.Flag: + ".. no_pii:": "No PII" +waffle.Sample: + ".. no_pii:": "No PII" +waffle.Switch: + ".. no_pii:": "No PII" + +# Via django-wiki https://github.com/edx/django-wiki +wiki.Article: + ".. no_pii:": "No PII" +wiki.ArticleForObject: + ".. no_pii:": "No PII" +wiki.ArticlePlugin: + ".. no_pii:": "No PII" +wiki.ArticleRevision: + ".. no_pii:": "No PII" +wiki.ReusablePlugin: + ".. no_pii:": "No PII" +wiki.RevisionPlugin: + ".. no_pii:": "No PII" +wiki.RevisionPluginRevision: + ".. no_pii:": "No PII" +wiki.SimplePlugin: + ".. no_pii:": "No PII" +wiki.URLPath: + ".. no_pii:": "No PII" diff --git a/.gitignore b/.gitignore index b9a688073b..440756a7e4 100644 --- a/.gitignore +++ b/.gitignore @@ -140,3 +140,6 @@ dist # Visual Studio Code .vscode + +# Locally generated PII reports +pii_report diff --git a/.pii_annotations.yml b/.pii_annotations.yml new file mode 100644 index 0000000000..5f0f822b58 --- /dev/null +++ b/.pii_annotations.yml @@ -0,0 +1,37 @@ +source_path: ./ +report_path: pii_report +safelist_path: .annotation_safe_list.yml +coverage_target: 100.0 +# See OEP-30 for more information on these values and what they mean: +# https://open-edx-proposals.readthedocs.io/en/latest/oep-0030-arch-pii-markup-and-auditing.html#docstring-annotations +annotations: + ".. no_pii:": + "pii_group": + - ".. pii:": + - ".. pii_types:": + choices: + - id + - name + - username + - password + - location + - phone_number + - email_address + - birth_date + - ip + - external_service + - biography + - gender + - sex + - image + - video + - other + - ".. pii_retirement:": + choices: + - retained + - local_api + - consumer_api + - third_party +extensions: + python: + - py diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index 7fa8efe4f9..fa69d45ec7 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -7,7 +7,11 @@ from django.db.models.fields import TextField class VideoUploadConfig(ConfigurationModel): - """Configuration for the video upload feature.""" + """ + Configuration for the video upload feature. + + .. no_pii: + """ profile_whitelist = TextField( blank=True, help_text="A comma-separated list of names of profiles to include in video encoding downloads." @@ -20,4 +24,8 @@ class VideoUploadConfig(ConfigurationModel): class PushNotificationConfig(ConfigurationModel): - """Configuration for mobile push notifications.""" + """ + Configuration for mobile push notifications. + + .. no_pii: + """ diff --git a/cms/djangoapps/course_creators/models.py b/cms/djangoapps/course_creators/models.py index 92f7f60bca..3ab29abe8f 100644 --- a/cms/djangoapps/course_creators/models.py +++ b/cms/djangoapps/course_creators/models.py @@ -21,6 +21,8 @@ send_user_notification = Signal(providing_args=["user", "state"]) class CourseCreator(models.Model): """ Creates the database table model. + + .. no_pii: """ UNREQUESTED = 'unrequested' PENDING = 'pending' diff --git a/cms/djangoapps/xblock_config/models.py b/cms/djangoapps/xblock_config/models.py index a2b46fb69d..5276db3b5b 100644 --- a/cms/djangoapps/xblock_config/models.py +++ b/cms/djangoapps/xblock_config/models.py @@ -15,6 +15,8 @@ from openedx.core.lib.cache_utils import request_cached class StudioConfig(ConfigurationModel): """ Configuration for XBlockAsides. + + .. no_pii: """ disabled_blocks = TextField( default="about course_info static_tab", @@ -36,6 +38,8 @@ class CourseEditLTIFieldsEnabledFlag(ConfigurationModel): """ Enables the editing of "request username" and "request email" fields of LTI consumer for a specific course. + + .. no_pii: """ KEY_FIELDS = ('course_id',) diff --git a/cms/lib/xblock/tagging/models.py b/cms/lib/xblock/tagging/models.py index 7406217903..3992c0e090 100644 --- a/cms/lib/xblock/tagging/models.py +++ b/cms/lib/xblock/tagging/models.py @@ -7,6 +7,8 @@ from django.db import models class TagCategories(models.Model): """ This model represents tag categories. + + .. no_pii: """ name = models.CharField(max_length=255, unique=True) title = models.CharField(max_length=255) @@ -30,6 +32,8 @@ class TagCategories(models.Model): class TagAvailableValues(models.Model): """ This model represents available values for tags. + + .. no_pii: """ category = models.ForeignKey(TagCategories, db_index=True, on_delete=models.CASCADE) value = models.CharField(max_length=255) diff --git a/common/djangoapps/course_action_state/models.py b/common/djangoapps/course_action_state/models.py index 0efadc8ba2..02166cd730 100644 --- a/common/djangoapps/course_action_state/models.py +++ b/common/djangoapps/course_action_state/models.py @@ -101,6 +101,8 @@ class CourseActionUIState(CourseActionState): class CourseRerunState(CourseActionUIState): """ A concrete django model for maintaining state specifically for the Action Course Reruns. + + .. no_pii: """ class Meta(object): """ diff --git a/common/djangoapps/course_modes/models.py b/common/djangoapps/course_modes/models.py index 9f80c4e733..e879ee9fe0 100644 --- a/common/djangoapps/course_modes/models.py +++ b/common/djangoapps/course_modes/models.py @@ -38,6 +38,7 @@ class CourseMode(models.Model): """ We would like to offer a course in a variety of modes. + .. no_pii: """ class Meta(object): app_label = "course_modes" @@ -819,6 +820,8 @@ class CourseModesArchive(models.Model): separate model, because there is a uniqueness contraint on (course_mode, course_id) field pair in CourseModes. Having a separate table allows us to have an audit trail of any changes such as course price changes + + .. no_pii: """ class Meta(object): app_label = "course_modes" @@ -852,6 +855,8 @@ class CourseModesArchive(models.Model): class CourseModeExpirationConfig(ConfigurationModel): """ Configuration for time period from end of course to auto-expire a course mode. + + .. no_pii: """ class Meta(object): app_label = "course_modes" diff --git a/common/djangoapps/django_comment_common/models.py b/common/djangoapps/django_comment_common/models.py index c9e3aead39..7d9d56aef4 100644 --- a/common/djangoapps/django_comment_common/models.py +++ b/common/djangoapps/django_comment_common/models.py @@ -65,6 +65,11 @@ def assign_role(course_id, user, rolename): class Role(models.Model): + """ + Maps users to django_comment_client roles for a given course + + .. no_pii: + """ objects = NoneToEmptyManager() @@ -100,7 +105,9 @@ class Role(models.Model): self.permissions.add(Permission.objects.get_or_create(name=permission)[0]) def has_permission(self, permission): - """Returns True if this role has the given permission, False otherwise.""" + """ + Returns True if this role has the given permission, False otherwise. + """ course = modulestore().get_course(self.course_id) if course is None: raise ItemNotFoundError(self.course_id) @@ -118,6 +125,11 @@ class Role(models.Model): class Permission(models.Model): + """ + Permissions for django_comment_client + + .. no_pii: + """ name = models.CharField(max_length=30, null=False, blank=False, primary_key=True) roles = models.ManyToManyField(Role, related_name="permissions") @@ -130,7 +142,8 @@ class Permission(models.Model): def permission_blacked_out(course, role_names, permission_name): - """Returns true if a user in course with the given roles would have permission_name blacked out. + """ + Returns true if a user in course with the given roles would have permission_name blacked out. This will return true if it is a permission that the user might have normally had for the course, but does not have right this moment because we are in a discussion blackout period (as defined by the settings on the course module). @@ -145,7 +158,9 @@ def permission_blacked_out(course, role_names, permission_name): def all_permissions_for_user_in_course(user, course_id): # pylint: disable=invalid-name - """Returns all the permissions the user has in the given course.""" + """ + Returns all the permissions the user has in the given course. + """ if not user.is_authenticated: return {} @@ -176,7 +191,11 @@ def all_permissions_for_user_in_course(user, course_id): # pylint: disable=inva class ForumsConfig(ConfigurationModel): - """Config for the connection to the cs_comments_service forums backend.""" + """ + Config for the connection to the cs_comments_service forums backend. + + .. no_pii: + """ connection_timeout = models.FloatField( default=5.0, @@ -189,11 +208,18 @@ class ForumsConfig(ConfigurationModel): return getattr(settings, "COMMENTS_SERVICE_KEY", None) def __unicode__(self): - """Simple representation so the admin screen looks less ugly.""" + """ + Simple representation so the admin screen looks less ugly. + """ return u"ForumsConfig: timeout={}".format(self.connection_timeout) class CourseDiscussionSettings(models.Model): + """ + Settings for course discussions + + .. no_pii: + """ course_id = CourseKeyField( unique=True, max_length=255, @@ -216,17 +242,25 @@ class CourseDiscussionSettings(models.Model): @property def divided_discussions(self): - """Jsonify the divided_discussions""" + """ + Jsonify the divided_discussions + """ return json.loads(self._divided_discussions) @divided_discussions.setter def divided_discussions(self, value): - """Un-Jsonify the divided_discussions""" + """ + Un-Jsonify the divided_discussions + """ self._divided_discussions = json.dumps(value) class DiscussionsIdMapping(models.Model): - """This model is a performance optimization, updated on course publish.""" + """ + This model is a performance optimization, updated on course publish. + + .. no_pii: + """ course_id = CourseKeyField(db_index=True, primary_key=True, max_length=255) mapping = JSONField( help_text="Key/value store mapping discussion IDs to discussion XBlock usage keys.", diff --git a/common/djangoapps/entitlements/models.py b/common/djangoapps/entitlements/models.py index ba39546b09..e2ce5b921f 100644 --- a/common/djangoapps/entitlements/models.py +++ b/common/djangoapps/entitlements/models.py @@ -26,6 +26,8 @@ log = logging.getLogger("common.entitlements.models") class CourseEntitlementPolicy(models.Model): """ Represents the Entitlement's policy for expiration, refunds, and regaining a used certificate + + .. no_pii: """ DEFAULT_EXPIRATION_PERIOD_DAYS = 730 @@ -146,6 +148,8 @@ class CourseEntitlementPolicy(models.Model): class CourseEntitlement(TimeStampedModel): """ Represents a Student's Entitlement to a Course Run for a given Course. + + .. no_pii: """ user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) uuid = models.UUIDField(default=uuid_tools.uuid4, editable=False, unique=True) @@ -442,6 +446,8 @@ class CourseEntitlement(TimeStampedModel): class CourseEntitlementSupportDetail(TimeStampedModel): """ Table recording support interactions with an entitlement + + .. no_pii: """ # Reasons deprecated LEAVE_SESSION = 'LEAVE' diff --git a/common/djangoapps/microsite_configuration/models.py b/common/djangoapps/microsite_configuration/models.py index bc2f7bbecb..11ac5d7d9d 100644 --- a/common/djangoapps/microsite_configuration/models.py +++ b/common/djangoapps/microsite_configuration/models.py @@ -28,6 +28,8 @@ class Microsite(models.Model): - The site field is django site. - The values field must be validated on save to prevent the platform from crashing badly in the case the string is not able to be loaded as json. + + .. no_pii: """ site = models.OneToOneField(Site, related_name='microsite', on_delete=models.CASCADE) key = models.CharField(max_length=63, db_index=True, unique=True) @@ -60,6 +62,8 @@ class MicrositeHistory(TimeStampedModel): """ This is an archive table for Microsites model, so that we can maintain a history of changes. Note that the key field is no longer unique + + .. no_pii: """ site = models.ForeignKey(Site, related_name='microsite_history', on_delete=models.CASCADE) key = models.CharField(max_length=63, db_index=True) @@ -109,6 +113,8 @@ def on_microsite_updated(sender, instance, **kwargs): # pylint: disable=unused- class MicrositeOrganizationMapping(models.Model): """ Mapping of Organization to which Microsite it belongs + + .. no_pii: """ organization = models.CharField(max_length=63, db_index=True, unique=True) @@ -145,6 +151,8 @@ class MicrositeOrganizationMapping(models.Model): class MicrositeTemplate(models.Model): """ A HTML template that a microsite can use + + .. no_pii: """ microsite = models.ForeignKey(Microsite, db_index=True, on_delete=models.CASCADE) diff --git a/common/djangoapps/static_replace/models.py b/common/djangoapps/static_replace/models.py index e598178a57..2ff78aec77 100644 --- a/common/djangoapps/static_replace/models.py +++ b/common/djangoapps/static_replace/models.py @@ -7,7 +7,11 @@ from django.db.models.fields import TextField class AssetBaseUrlConfig(ConfigurationModel): - """Configuration for the base URL used for static assets.""" + """ + Configuration for the base URL used for static assets. + + .. no_pii: + """ class Meta(object): app_label = 'static_replace' @@ -30,7 +34,11 @@ class AssetBaseUrlConfig(ConfigurationModel): class AssetExcludedExtensionsConfig(ConfigurationModel): - """Configuration for the the excluded file extensions when canonicalizing static asset paths.""" + """ + Configuration for the the excluded file extensions when canonicalizing static asset paths. + + .. no_pii: + """ class Meta(object): app_label = 'static_replace' diff --git a/common/djangoapps/status/models.py b/common/djangoapps/status/models.py index 42bb3117d5..9579439968 100644 --- a/common/djangoapps/status/models.py +++ b/common/djangoapps/status/models.py @@ -9,10 +9,14 @@ from django.core.cache import cache from django.db import models from opaque_keys.edx.django.models import CourseKeyField +from openedx.core.djangolib.markup import HTML + class GlobalStatusMessage(ConfigurationModel): """ Model that represents the current status message. + + .. no_pii: """ message = models.TextField( blank=True, @@ -37,7 +41,7 @@ class GlobalStatusMessage(ConfigurationModel): course_home_message = self.coursemessage_set.get(course_key=course_key) # Don't override the message if course_home_message is blank. if course_home_message: - msg = u"{}
{}".format(msg, course_home_message.message) + msg = HTML(u"{}
{}").format(HTML(msg), HTML(course_home_message.message)) except CourseMessage.DoesNotExist: # We don't have a course-specific message, so pass. pass @@ -54,6 +58,8 @@ class CourseMessage(models.Model): This is not a ConfigurationModel because using it's not designed to support multiple configurations at once, which would be problematic if separate courses need separate error messages. + + .. no_pii: """ global_message = models.ForeignKey(GlobalStatusMessage, on_delete=models.CASCADE) course_key = CourseKeyField(max_length=255, blank=True, db_index=True) diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index 94fea1d4a8..d1826b683f 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -126,6 +126,8 @@ class AnonymousUserId(models.Model): We generate anonymous_user_id using md5 algorithm, and use result in hex form, so its length is equal to 32 bytes. + + .. no_pii: We store anonymous_user_ids here, but do not consider them PII under OEP-30. """ objects = NoneToEmptyManager() @@ -376,6 +378,8 @@ class UserStanding(models.Model): Currently, we're only disabling accounts; in the future we can imagine taking away more specific privileges, like forums access, or adding more specific karma levels or probationary stages. + + .. no_pii: """ ACCOUNT_DISABLED = "disabled" ACCOUNT_ENABLED = "enabled" @@ -409,6 +413,10 @@ class UserProfile(models.Model): Some of the fields are legacy ones that were captured during the initial MITx fall prototype. + + .. pii: Contains many PII fields. Retired in AccountRetirementView. + .. pii_types: name, location, birth_date, gender, biography + .. pii_retirement: local_api """ # cache key format e.g user..profile.country = 'SG' PROFILE_COUNTRY_CACHE_KEY = u"user.{user_id}.profile.country" @@ -701,6 +709,8 @@ class UserSignupSource(models.Model): """ This table contains information about users registering via Micro-Sites + + .. no_pii: """ user = models.ForeignKey(User, db_index=True, on_delete=models.CASCADE) site = models.CharField(max_length=255, db_index=True) @@ -722,16 +732,23 @@ def unique_id_for_user(user, save=True): # TODO: Should be renamed to generic UserGroup, and possibly # Given an optional field for type of group class UserTestGroup(models.Model): + """ + .. no_pii: + """ users = models.ManyToManyField(User, db_index=True) name = models.CharField(blank=False, max_length=32, db_index=True) description = models.TextField(blank=True) class Registration(models.Model): - ''' Allows us to wait for e-mail before user is registered. A - registration profile is created when the user creates an - account, but that account is inactive. Once the user clicks - on the activation key, it becomes active. ''' + """ + Allows us to wait for e-mail before user is registered. A + registration profile is created when the user creates an + account, but that account is inactive. Once the user clicks + on the activation key, it becomes active. + + .. no_pii: + """ class Meta(object): db_table = "auth_registration" @@ -752,10 +769,15 @@ class Registration(models.Model): log.info(u'User %s (%s) account is successfully activated.', self.user.username, self.user.email) def _track_activation(self): - """ Update the isActive flag in mailchimp for activated users.""" + """ + Update the isActive flag in mailchimp for activated users. + """ has_segment_key = getattr(settings, 'LMS_SEGMENT_KEY', None) has_mailchimp_id = hasattr(settings, 'MAILCHIMP_NEW_USER_LIST_ID') if has_segment_key and has_mailchimp_id: + # .. pii: Username and email are sent to Segment here. Retired directly through Segment API call in Tubular. + # .. pii_types: email_address, username + # .. pii_retirement: third_party segment.identify( self.user.id, # pylint: disable=no-member { @@ -772,6 +794,13 @@ class Registration(models.Model): class PendingNameChange(DeletableByUserValue, models.Model): + """ + This model keeps track of pending requested changes to a user's email address. + + .. pii: Contains new_name, retired in LMSAccountRetirementView + .. pii_types: name + .. pii_retirement: local_api + """ user = models.OneToOneField(User, unique=True, db_index=True, on_delete=models.CASCADE) new_name = models.CharField(blank=True, max_length=255) rationale = models.CharField(blank=True, max_length=1024) @@ -780,6 +809,10 @@ class PendingNameChange(DeletableByUserValue, models.Model): class PendingEmailChange(DeletableByUserValue, models.Model): """ This model keeps track of pending requested changes to a user's email address. + + .. pii: Contains new_email, retired in AccountRetirementView + .. pii_types: email_address + .. pii_retirement: local_api """ user = models.OneToOneField(User, unique=True, db_index=True, on_delete=models.CASCADE) new_email = models.CharField(blank=True, max_length=255, db_index=True) @@ -806,6 +839,10 @@ class PendingEmailChange(DeletableByUserValue, models.Model): class PendingSecondaryEmailChange(DeletableByUserValue, models.Model): """ This model keeps track of pending requested changes to a user's secondary email address. + + .. pii: Contains new_secondary_email, not currently retired + .. pii_types: email_address + .. pii_retirement: retained """ user = models.OneToOneField(User, unique=True, db_index=True, on_delete=models.CASCADE) new_secondary_email = models.CharField(blank=True, max_length=255, db_index=True) @@ -819,7 +856,9 @@ EVENT_NAME_ENROLLMENT_MODE_CHANGED = 'edx.course.enrollment.mode_changed' class LoginFailures(models.Model): """ - This model will keep track of failed login attempts + This model will keep track of failed login attempts. + + .. no_pii: """ user = models.ForeignKey(User, on_delete=models.CASCADE) failure_count = models.IntegerField(default=0) @@ -1023,6 +1062,8 @@ class CourseEnrollment(models.Model): more should be brought in (such as checking against CourseEnrollmentAllowed, checking course dates, user permissions, etc.) This logic is currently scattered across our views. + + .. no_pii: """ MODEL_TAGS = ['course', 'is_active', 'mode'] @@ -1938,7 +1979,9 @@ class CourseEnrollment(models.Model): @receiver(models.signals.post_save, sender=CourseEnrollment) @receiver(models.signals.post_delete, sender=CourseEnrollment) def invalidate_enrollment_mode_cache(sender, instance, **kwargs): # pylint: disable=unused-argument, invalid-name - """Invalidate the cache of CourseEnrollment model. """ + """ + Invalidate the cache of CourseEnrollment model. + """ cache_key = CourseEnrollment.cache_key_name( instance.user.id, @@ -1950,6 +1993,10 @@ def invalidate_enrollment_mode_cache(sender, instance, **kwargs): # pylint: dis class ManualEnrollmentAudit(models.Model): """ Table for tracking which enrollments were performed through manual enrollment. + + .. pii: Contains enrolled_email, retired in LMSAccountRetirementView + .. pii_types: email_address + .. pii_retirement: local_api """ enrollment = models.ForeignKey(CourseEnrollment, null=True, on_delete=models.CASCADE) enrolled_by = models.ForeignKey(User, null=True, on_delete=models.CASCADE) @@ -2016,6 +2063,8 @@ class CourseEnrollmentAllowed(DeletableByUserValue, models.Model): even if the enrollment time window is past. Once an enrollment from this list effectively happens, the object is marked with the student who enrolled, to prevent students from changing e-mails and enrolling many accounts through the same e-mail. + + .. no_pii: """ email = models.CharField(max_length=255, db_index=True) course_id = CourseKeyField(max_length=255, db_index=True) @@ -2072,6 +2121,8 @@ class CourseAccessRole(models.Model): Maps users to org, courses, and roles. Used by student.roles.CourseRole and OrgRole. To establish a user as having a specific role over all courses in the org, create an entry without a course_id. + + .. no_pii: """ objects = NoneToEmptyManager() @@ -2285,10 +2336,12 @@ def enforce_single_login(sender, request, user, signal, **kwargs): # pylint: class DashboardConfiguration(ConfigurationModel): - """Dashboard Configuration settings. + """ + Dashboard Configuration settings. Includes configuration options for the dashboard, which impact behavior and rendering for the application. + .. no_pii: """ recent_enrollment_time_delta = models.PositiveIntegerField( default=0, @@ -2311,6 +2364,8 @@ class LinkedInAddToProfileConfiguration(ConfigurationModel): users are sent to the LinkedIn site with a pre-filled form allowing them to add the certificate to their LinkedIn profile. + + .. no_pii: """ MODE_TO_CERT_NAME = { @@ -2437,6 +2492,8 @@ class LinkedInAddToProfileConfiguration(ConfigurationModel): class EntranceExamConfiguration(models.Model): """ Represents a Student's entrance exam specific data for a single Course + + .. no_pii: """ user = models.ForeignKey(User, db_index=True, on_delete=models.CASCADE) @@ -2504,6 +2561,8 @@ class LanguageProficiency(models.Model): to go through the accounts API (AccountsView) defined in /edx-platform/openedx/core/djangoapps/user_api/accounts/views.py or its associated api method (update_account_settings) so that the events are emitted. + + .. no_pii: Language is not PII value according to OEP-30. """ class Meta(object): unique_together = (('code', 'user_profile'),) @@ -2527,6 +2586,10 @@ class SocialLink(models.Model): # pylint: disable=model-missing-unicode component of the stored URL and an example of a valid URL. The stored social_link value must adhere to the form 'https://www.[url_stub][username]'. + + .. pii: Stores linkage from User to a learner's social media profiles. Retired in AccountRetirementView. + .. pii_types: external_service + .. pii_retirement: local_api """ user_profile = models.ForeignKey(UserProfile, db_index=True, related_name='social_links', on_delete=models.CASCADE) platform = models.CharField(max_length=30) @@ -2536,6 +2599,8 @@ class SocialLink(models.Model): # pylint: disable=model-missing-unicode class CourseEnrollmentAttribute(models.Model): """ Provide additional information about the user's enrollment. + + .. no_pii: This stores key/value pairs, of which there is no full list, but the ones currently in use are not PII """ enrollment = models.ForeignKey(CourseEnrollment, related_name="attributes", on_delete=models.CASCADE) namespace = models.CharField( @@ -2561,12 +2626,13 @@ class CourseEnrollmentAttribute(models.Model): @classmethod def add_enrollment_attr(cls, enrollment, data_list): - """Delete all the enrollment attributes for the given enrollment and + """ + Delete all the enrollment attributes for the given enrollment and add new attributes. Args: - enrollment(CourseEnrollment): 'CourseEnrollment' for which attribute is to be added - data(list): list of dictionaries containing data to save + enrollment (CourseEnrollment): 'CourseEnrollment' for which attribute is to be added + data_list: list of dictionaries containing data to save """ cls.objects.filter(enrollment=enrollment).delete() attributes = [ @@ -2607,6 +2673,8 @@ class CourseEnrollmentAttribute(models.Model): class EnrollmentRefundConfiguration(ConfigurationModel): """ Configuration for course enrollment refunds. + + .. no_pii: """ # TODO: Django 1.8 introduces a DurationField @@ -2638,6 +2706,8 @@ class EnrollmentRefundConfiguration(ConfigurationModel): class RegistrationCookieConfiguration(ConfigurationModel): """ Configuration for registration cookies. + + .. no_pii: """ utm_cookie_name = models.CharField( max_length=255, @@ -2660,6 +2730,8 @@ class RegistrationCookieConfiguration(ConfigurationModel): class UserAttribute(TimeStampedModel): """ Record additional metadata about a user, stored as key/value pairs of text. + + .. no_pii: """ class Meta(object): @@ -2700,10 +2772,16 @@ class UserAttribute(TimeStampedModel): class LogoutViewConfiguration(ConfigurationModel): - """ DEPRECATED: Configuration for the logout view. """ + """ + DEPRECATED: Configuration for the logout view. + + .. no_pii: + """ def __unicode__(self): - """Unicode representation of the instance. """ + """ + Unicode representation of the instance. + """ return u'Logout view configuration: {enabled}'.format(enabled=self.enabled) diff --git a/common/djangoapps/third_party_auth/models.py b/common/djangoapps/third_party_auth/models.py index 9adbac3c34..00423b8159 100644 --- a/common/djangoapps/third_party_auth/models.py +++ b/common/djangoapps/third_party_auth/models.py @@ -86,6 +86,8 @@ class AuthNotConfigured(SocialAuthBaseException): class ProviderConfig(ConfigurationModel): """ Abstract Base Class for configuring a third_party_auth provider + + .. no_pii: """ KEY_FIELDS = ('slug',) @@ -327,6 +329,8 @@ class OAuth2ProviderConfig(ProviderConfig): """ Configuration Entry for an OAuth2 based provider. Also works for OAuth1 providers. + + .. no_pii: """ prefix = 'oa2' backend_name = models.CharField( @@ -381,6 +385,8 @@ class SAMLConfiguration(ConfigurationModel): General configuration required for this edX instance to act as a SAML Service Provider and allow users to authenticate via third party SAML Identity Providers (IdPs) + + .. no_pii: """ KEY_FIELDS = ('site_id', 'slug') site = models.ForeignKey( @@ -523,6 +529,8 @@ def active_saml_configurations_filter(): class SAMLProviderConfig(ProviderConfig): """ Configuration Entry for a SAML/Shibboleth provider. + + .. no_pii: """ prefix = 'saml' backend_name = models.CharField( @@ -704,6 +712,8 @@ class SAMLProviderData(models.Model): Data about a SAML IdP that is fetched automatically by 'manage.py saml pull' This data is only required during the actual authentication process. + + .. no_pii: """ cache_timeout = 600 fetched_at = models.DateTimeField(db_index=True, null=False) @@ -754,6 +764,8 @@ class LTIProviderConfig(ProviderConfig): Configuration required for this edX instance to act as a LTI Tool Provider and allow users to authenticate and be enrolled in a course via third party LTI Tool Consumers. + + .. no_pii: """ prefix = 'lti' backend_name = 'lti' @@ -844,6 +856,8 @@ class ProviderApiPermissions(models.Model): This model links OAuth2 client with provider Id. It gives permission for a OAuth2 client to access the information under certain IdPs. + + .. no_pii: """ client = models.ForeignKey(Client, on_delete=models.CASCADE) provider_id = models.CharField( diff --git a/common/djangoapps/track/backends/django.py b/common/djangoapps/track/backends/django.py index d6eb58f63b..d04103a1a4 100644 --- a/common/djangoapps/track/backends/django.py +++ b/common/djangoapps/track/backends/django.py @@ -32,7 +32,13 @@ LOGFIELDS = [ class TrackingLog(models.Model): - """Defines the fields that are stored in the tracking log database.""" + """ + Defines the fields that are stored in the tracking log database. + + .. pii: Stores a great deal of PII as it is an event tracker of browsing history, unused and empty on edx.org + .. pii_types: username, ip, other + .. pii_retirement: retained + """ dtcreated = models.DateTimeField('creation date', auto_now_add=True) username = models.CharField(max_length=32, blank=True) diff --git a/common/djangoapps/track/segment.py b/common/djangoapps/track/segment.py index e1470a83f0..de73bf24b0 100644 --- a/common/djangoapps/track/segment.py +++ b/common/djangoapps/track/segment.py @@ -16,7 +16,9 @@ from eventtracking import tracker def track(user_id, event_name, properties=None, context=None): - """Wrapper for emitting Segment track event, including augmenting context information from middleware.""" + """ + Wrapper for emitting Segment track event, including augmenting context information from middleware. + """ if event_name is not None and hasattr(settings, 'LMS_SEGMENT_KEY') and settings.LMS_SEGMENT_KEY: properties = properties or {} @@ -60,7 +62,9 @@ def track(user_id, event_name, properties=None, context=None): def identify(user_id, properties, context=None): - """Wrapper for emitting Segment identify event.""" + """ + Wrapper for emitting Segment identify event. + """ if hasattr(settings, 'LMS_SEGMENT_KEY') and settings.LMS_SEGMENT_KEY: segment_context = dict(context) if context else {} analytics.identify(user_id, properties, segment_context) diff --git a/common/djangoapps/util/models.py b/common/djangoapps/util/models.py index 9768d3a619..066b708290 100644 --- a/common/djangoapps/util/models.py +++ b/common/djangoapps/util/models.py @@ -13,13 +13,16 @@ logger = logging.getLogger(__name__) # pylint: disable=invalid-name class RateLimitConfiguration(ConfigurationModel): - """Configuration flag to enable/disable rate limiting. + """ + Configuration flag to enable/disable rate limiting. Applies to Django Rest Framework views. This is useful for disabling rate limiting for performance tests. When enabled, it will disable rate limiting on any view decorated with the `can_disable_rate_limit` class decorator. + + .. no_pii: """ class Meta(ConfigurationModel.Meta): app_label = "util" @@ -43,11 +46,15 @@ def decompress_string(value): class CompressedTextField(CreatorMixin, models.TextField): - """ TextField that transparently compresses data when saving to the database, and decompresses the data - when retrieving it from the database. """ + """ + TextField that transparently compresses data when saving to the database, and decompresses the data + when retrieving it from the database. + """ def get_prep_value(self, value): - """ Compress the text data. """ + """ + Compress the text data. + """ if value is not None: if isinstance(value, unicode): value = value.encode('utf8') @@ -56,7 +63,9 @@ class CompressedTextField(CreatorMixin, models.TextField): return value def to_python(self, value): - """ Decompresses the value from the database. """ + """ + Decompresses the value from the database. + """ if isinstance(value, unicode): value = decompress_string(value) diff --git a/common/djangoapps/xblock_django/models.py b/common/djangoapps/xblock_django/models.py index 78ac02ca0f..ffd0082238 100644 --- a/common/djangoapps/xblock_django/models.py +++ b/common/djangoapps/xblock_django/models.py @@ -10,6 +10,8 @@ from django.utils.translation import ugettext_lazy as _ class XBlockConfiguration(ConfigurationModel): """ XBlock configuration used by both LMS and Studio, and not specific to a particular template. + + .. no_pii: """ KEY_FIELDS = ('name',) # xblock name is unique @@ -33,6 +35,8 @@ class XBlockConfiguration(ConfigurationModel): class XBlockStudioConfigurationFlag(ConfigurationModel): """ Enables site-wide Studio configuration for XBlocks. + + .. no_pii: """ class Meta(object): @@ -47,6 +51,8 @@ class XBlockStudioConfigurationFlag(ConfigurationModel): class XBlockStudioConfiguration(ConfigurationModel): """ Studio editing configuration for a specific XBlock/template combination. + + .. no_pii: """ KEY_FIELDS = ('name', 'template') # xblock name/template combination is unique diff --git a/lms/djangoapps/badges/models.py b/lms/djangoapps/badges/models.py index 619065374b..75cd86398c 100644 --- a/lms/djangoapps/badges/models.py +++ b/lms/djangoapps/badges/models.py @@ -17,6 +17,7 @@ from opaque_keys.edx.django.models import CourseKeyField from opaque_keys.edx.keys import CourseKey from badges.utils import deserialize_count_specs +from openedx.core.djangolib.markup import HTML, Text from xmodule.modulestore.django import modulestore @@ -47,6 +48,8 @@ class CourseBadgesDisabledError(Exception): class BadgeClass(models.Model): """ Specifies a badge class to be registered with a backend. + + .. no_pii: """ slug = models.SlugField(max_length=255, validators=[validate_lowercase]) issuing_component = models.SlugField(max_length=50, default='', blank=True, validators=[validate_lowercase]) @@ -59,8 +62,8 @@ class BadgeClass(models.Model): image = models.ImageField(upload_to='badge_classes', validators=[validate_badge_image]) def __unicode__(self): - return u"".format( - slug=self.slug, issuing_component=self.issuing_component + return HTML(u"").format( + slug=HTML(self.slug), issuing_component=HTML(self.issuing_component) ) @classmethod @@ -140,6 +143,8 @@ class BadgeClass(models.Model): class BadgeAssertion(TimeStampedModel): """ Tracks badges on our side of the badge baking transaction + + .. no_pii: """ user = models.ForeignKey(User, on_delete=models.CASCADE) badge_class = models.ForeignKey(BadgeClass, on_delete=models.CASCADE) @@ -149,9 +154,10 @@ class BadgeAssertion(TimeStampedModel): assertion_url = models.URLField() def __unicode__(self): - return u"<{username} Badge Assertion for {slug} for {issuing_component}".format( - username=self.user.username, slug=self.badge_class.slug, - issuing_component=self.badge_class.issuing_component, + return HTML(u"<{username} Badge Assertion for {slug} for {issuing_component}").format( + username=HTML(self.user.username), + slug=HTML(self.badge_class.slug), + issuing_component=HTML(self.badge_class.issuing_component), ) @classmethod @@ -174,6 +180,8 @@ BadgeAssertion._meta.get_field('created').db_index = True class CourseCompleteImageConfiguration(models.Model): """ Contains the icon configuration for badges for a specific course mode. + + .. no_pii: """ mode = models.CharField( max_length=125, @@ -197,9 +205,9 @@ class CourseCompleteImageConfiguration(models.Model): ) def __unicode__(self): - return u"".format( - mode=self.mode, - default=u" (default)" if self.default else u'' + return HTML(u"").format( + mode=HTML(self.mode), + default=HTML(u" (default)") if self.default else HTML(u'') ) def clean(self): @@ -228,6 +236,8 @@ class CourseEventBadgesConfiguration(ConfigurationModel): """ Determines the settings for meta course awards-- such as completing a certain number of courses or enrolling in a certain number of them. + + .. no_pii: """ courses_completed = models.TextField( blank=True, default='', @@ -256,7 +266,9 @@ class CourseEventBadgesConfiguration(ConfigurationModel): ) def __unicode__(self): - return u"".format(u"Enabled" if self.enabled else u"Disabled") + return HTML(u"").format( + Text(u"Enabled") if self.enabled else Text(u"Disabled") + ) @property def completed_settings(self): diff --git a/lms/djangoapps/branding/models.py b/lms/djangoapps/branding/models.py index 2626a55d04..26607c3918 100644 --- a/lms/djangoapps/branding/models.py +++ b/lms/djangoapps/branding/models.py @@ -24,6 +24,8 @@ class BrandingInfoConfig(ConfigurationModel): "logo_tag": "Video hosted by XuetangX.com" } } + + .. no_pii: """ class Meta(ConfigurationModel.Meta): app_label = "branding" @@ -57,6 +59,8 @@ class BrandingApiConfig(ConfigurationModel): When this flag is disabled, the api will return 404. When the flag is enabled, the api will returns the valid reponse. + + .. no_pii: """ class Meta(ConfigurationModel.Meta): app_label = "branding" diff --git a/lms/djangoapps/bulk_email/models.py b/lms/djangoapps/bulk_email/models.py index c427c91b3a..1ad3b0a744 100644 --- a/lms/djangoapps/bulk_email/models.py +++ b/lms/djangoapps/bulk_email/models.py @@ -27,6 +27,8 @@ log = logging.getLogger(__name__) class Email(models.Model): """ Abstract base class for common information for an email. + + .. no_pii: """ sender = models.ForeignKey(User, default=1, blank=True, null=True, on_delete=models.CASCADE) slug = models.CharField(max_length=128, db_index=True) @@ -65,6 +67,8 @@ class Target(models.Model): SEND_TO_COHORT), then explicitly call the method on self.cohorttarget, which is created by django as part of this inheritance setup. These calls require pylint disable no-member in several locations in this class. + + .. no_pii: """ target_type = models.CharField(max_length=64, choices=EMAIL_TARGET_CHOICES) @@ -138,6 +142,8 @@ class Target(models.Model): class CohortTarget(Target): """ Subclass of Target, specifically referring to a cohort. + + .. no_pii: """ cohort = models.ForeignKey('course_groups.CourseUserGroup', on_delete=models.CASCADE) @@ -181,6 +187,8 @@ class CohortTarget(Target): class CourseModeTarget(Target): """ Subclass of Target, specifically for course modes. + + .. no_pii: """ track = models.ForeignKey('course_modes.CourseMode', on_delete=models.CASCADE) @@ -226,6 +234,8 @@ class CourseModeTarget(Target): class CourseEmail(Email): """ Stores information for an email to a course. + + .. no_pii: """ class Meta(object): app_label = "bulk_email" @@ -302,6 +312,8 @@ class CourseEmail(Email): class Optout(models.Model): """ Stores users that have opted out of receiving emails from a course. + + .. no_pii: """ # Allowing null=True to support data migration from email->user. # We need to first create the 'user' column with some sort of default in order to run the data migration, @@ -327,6 +339,8 @@ class CourseEmailTemplate(models.Model): Initialization takes place in a migration that in turn loads a fixture. The admin console interface disables add and delete operations. Validation is handled in the CourseEmailTemplateForm class. + + .. no_pii: """ class Meta(object): app_label = "bulk_email" @@ -407,6 +421,8 @@ class CourseEmailTemplate(models.Model): class CourseAuthorization(models.Model): """ Enable the course email feature on a course-by-course basis. + + .. no_pii: """ class Meta(object): app_label = "bulk_email" @@ -442,6 +458,8 @@ class BulkEmailFlag(ConfigurationModel): Staff can only send bulk email for a course if all the following conditions are true: 1. BulkEmailFlag is enabled. 2. Course-specific authorization not required, or course authorized to use bulk email. + + .. no_pii: """ # boolean field 'enabled' inherited from parent ConfigurationModel require_course_email_auth = models.BooleanField(default=True) diff --git a/lms/djangoapps/ccx/models.py b/lms/djangoapps/ccx/models.py index e47e918431..d05a5f3047 100644 --- a/lms/djangoapps/ccx/models.py +++ b/lms/djangoapps/ccx/models.py @@ -23,6 +23,8 @@ log = logging.getLogger("edx.ccx") class CustomCourseForEdX(models.Model): """ A Custom Course. + + .. no_pii: """ course_id = CourseKeyField(max_length=255, db_index=True) display_name = models.CharField(max_length=255) @@ -106,6 +108,8 @@ class CustomCourseForEdX(models.Model): class CcxFieldOverride(models.Model): """ Field overrides for custom courses. + + .. no_pii: """ ccx = models.ForeignKey(CustomCourseForEdX, db_index=True, on_delete=models.CASCADE) location = UsageKeyField(max_length=255, db_index=True) diff --git a/lms/djangoapps/certificates/models.py b/lms/djangoapps/certificates/models.py index a8a858ccb0..2935ea3343 100644 --- a/lms/djangoapps/certificates/models.py +++ b/lms/djangoapps/certificates/models.py @@ -130,6 +130,8 @@ class CertificateWhitelist(models.Model): regardless of their grade unless they are on the embargoed country restriction list (allow_certificate set to False in userprofile). + + .. no_pii: """ class Meta(object): app_label = "certificates" @@ -213,6 +215,10 @@ class EligibleCertificateManager(models.Manager): class GeneratedCertificate(models.Model): """ Base model for generated certificates + + .. pii: PII can exist in the generated certificate linked to in this model. Certificate data is currently retained. + .. pii_types: name, username + .. pii_retirement: retained """ # Import here instead of top of file since this module gets imported before # the course_modes app is loaded, resulting in a Django deprecation warning. @@ -374,6 +380,8 @@ class GeneratedCertificate(models.Model): class CertificateGenerationHistory(TimeStampedModel): """ Model for storing Certificate Generation History. + + .. no_pii: """ course_id = CourseKeyField(max_length=255) @@ -436,6 +444,8 @@ class CertificateGenerationHistory(TimeStampedModel): class CertificateInvalidation(TimeStampedModel): """ Model for storing Certificate Invalidation. + + .. no_pii: """ generated_certificate = models.ForeignKey(GeneratedCertificate, on_delete=models.CASCADE) invalidated_by = models.ForeignKey(User, on_delete=models.CASCADE) @@ -527,7 +537,7 @@ def certificate_status_for_student(student, course_id): def certificate_status(generated_certificate): - ''' + """ This returns a dictionary with a key for status, and other information. The status is one of the following: @@ -554,7 +564,7 @@ def certificate_status(generated_certificate): If the student has been graded, the dictionary also contains their grade for the course with the key "grade". - ''' + """ # Import here instead of top of file since this module gets imported before # the course_modes app is loaded, resulting in a Django deprecation warning. from course_modes.models import CourseMode @@ -610,7 +620,8 @@ def certificate_info_for_user(user, course_id, grade, user_is_whitelisted, user_ class ExampleCertificateSet(TimeStampedModel): - """A set of example certificates. + """ + A set of example certificates. Example certificates are used to verify that certificate generation is working for a particular course. @@ -619,6 +630,7 @@ class ExampleCertificateSet(TimeStampedModel): (e.g. honor and verified), in which case we generate multiple example certificates for the course. + .. no_pii: """ course_key = CourseKeyField(max_length=255, db_index=True) @@ -701,7 +713,8 @@ def _make_uuid(): class ExampleCertificate(TimeStampedModel): - """Example certificate. + """ + Example certificate. Example certificates are used to verify that certificate generation is working for a particular course. @@ -717,6 +730,7 @@ class ExampleCertificate(TimeStampedModel): 3) We use dummy values. + .. no_pii: """ class Meta(object): app_label = "certificates" @@ -870,12 +884,15 @@ class ExampleCertificate(TimeStampedModel): class CertificateGenerationCourseSetting(TimeStampedModel): - """Enable or disable certificate generation for a particular course. + """ + Enable or disable certificate generation for a particular course. In general, we should only enable self-generated certificates for a course once we successfully generate example certificates for the course. This is enforced in the UI layer, but not in the data layer. + + .. no_pii: """ course_key = CourseKeyField(max_length=255, db_index=True) @@ -973,6 +990,7 @@ class CertificateGenerationConfiguration(ConfigurationModel): will appear for courses that have enabled self-generated certificates. + .. no_pii: """ class Meta(ConfigurationModel.Meta): app_label = "certificates" @@ -993,6 +1011,8 @@ class CertificateHtmlViewConfiguration(ConfigurationModel): "logo_src": "http://www.edx.org/static/images/honor-logo.png" } } + + .. no_pii: """ class Meta(ConfigurationModel.Meta): app_label = "certificates" @@ -1029,6 +1049,7 @@ class CertificateTemplate(TimeStampedModel): A particular course may have several kinds of certificate templates (e.g. honor and verified). + .. no_pii: """ name = models.CharField( max_length=255, @@ -1071,7 +1092,8 @@ class CertificateTemplate(TimeStampedModel): max_length=2, blank=True, null=True, - help_text=u'Only certificates for courses in the selected language will be rendered using this template. Course language is determined by the first two letters of the language code.' + help_text=u'Only certificates for courses in the selected language will be rendered using this template. ' + u'Course language is determined by the first two letters of the language code.' ) def __unicode__(self): @@ -1104,6 +1126,7 @@ class CertificateTemplateAsset(TimeStampedModel): This model stores assets used in custom web certificate templates such as image, css files. + .. no_pii: """ description = models.CharField( max_length=255, diff --git a/lms/djangoapps/commerce/models.py b/lms/djangoapps/commerce/models.py index d95921a4d9..03e8afe22b 100644 --- a/lms/djangoapps/commerce/models.py +++ b/lms/djangoapps/commerce/models.py @@ -7,7 +7,11 @@ from django.utils.translation import ugettext_lazy as _ class CommerceConfiguration(ConfigurationModel): - """ Commerce configuration """ + """ + Commerce configuration + + .. no_pii: + """ class Meta(object): app_label = "commerce" diff --git a/lms/djangoapps/course_goals/models.py b/lms/djangoapps/course_goals/models.py index 366dbfa470..e2a6f15ae5 100644 --- a/lms/djangoapps/course_goals/models.py +++ b/lms/djangoapps/course_goals/models.py @@ -25,6 +25,8 @@ GOAL_KEY_CHOICES = Choices( class CourseGoal(models.Model): """ Represents a course goal set by a user on the course home page. + + .. no_pii: """ user = models.ForeignKey(User, blank=False, on_delete=models.CASCADE) course_key = CourseKeyField(max_length=255, db_index=True) diff --git a/lms/djangoapps/courseware/models.py b/lms/djangoapps/courseware/models.py index d54fcb96df..f6d89130fc 100644 --- a/lms/djangoapps/courseware/models.py +++ b/lms/djangoapps/courseware/models.py @@ -27,6 +27,8 @@ from six import text_type import coursewarehistoryextended from opaque_keys.edx.django.models import BlockTypeKeyField, CourseKeyField, UsageKeyField +from openedx.core.djangolib.markup import HTML + log = logging.getLogger("edx.courseware") @@ -74,6 +76,8 @@ class ChunkingManager(models.Manager): class StudentModule(models.Model): """ Keeps student state for a particular module in a particular course. + + .. no_pii: """ objects = ChunkingManager() MODEL_TAGS = ['course_id', 'module_type'] @@ -175,8 +179,11 @@ class StudentModule(models.Model): class BaseStudentModuleHistory(models.Model): - """Abstract class containing most fields used by any class - storing Student Module History""" + """ + Abstract class containing most fields used by any class storing Student Module History + + .. no_pii: + """ objects = ChunkingManager() HISTORY_SAVING_TYPES = {'problem'} @@ -265,6 +272,8 @@ class StudentModuleHistory(BaseStudentModuleHistory): class XBlockFieldBase(models.Model): """ Base class for all XBlock field storage. + + .. no_pii: """ objects = ChunkingManager() @@ -283,7 +292,10 @@ class XBlockFieldBase(models.Model): def __unicode__(self): keys = [field.name for field in self._meta.get_fields() if field.name not in ('created', 'modified')] - return u'{}<{!r}'.format(self.__class__.__name__, {key: getattr(self, key) for key in keys}) + return HTML(u'{}<{!r}').format( + HTML(self.__class__.__name__), + {key: HTML(getattr(self, key)) for key in keys} + ) class XModuleUserStateSummaryField(XBlockFieldBase): @@ -329,6 +341,8 @@ class XModuleStudentInfoField(XBlockFieldBase): class OfflineComputedGrade(models.Model): """ Table of grades computed offline for a given user and course. + + .. no_pii: """ user = models.ForeignKey(User, db_index=True, on_delete=models.CASCADE) course_id = CourseKeyField(max_length=255, db_index=True) @@ -350,6 +364,8 @@ class OfflineComputedGradeLog(models.Model): """ Log of when offline grades are computed. Use this to be able to show instructor when the last computed grades were done. + + .. no_pii: """ class Meta(object): @@ -371,6 +387,8 @@ class StudentFieldOverride(TimeStampedModel): Holds the value of a specific field overriden for a student. This is used by the code in the `lms.djangoapps.courseware.student_field_overrides` module to provide overrides of xblock fields on a per user basis. + + .. no_pii: """ course_id = CourseKeyField(max_length=255, db_index=True) location = UsageKeyField(max_length=255, db_index=True) @@ -385,9 +403,12 @@ class StudentFieldOverride(TimeStampedModel): class DynamicUpgradeDeadlineConfiguration(ConfigurationModel): - """ Dynamic upgrade deadline configuration. + """ + Dynamic upgrade deadline configuration. This model controls the behavior of the dynamic upgrade deadline for self-paced courses. + + .. no_pii: """ class Meta(object): app_label = 'courseware' @@ -418,6 +439,8 @@ class CourseDynamicUpgradeDeadlineConfiguration(OptOutDynamicUpgradeDeadlineMixi This model controls dynamic upgrade deadlines on a per-course run level, allowing course runs to have different deadlines or opt out of the functionality altogether. + + .. no_pii: """ KEY_FIELDS = ('course_id',) @@ -440,6 +463,8 @@ class OrgDynamicUpgradeDeadlineConfiguration(OptOutDynamicUpgradeDeadlineMixin, This model controls dynamic upgrade deadlines on a per-org level, allowing organizations to have different deadlines or opt out of the functionality altogether. + + .. no_pii: """ KEY_FIELDS = ('org_id',) diff --git a/lms/djangoapps/email_marketing/models.py b/lms/djangoapps/email_marketing/models.py index 3294c78886..2478143705 100644 --- a/lms/djangoapps/email_marketing/models.py +++ b/lms/djangoapps/email_marketing/models.py @@ -7,7 +7,11 @@ from django.utils.translation import ugettext_lazy as _ class EmailMarketingConfiguration(ConfigurationModel): - """ Email marketing configuration """ + """ + Email marketing configuration + + .. no_pii: + """ class Meta(object): app_label = "email_marketing" diff --git a/lms/djangoapps/experiments/models.py b/lms/djangoapps/experiments/models.py index 226c8ab71e..a3c5766e91 100644 --- a/lms/djangoapps/experiments/models.py +++ b/lms/djangoapps/experiments/models.py @@ -4,6 +4,9 @@ from model_utils.models import TimeStampedModel class ExperimentData(TimeStampedModel): + """ + .. no_pii: + """ user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) experiment_id = models.PositiveSmallIntegerField( null=False, blank=False, db_index=True, verbose_name='Experiment ID' @@ -23,6 +26,9 @@ class ExperimentData(TimeStampedModel): class ExperimentKeyValue(TimeStampedModel): + """ + .. no_pii: + """ experiment_id = models.PositiveSmallIntegerField( null=False, blank=False, db_index=True, verbose_name='Experiment ID' ) diff --git a/lms/djangoapps/grades/config/models.py b/lms/djangoapps/grades/config/models.py index df0ec8c31c..720207a0d4 100644 --- a/lms/djangoapps/grades/config/models.py +++ b/lms/djangoapps/grades/config/models.py @@ -17,6 +17,8 @@ class PersistentGradesEnabledFlag(ConfigurationModel): When this feature flag is set to true, individual courses must also have persistent grades enabled for the feature to take effect. + + .. no_pii: """ # this field overrides course-specific settings to enable the feature for all courses enabled_for_all_courses = BooleanField(default=False) @@ -58,6 +60,8 @@ class CoursePersistentGradesFlag(ConfigurationModel): Enables persistent grades for a specific course. Only has an effect if the general flag above is set to True. + + .. no_pii: """ KEY_FIELDS = ('course_id',) @@ -76,7 +80,7 @@ class CoursePersistentGradesFlag(ConfigurationModel): class ComputeGradesSetting(ConfigurationModel): """ - ... + .. no_pii: """ class Meta(object): app_label = "grades" diff --git a/lms/djangoapps/grades/models.py b/lms/djangoapps/grades/models.py index 3b22c414f2..7fc01f5672 100644 --- a/lms/djangoapps/grades/models.py +++ b/lms/djangoapps/grades/models.py @@ -131,6 +131,8 @@ class VisibleBlocks(models.Model): This state is represented using an array of BlockRecord, stored in the blocks_json field. A hash of this json array is used for lookup purposes. + + .. no_pii: """ blocks_json = models.TextField() hashed = models.CharField(max_length=100, unique=True) @@ -259,6 +261,8 @@ class VisibleBlocks(models.Model): class PersistentSubsectionGrade(TimeStampedModel): """ A django model tracking persistent grades at the subsection level. + + .. no_pii: """ class Meta(object): @@ -492,6 +496,8 @@ class PersistentSubsectionGrade(TimeStampedModel): class PersistentCourseGrade(TimeStampedModel): """ A django model tracking persistent course grades. + + .. no_pii: """ class Meta(object): @@ -626,6 +632,8 @@ class PersistentCourseGrade(TimeStampedModel): class PersistentSubsectionGradeOverride(models.Model): """ A django model tracking persistent grades overrides at the subsection level. + + .. no_pii: """ class Meta(object): app_label = "grades" @@ -732,6 +740,8 @@ class PersistentSubsectionGradeOverride(models.Model): class PersistentSubsectionGradeOverrideHistory(models.Model): """ A django model tracking persistent grades override audit records. + + .. no_pii: """ PROCTORING = 'PROCTORING' GRADEBOOK = 'GRADEBOOK' diff --git a/lms/djangoapps/instructor_task/config/models.py b/lms/djangoapps/instructor_task/config/models.py index 795c1138df..701f0f2452 100644 --- a/lms/djangoapps/instructor_task/config/models.py +++ b/lms/djangoapps/instructor_task/config/models.py @@ -10,5 +10,7 @@ class GradeReportSetting(ConfigurationModel): """ Sets the batch size used when running grade reports with multiple celery workers. + + .. no_pii: """ batch_size = IntegerField(default=100) diff --git a/lms/djangoapps/instructor_task/models.py b/lms/djangoapps/instructor_task/models.py index 612f07f075..ab7143afd8 100644 --- a/lms/djangoapps/instructor_task/models.py +++ b/lms/djangoapps/instructor_task/models.py @@ -58,6 +58,8 @@ class InstructorTask(models.Model): `requester` stores id of user who submitted the task `created` stores date that entry was first created `updated` stores date that entry was last modified + + .. no_pii: """ class Meta(object): app_label = "instructor_task" diff --git a/lms/djangoapps/lms_xblock/models.py b/lms/djangoapps/lms_xblock/models.py index 14129647ca..ca38670617 100644 --- a/lms/djangoapps/lms_xblock/models.py +++ b/lms/djangoapps/lms_xblock/models.py @@ -14,6 +14,8 @@ from xblock.core import XBlockAside class XBlockAsidesConfig(ConfigurationModel): """ Configuration for XBlockAsides. + + .. no_pii: """ class Meta(ConfigurationModel.Meta): diff --git a/lms/djangoapps/lti_provider/models.py b/lms/djangoapps/lti_provider/models.py index 12498aed6f..06965f9c9d 100644 --- a/lms/djangoapps/lti_provider/models.py +++ b/lms/djangoapps/lti_provider/models.py @@ -25,6 +25,8 @@ class LtiConsumer(models.Model): Database model representing an LTI consumer. This model stores the consumer specific settings, such as the OAuth key/secret pair and any LTI fields that must be persisted. + + .. no_pii: """ consumer_name = models.CharField(max_length=255, unique=True) consumer_key = models.CharField(max_length=32, unique=True, db_index=True, default=short_token) @@ -91,6 +93,8 @@ class OutcomeService(models.Model): Some LTI-specified fields use the prefix lis_; this refers to the IMS Learning Information Services standard from which LTI inherits some properties + + .. no_pii: """ lis_outcome_service_url = models.CharField(max_length=255, unique=True) lti_consumer = models.ForeignKey(LtiConsumer, on_delete=models.CASCADE) @@ -109,6 +113,8 @@ class GradedAssignment(models.Model): Some LTI-specified fields use the prefix lis_; this refers to the IMS Learning Information Services standard from which LTI inherits some properties + + .. no_pii: """ user = models.ForeignKey(User, db_index=True, on_delete=models.CASCADE) course_key = CourseKeyField(max_length=255, db_index=True) @@ -127,6 +133,8 @@ class LtiUser(models.Model): The LTI user_id field is guaranteed to be unique per LTI consumer (per to the LTI spec), so we guarantee a unique mapping from LTI to edX account by using the lti_consumer/lti_user_id tuple. + + .. no_pii: """ lti_consumer = models.ForeignKey(LtiConsumer, on_delete=models.CASCADE) lti_user_id = models.CharField(max_length=255) diff --git a/lms/djangoapps/mobile_api/models.py b/lms/djangoapps/mobile_api/models.py index 839a496c06..34043a9d04 100644 --- a/lms/djangoapps/mobile_api/models.py +++ b/lms/djangoapps/mobile_api/models.py @@ -14,6 +14,8 @@ class MobileApiConfig(ConfigurationModel): The order in which the comma-separated list of names of profiles are given is in priority order. + + .. no_pii: """ video_profiles = models.TextField( blank=True, @@ -34,6 +36,8 @@ class MobileApiConfig(ConfigurationModel): class AppVersionConfig(models.Model): """ Configuration for mobile app versions available. + + .. no_pii: """ PLATFORM_CHOICES = tuple([ (platform, platform) @@ -90,6 +94,8 @@ class IgnoreMobileAvailableFlagConfig(ConfigurationModel): # pylint: disable=W5 Enabling this configuration will cause the mobile_available flag check in access.py._is_descriptor_mobile_available to ignore the mobile_available flag. + + .. no_pii: """ class Meta(object): diff --git a/lms/djangoapps/notes/models.py b/lms/djangoapps/notes/models.py index a1f7f2fcdf..f70dffffa2 100644 --- a/lms/djangoapps/notes/models.py +++ b/lms/djangoapps/notes/models.py @@ -10,6 +10,14 @@ from six import text_type class Note(models.Model): + """ + Stores user Notes for the LMS local Notes service. + + .. pii: Legacy model for an app that edx.org hasn't used since 2013 + .. pii_types: other + .. pii_retirement: retained + """ + user = models.ForeignKey(User, db_index=True, on_delete=models.CASCADE) course_id = CourseKeyField(max_length=255, db_index=True) uri = models.CharField(max_length=255, db_index=True) diff --git a/lms/djangoapps/rss_proxy/models.py b/lms/djangoapps/rss_proxy/models.py index 5694b0096c..49afb7030f 100644 --- a/lms/djangoapps/rss_proxy/models.py +++ b/lms/djangoapps/rss_proxy/models.py @@ -9,6 +9,8 @@ class WhitelistedRssUrl(TimeStampedModel): """ Model for persisting RSS feed URLs which are whitelisted for proxying via this rss_proxy djangoapp. + + .. no_pii: """ url = models.CharField(max_length=255, unique=True, db_index=True) diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index c5967fd3be..1ed7beb44d 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -110,6 +110,10 @@ class Order(models.Model): This is the model for an order. Before purchase, an Order and its related OrderItems are used as the shopping cart. FOR ANY USER, THERE SHOULD ONLY EVER BE ZERO OR ONE ORDER WITH STATUS='cart'. + + .. pii: Contains many PII fields in an app edx.org does not currently use. "other" data is payment information. + .. pii_types: name, location, email_address, other + .. pii_retirement: retained """ class Meta(object): app_label = "shoppingcart" @@ -639,6 +643,8 @@ class OrderItem(TimeStampedModel): Each implementation of OrderItem should provide its own purchased_callback as a method. + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -824,6 +830,10 @@ class Invoice(TimeStampedModel): This table capture all the information needed to support "invoicing" which is when a user wants to purchase Registration Codes, but will not do so via a Credit Card transaction. + + .. pii: Contains many PII fields in an app edx.org does not currently use + .. pii_types: name, location, email_address + .. pii_retirement: retained """ class Meta(object): app_label = "shoppingcart" @@ -996,6 +1006,7 @@ class InvoiceTransaction(TimeStampedModel): create a transaction with a negative amount to represent the refund. + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1086,6 +1097,8 @@ class InvoiceItem(TimeStampedModel): there might be an invoice item representing 10 registration codes for the DemoX course. + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1130,6 +1143,7 @@ class CourseRegistrationCodeInvoiceItem(InvoiceItem): This is an invoice item that represents a payment for a course registration. + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1166,6 +1180,7 @@ class InvoiceHistory(models.Model): transaction, so the history record is created only if the invoice change is successfully persisted. + .. no_pii: """ timestamp = models.DateTimeField(auto_now_add=True, db_index=True) invoice = models.ForeignKey(Invoice, on_delete=models.CASCADE) @@ -1225,6 +1240,8 @@ class CourseRegistrationCode(models.Model): """ This table contains registration codes With registration code, a user can register for a course for free + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1263,6 +1280,8 @@ class CourseRegistrationCode(models.Model): class RegistrationCodeRedemption(models.Model): """ This model contains the registration-code redemption info + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1323,6 +1342,8 @@ class Coupon(models.Model): """ This table contains coupon codes A user can get a discount offer on course if provide coupon code + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1350,6 +1371,8 @@ class Coupon(models.Model): class CouponRedemption(models.Model): """ This table contain coupon redemption info + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1466,6 +1489,8 @@ class CouponRedemption(models.Model): class PaidCourseRegistration(OrderItem): """ This is an inventory item for paying for a course registration + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1660,6 +1685,8 @@ class CourseRegCodeItem(OrderItem): """ This is an inventory item for paying for generating course registration codes + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1827,6 +1854,8 @@ class CourseRegCodeItemAnnotation(models.Model): generates report for the paid courses, each report item must contain the payment account associated with a course. And unfortunately we didn't have the concept of a "SKU" or stock item where we could keep this association, so this is to retrofit it. + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1844,6 +1873,8 @@ class PaidCourseRegistrationAnnotation(models.Model): generates report for the paid courses, each report item must contain the payment account associated with a course. And unfortunately we didn't have the concept of a "SKU" or stock item where we could keep this association, so this is to retrofit it. + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -1858,6 +1889,8 @@ class PaidCourseRegistrationAnnotation(models.Model): class CertificateItem(OrderItem): """ This is an inventory item for purchasing certificates + + .. no_pii: """ class Meta(object): app_label = "shoppingcart" @@ -2082,16 +2115,23 @@ class CertificateItem(OrderItem): class DonationConfiguration(ConfigurationModel): - """Configure whether donations are enabled on the site.""" + """ + Configure whether donations are enabled on the site. + + .. no_pii: + """ class Meta(ConfigurationModel.Meta): app_label = "shoppingcart" class Donation(OrderItem): - """A donation made by a user. + """ + A donation made by a user. Donations can be made for a specific course or to the organization as a whole. Users can choose the donation amount. + + .. no_pii: """ class Meta(object): diff --git a/lms/djangoapps/survey/models.py b/lms/djangoapps/survey/models.py index e7500c7878..52b3bc6955 100644 --- a/lms/djangoapps/survey/models.py +++ b/lms/djangoapps/survey/models.py @@ -24,6 +24,8 @@ class SurveyForm(TimeStampedModel): that is presented to the end user. A SurveyForm is not tied to a particular run of a course, to allow for sharing of Surveys across courses + + .. no_pii: """ name = models.CharField(max_length=255, db_index=True, unique=True) form = models.TextField() @@ -162,8 +164,13 @@ class SurveyForm(TimeStampedModel): class SurveyAnswer(TimeStampedModel): + # pylint: disable=line-too-long """ Model for the answers that a user gives for a particular form in a course + + .. pii: These are free-form questions asked by course authors. Types below are current as of Feb 2019, new ones could be added. "other" PII currently includes "company", "job title", and "work experience". + .. pii_types: name, location, other + .. pii_retirement: retained """ user = models.ForeignKey(User, db_index=True, on_delete=models.CASCADE) form = models.ForeignKey(SurveyForm, db_index=True, on_delete=models.CASCADE) diff --git a/lms/djangoapps/teams/models.py b/lms/djangoapps/teams/models.py index 4064dd0567..cb9e31379a 100644 --- a/lms/djangoapps/teams/models.py +++ b/lms/djangoapps/teams/models.py @@ -1,4 +1,6 @@ -"""Django models related to teams functionality.""" +""" +Django models related to teams functionality. +""" from datetime import datetime from uuid import uuid4 @@ -39,16 +41,20 @@ from .errors import AlreadyOnTeamInCourse, ImmutableMembershipFieldException, No @receiver(comment_voted) @receiver(comment_created) def post_create_vote_handler(sender, **kwargs): # pylint: disable=unused-argument - """Update the user's last activity date upon creating or voting for a - post.""" + """ + Update the user's last activity date upon creating or voting for a + post. + """ handle_activity(kwargs['user'], kwargs['post']) @receiver(thread_followed) @receiver(thread_unfollowed) def post_followed_unfollowed_handler(sender, **kwargs): # pylint: disable=unused-argument - """Update the user's last activity date upon followed or unfollowed of a - post.""" + """ + Update the user's last activity date upon followed or unfollowed of a + post. + """ handle_activity(kwargs['user'], kwargs['post']) @@ -57,21 +63,26 @@ def post_followed_unfollowed_handler(sender, **kwargs): # pylint: disable=unuse @receiver(comment_edited) @receiver(comment_deleted) def post_edit_delete_handler(sender, **kwargs): # pylint: disable=unused-argument - """Update the user's last activity date upon editing or deleting a - post.""" + """ + Update the user's last activity date upon editing or deleting a + post. + """ post = kwargs['post'] handle_activity(kwargs['user'], post, long(post.user_id)) @receiver(comment_endorsed) def comment_endorsed_handler(sender, **kwargs): # pylint: disable=unused-argument - """Update the user's last activity date upon endorsing a comment.""" + """ + Update the user's last activity date upon endorsing a comment. + """ comment = kwargs['post'] handle_activity(kwargs['user'], comment, long(comment.thread.user_id)) def handle_activity(user, post, original_author_id=None): - """Handle user activity from django_comment_client and discussion_api + """ + Handle user activity from django_comment_client and discussion_api and update the user's last activity date. Checks if the user who performed the action is the original author, and that the discussion has the team context. @@ -83,7 +94,11 @@ def handle_activity(user, post, original_author_id=None): class CourseTeam(models.Model): - """This model represents team related info.""" + """ + This model represents team related info. + + .. no_pii: + """ class Meta(object): app_label = "teams" @@ -165,7 +180,11 @@ class CourseTeam(models.Model): class CourseTeamMembership(models.Model): - """This model represents the membership of a single user in a single team.""" + """ + This model represents the membership of a single user in a single team. + + .. no_pii: + """ class Meta(object): app_label = "teams" diff --git a/lms/djangoapps/verify_student/models.py b/lms/djangoapps/verify_student/models.py index 83f7d58342..425437a666 100644 --- a/lms/djangoapps/verify_student/models.py +++ b/lms/djangoapps/verify_student/models.py @@ -93,6 +93,10 @@ class IDVerificationAttempt(StatusModel): Each IDVerificationAttempt represents a Student's attempt to establish their identity through one of several methods that inherit from this Model, including PhotoVerification and SSOVerification. + + .. pii: The User's name is stored in this and sub-models + .. pii_types: name + .. pii_retirement: retained """ STATUS = Choices('created', 'ready', 'submitted', 'must_retry', 'approved', 'denied') user = models.ForeignKey(User, db_index=True, on_delete=models.CASCADE) @@ -142,6 +146,10 @@ class ManualVerification(IDVerificationAttempt): """ Each ManualVerification represents a user's verification that bypasses the need for any other verification. + + .. pii: The User's name is stored in the parent model + .. pii_types: name + .. pii_retirement: retained """ reason = models.CharField( @@ -173,6 +181,8 @@ class SSOVerification(IDVerificationAttempt): Each SSOVerification represents a Student's attempt to establish their identity by signing in with SSO. ID verification through SSO bypasses the need for photo verification. + + .. no_pii: """ OAUTH2 = 'third_party_auth.models.OAuth2ProviderConfig' @@ -254,6 +264,10 @@ class PhotoVerification(IDVerificationAttempt): attempt.status == PhotoVerification.STATUS.created attempt.status == "created" pending_requests = PhotoVerification.submitted.all() + + .. pii: The User's name is stored in the parent model, this one stores links to face and photo ID images + .. pii_types: name, image + .. pii_retirement: retained """ ######################## Fields Set During Creation ######################## # See class docstring for description of status states @@ -526,6 +540,10 @@ class SoftwareSecurePhotoVerification(PhotoVerification): Note: this model handles *inital* verifications (which you must perform at the time you register for a verified cert). + + .. pii: The User's name is stored in the parent model, this one stores links to face and photo ID images + .. pii_types: name, image + .. pii_retirement: retained """ # This is a base64.urlsafe_encode(rsa_encrypt(photo_id_aes_key), ss_pub_key) # So first we generate a random AES-256 key to encrypt our photo ID with. @@ -909,6 +927,8 @@ class VerificationDeadline(TimeStampedModel): If no verification deadline record exists for a course, then that course does not have a deadline. This means that users can submit photos at any time. + + .. no_pii: """ class Meta(object): app_label = "verify_student" diff --git a/openedx/core/djangoapps/api_admin/models.py b/openedx/core/djangoapps/api_admin/models.py index 8efa30796e..ab60261fa6 100644 --- a/openedx/core/djangoapps/api_admin/models.py +++ b/openedx/core/djangoapps/api_admin/models.py @@ -22,7 +22,13 @@ log = logging.getLogger(__name__) class ApiAccessRequest(TimeStampedModel): - """Model to track API access for a user.""" + """ + Model to track API access for a user. + + .. pii: Stores a website, company name, company address for this user + .. pii_types: location, external_service, other + .. pii_retirement: local_api + """ PENDING = 'pending' DENIED = 'denied' @@ -121,7 +127,11 @@ class ApiAccessRequest(TimeStampedModel): class ApiAccessConfig(ConfigurationModel): - """Configuration for API management.""" + """ + Configuration for API management. + + .. no_pii: + """ def __unicode__(self): return u'ApiAccessConfig [enabled={}]'.format(self.enabled) @@ -208,7 +218,11 @@ def _send_decision_email(instance): class Catalog(models.Model): - """A (non-Django-managed) model for Catalogs in the course discovery service.""" + """ + A (non-Django-managed) model for Catalogs in the course discovery service. + + .. no_pii: + """ id = models.IntegerField(primary_key=True) # pylint: disable=invalid-name name = models.CharField(max_length=255, null=False, blank=False) diff --git a/openedx/core/djangoapps/bookmarks/models.py b/openedx/core/djangoapps/bookmarks/models.py index c60aa8ad3b..4f192177f7 100644 --- a/openedx/core/djangoapps/bookmarks/models.py +++ b/openedx/core/djangoapps/bookmarks/models.py @@ -41,6 +41,8 @@ def parse_path_data(path_data): class Bookmark(TimeStampedModel): """ Bookmarks model. + + .. no_pii: """ user = models.ForeignKey(User, db_index=True, on_delete=models.CASCADE) course_key = CourseKeyField(max_length=255, db_index=True) @@ -189,6 +191,8 @@ class Bookmark(TimeStampedModel): class XBlockCache(TimeStampedModel): """ XBlockCache model to store info about xblocks. + + .. no_pii: """ course_key = CourseKeyField(max_length=255, db_index=True) diff --git a/openedx/core/djangoapps/catalog/models.py b/openedx/core/djangoapps/catalog/models.py index 9ee5b263b6..691bc6a8f9 100644 --- a/openedx/core/djangoapps/catalog/models.py +++ b/openedx/core/djangoapps/catalog/models.py @@ -9,7 +9,11 @@ from openedx.core.djangoapps.site_configuration import helpers class CatalogIntegration(ConfigurationModel): - """Manages configuration for connecting to the catalog service and using its API.""" + """ + Manages configuration for connecting to the catalog service and using its API. + + .. no_pii: + """ API_NAME = 'catalog' CACHE_KEY = 'catalog.api.data' diff --git a/openedx/core/djangoapps/ccxcon/models.py b/openedx/core/djangoapps/ccxcon/models.py index bac4b25f09..a93a95ac50 100644 --- a/openedx/core/djangoapps/ccxcon/models.py +++ b/openedx/core/djangoapps/ccxcon/models.py @@ -7,9 +7,10 @@ from django.db import models class CCXCon(models.Model): """ - The definition of the CCXCon model. - This will store the url and the oauth key to access the REST APIs - on the CCX Connector. + Definition of the CCXCon model. + Stores the url and the oauth key to access the REST APIs on the CCX Connector. + + .. no_pii: """ url = models.URLField(unique=True, db_index=True) oauth_client_id = models.CharField(max_length=255) diff --git a/openedx/core/djangoapps/content/block_structure/config/models.py b/openedx/core/djangoapps/content/block_structure/config/models.py index 88d2005a5b..b8db5e25eb 100644 --- a/openedx/core/djangoapps/content/block_structure/config/models.py +++ b/openedx/core/djangoapps/content/block_structure/config/models.py @@ -8,6 +8,8 @@ from config_models.models import ConfigurationModel class BlockStructureConfiguration(ConfigurationModel): """ Configuration model for Block Structures. + + .. no_pii: """ DEFAULT_PRUNE_KEEP_COUNT = 5 DEFAULT_CACHE_TIMEOUT_IN_SECONDS = 60 * 60 * 24 # 24 hours diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index 69a7750f69..2c685ef16f 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -124,6 +124,8 @@ def _storage_error_handling(bs_model, operation, is_read_operation=False): class BlockStructureModel(TimeStampedModel): """ Model for storing Block Structure information. + + .. no_pii: """ VERSION_FIELDS = [ u'data_version', diff --git a/openedx/core/djangoapps/content/course_overviews/models.py b/openedx/core/djangoapps/content/course_overviews/models.py index e6f4382f64..96f2334302 100644 --- a/openedx/core/djangoapps/content/course_overviews/models.py +++ b/openedx/core/djangoapps/content/course_overviews/models.py @@ -40,6 +40,8 @@ class CourseOverview(TimeStampedModel): user dashboard (enrolled courses) course catalog (courses to enroll in) course about (meta data about the course) + + .. no_pii: """ class Meta(object): @@ -707,6 +709,8 @@ class CourseOverview(TimeStampedModel): class CourseOverviewTab(models.Model): """ Model for storing and caching tabs information of a course. + + .. no_pii: """ tab_id = models.CharField(max_length=50) course_overview = models.ForeignKey(CourseOverview, db_index=True, related_name="tabs", on_delete=models.CASCADE) @@ -779,6 +783,8 @@ class CourseOverviewImageSet(TimeStampedModel): process to do it, and it can happen in a follow-on PR if anyone is interested in extending this functionality. + + .. no_pii: """ course_overview = models.OneToOneField(CourseOverview, db_index=True, related_name="image_set", on_delete=models.CASCADE) @@ -860,6 +866,8 @@ class CourseOverviewImageConfig(ConfigurationModel): to take effect. You might want to do this if you're doing precise theming of your install of edx-platform... but really, you probably don't want to do this at all at the moment, given how new this is. :-P + + .. no_pii: """ # Small thumbnail, for things like the student dashboard small_width = models.IntegerField(default=375) diff --git a/openedx/core/djangoapps/contentserver/models.py b/openedx/core/djangoapps/contentserver/models.py index ba4c08d97a..23c9c6f16b 100644 --- a/openedx/core/djangoapps/contentserver/models.py +++ b/openedx/core/djangoapps/contentserver/models.py @@ -7,7 +7,11 @@ from django.db.models.fields import PositiveIntegerField, TextField class CourseAssetCacheTtlConfig(ConfigurationModel): - """Configuration for the TTL of course assets.""" + """ + Configuration for the TTL of course assets. + + .. no_pii: + """ class Meta(object): app_label = 'contentserver' @@ -30,7 +34,11 @@ class CourseAssetCacheTtlConfig(ConfigurationModel): class CdnUserAgentsConfig(ConfigurationModel): - """Configuration for the user agents we expect to see from CDNs.""" + """ + Configuration for the user agents we expect to see from CDNs. + + .. no_pii: + """ class Meta(object): app_label = 'contentserver' diff --git a/openedx/core/djangoapps/cors_csrf/models.py b/openedx/core/djangoapps/cors_csrf/models.py index 44194fb246..1cc1410245 100644 --- a/openedx/core/djangoapps/cors_csrf/models.py +++ b/openedx/core/djangoapps/cors_csrf/models.py @@ -5,10 +5,12 @@ from django.utils.translation import ugettext_lazy as _ class XDomainProxyConfiguration(ConfigurationModel): - """Cross-domain proxy configuration. + """ + Cross-domain proxy configuration. See `openedx.core.djangoapps.cors_csrf.views.xdomain_proxy` for an explanation of how this works. + .. no_pii: """ whitelist = models.fields.TextField( diff --git a/openedx/core/djangoapps/course_groups/models.py b/openedx/core/djangoapps/course_groups/models.py index 7bb9a015b2..c576c819cf 100644 --- a/openedx/core/djangoapps/course_groups/models.py +++ b/openedx/core/djangoapps/course_groups/models.py @@ -22,6 +22,8 @@ class CourseUserGroup(models.Model): This model represents groups of users in a course. Groups may have different types, which may be treated specially. For example, a user can be in at most one cohort per course, and cohorts are used to split up the forums by group. + + .. no_pii: """ class Meta(object): unique_together = (('name', 'course_id'), ) @@ -67,8 +69,11 @@ class CourseUserGroup(models.Model): class CohortMembership(models.Model): - """Used internally to enforce our particular definition of uniqueness""" + """ + Used internally to enforce our particular definition of uniqueness. + .. no_pii: + """ course_user_group = models.ForeignKey(CourseUserGroup, on_delete=models.CASCADE) user = models.ForeignKey(User, on_delete=models.CASCADE) course_id = CourseKeyField(max_length=255) @@ -143,6 +148,8 @@ def remove_user_from_cohort(sender, instance, **kwargs): # pylint: disable=unus class CourseUserGroupPartitionGroup(models.Model): """ Create User Partition Info. + + .. no_pii: """ course_user_group = models.OneToOneField(CourseUserGroup, on_delete=models.CASCADE) partition_id = models.IntegerField( @@ -159,6 +166,8 @@ class CourseCohortsSettings(models.Model): """ This model represents cohort settings for courses. The only non-deprecated fields are `is_cohorted` and `course_id`. + + .. no_pii: """ is_cohorted = models.BooleanField(default=False) @@ -197,6 +206,8 @@ class CourseCohortsSettings(models.Model): class CourseCohort(models.Model): """ This model represents cohort related info. + + .. no_pii: """ course_user_group = models.OneToOneField(CourseUserGroup, unique=True, related_name='cohort', on_delete=models.CASCADE) @@ -231,6 +242,10 @@ class CourseCohort(models.Model): class UnregisteredLearnerCohortAssignments(DeletableByUserValue, models.Model): """ Tracks the assignment of an unregistered learner to a course's cohort. + + .. pii: The email field stores PII. + .. pii_types: email_address + .. pii_retirement: local_api """ # pylint: disable=model-missing-unicode class Meta(object): diff --git a/openedx/core/djangoapps/crawlers/models.py b/openedx/core/djangoapps/crawlers/models.py index b7c850b40e..56719c731a 100644 --- a/openedx/core/djangoapps/crawlers/models.py +++ b/openedx/core/djangoapps/crawlers/models.py @@ -8,7 +8,11 @@ from django.db import models class CrawlersConfig(ConfigurationModel): - """Configuration for the crawlers django app.""" + """ + Configuration for the crawlers django app. + + .. no_pii: + """ class Meta(object): app_label = "crawlers" diff --git a/openedx/core/djangoapps/credentials/models.py b/openedx/core/djangoapps/credentials/models.py index f872e49ccf..0d4a7a77ea 100644 --- a/openedx/core/djangoapps/credentials/models.py +++ b/openedx/core/djangoapps/credentials/models.py @@ -18,6 +18,8 @@ class CredentialsApiConfig(ConfigurationModel): """ Manages configuration for connecting to the Credential service and using its API. + + .. no_pii: """ class Meta(object): @@ -113,6 +115,8 @@ class CredentialsApiConfig(ConfigurationModel): class NotifyCredentialsConfig(ConfigurationModel): """ Manages configuration for a run of the notify_credentials management command. + + .. no_pii: """ class Meta(object): diff --git a/openedx/core/djangoapps/credit/models.py b/openedx/core/djangoapps/credit/models.py index fff0159774..11693fe9cd 100644 --- a/openedx/core/djangoapps/credit/models.py +++ b/openedx/core/djangoapps/credit/models.py @@ -38,6 +38,8 @@ class CreditProvider(TimeStampedModel): includes a `url` where the student will be sent when he/she will try to get credit for course. Eligibility duration will be use to set duration for which credit eligible message appears on dashboard. + + .. no_pii: """ provider_id = models.CharField( max_length=255, @@ -213,6 +215,8 @@ def invalidate_provider_cache(sender, **kwargs): # pylint: disable=unused-argum class CreditCourse(models.Model): """ Model for tracking a credit course. + + .. no_pii: """ course_key = CourseKeyField(max_length=255, db_index=True, unique=True) @@ -282,6 +286,8 @@ class CreditRequirement(TimeStampedModel): The 'display_name' field stores the display name of the requirement. The 'criteria' field dictionary provides additional information, clients may need to determine whether a user has satisfied the requirement. + + .. no_pii: """ course = models.ForeignKey(CreditCourse, related_name="credit_requirements", on_delete=models.CASCADE) @@ -418,6 +424,7 @@ class CreditRequirementStatus(TimeStampedModel): In case (3), no CreditRequirementStatus record will exist for the requirement and user. + .. no_pii: """ REQUIREMENT_STATUS_CHOICES = ( @@ -527,14 +534,20 @@ class CreditRequirementStatus(TimeStampedModel): def default_deadline_for_credit_eligibility(): - """ The default deadline to use when creating a new CreditEligibility model. """ + """ + The default deadline to use when creating a new CreditEligibility model. + """ return datetime.datetime.now(pytz.UTC) + datetime.timedelta( days=getattr(settings, "CREDIT_ELIGIBILITY_EXPIRATION_DAYS", 365) ) class CreditEligibility(TimeStampedModel): - """ A record of a user's eligibility for credit for a specific course. """ + """ + A record of a user's eligibility for credit for a specific course. + + .. no_pii: + """ username = models.CharField(max_length=255, db_index=True) course = models.ForeignKey(CreditCourse, related_name="eligibilities", on_delete=models.CASCADE) @@ -645,6 +658,8 @@ class CreditRequest(TimeStampedModel): at the time the request is made. If the user re-issues the request (perhaps because the user did not finish filling in forms on the credit provider's site), the request record will be updated, but the UUID will remain the same. + + .. no_pii: """ uuid = models.CharField(max_length=32, unique=True, db_index=True) @@ -760,7 +775,11 @@ class CreditRequest(TimeStampedModel): class CreditConfig(ConfigurationModel): - """ Manage credit configuration """ + """ + Manage credit configuration + + .. no_pii: + """ CACHE_KEY = 'credit.providers.api.data' cache_ttl = models.PositiveIntegerField( diff --git a/openedx/core/djangoapps/dark_lang/models.py b/openedx/core/djangoapps/dark_lang/models.py index fa91bd6208..0492ae8069 100644 --- a/openedx/core/djangoapps/dark_lang/models.py +++ b/openedx/core/djangoapps/dark_lang/models.py @@ -8,6 +8,8 @@ from django.db import models class DarkLangConfig(ConfigurationModel): """ Configuration for the dark_lang django app. + + .. no_pii: """ released_languages = models.TextField( blank=True, diff --git a/openedx/core/djangoapps/embargo/models.py b/openedx/core/djangoapps/embargo/models.py index 365f0e0b48..daa5b09f6f 100644 --- a/openedx/core/djangoapps/embargo/models.py +++ b/openedx/core/djangoapps/embargo/models.py @@ -40,6 +40,8 @@ class EmbargoedCourse(models.Model): Enable course embargo on a course-by-course basis. Deprecated by `RestrictedCourse` + + .. no_pii: """ objects = NoneToEmptyManager() @@ -74,6 +76,8 @@ class EmbargoedState(ConfigurationModel): Register countries to be embargoed. Deprecated by `Country`. + + .. no_pii: """ # The countries to embargo embargoed_countries = models.TextField( @@ -95,7 +99,8 @@ class EmbargoedState(ConfigurationModel): class RestrictedCourse(models.Model): - """Course with access restrictions. + """ + Course with access restrictions. Restricted courses can block users at two points: @@ -110,6 +115,7 @@ class RestrictedCourse(models.Model): messages to users when they are blocked. These displayed on pages served by the embargo app. + .. no_pii: """ COURSE_LIST_CACHE_KEY = 'embargo.restricted_courses' MESSAGE_URL_CACHE_KEY = 'embargo.message_url_path.{access_point}.{course_key}' @@ -370,6 +376,7 @@ class Country(models.Model): There is a data migration that creates entries for each country code. + .. no_pii: """ country = CountryField( db_index=True, unique=True, @@ -403,6 +410,7 @@ class CountryAccessRule(models.Model): 2) From the initial list, remove all blacklisted countries for the course. + .. no_pii: """ WHITELIST_RULE = 'whitelist' @@ -579,7 +587,11 @@ post_delete.connect(invalidate_country_rule_cache, sender=RestrictedCourse) class CourseAccessRuleHistory(models.Model): - """History of course access rule changes. """ + """ + History of course access rule changes. + + .. no_pii: + """ # pylint: disable=model-missing-unicode timestamp = models.DateTimeField(db_index=True, auto_now_add=True) @@ -667,6 +679,8 @@ post_delete.connect(CourseAccessRuleHistory.snapshot_post_delete_receiver, sende class IPFilter(ConfigurationModel): """ Register specific IP addresses to explicitly block or unblock. + + .. no_pii: """ whitelist = models.TextField( blank=True, diff --git a/openedx/core/djangoapps/external_auth/models.py b/openedx/core/djangoapps/external_auth/models.py index c25529240c..aabbf92b0c 100644 --- a/openedx/core/djangoapps/external_auth/models.py +++ b/openedx/core/djangoapps/external_auth/models.py @@ -16,6 +16,10 @@ from django.db import models class ExternalAuthMap(models.Model): """ Model class for external auth. + + .. pii: Contains PII used in mapping external auth. Unused and empty on edx.org. + .. pii_types: name, email_address, password, external_service + .. pii_retirement: retained """ class Meta(object): app_label = "external_auth" diff --git a/openedx/core/djangoapps/oauth_dispatch/models.py b/openedx/core/djangoapps/oauth_dispatch/models.py index 0c4285e3da..9f42502aea 100644 --- a/openedx/core/djangoapps/oauth_dispatch/models.py +++ b/openedx/core/djangoapps/oauth_dispatch/models.py @@ -12,6 +12,7 @@ from organizations.models import Organization from pytz import utc from openedx.core.djangoapps.oauth_dispatch.toggles import ENFORCE_JWT_SCOPES +from openedx.core.djangolib.markup import HTML from openedx.core.lib.request_utils import get_request_or_stub @@ -22,6 +23,8 @@ class RestrictedApplication(models.Model): A restricted Application will only get expired token/JWT payloads so that they cannot be used to call into APIs. + + .. no_pii: """ application = models.ForeignKey(oauth2_settings.APPLICATION_MODEL, null=False, on_delete=models.CASCADE) @@ -33,8 +36,8 @@ class RestrictedApplication(models.Model): """ Return a unicode representation of this object """ - return u"".format( - name=self.application.name + return HTML(u"").format( + name=HTML(self.application.name) ) @classmethod @@ -56,6 +59,8 @@ class RestrictedApplication(models.Model): class ApplicationAccess(models.Model): """ Specifies access control information for the associated Application. + + .. no_pii: """ application = models.OneToOneField(oauth2_settings.APPLICATION_MODEL, related_name='access') @@ -89,6 +94,8 @@ class ApplicationOrganization(models.Model): See openedx/core/djangoapps/oauth_dispatch/docs/decisions/0007-include-organizations-in-tokens.rst for the intended use of this model. + + .. no_pii: """ RELATION_TYPE_CONTENT_ORG = 'content_org' RELATION_TYPES = ( diff --git a/openedx/core/djangoapps/programs/models.py b/openedx/core/djangoapps/programs/models.py index a5dffd1cce..89b2dd77c8 100644 --- a/openedx/core/djangoapps/programs/models.py +++ b/openedx/core/djangoapps/programs/models.py @@ -10,6 +10,8 @@ class ProgramsApiConfig(ConfigurationModel): This model no longer fronts an API, but now sets a few config-related values for the idea of programs in general. A rename to ProgramsConfig would be more accurate, but costly in terms of developer time. + + .. no_pii: """ class Meta(object): app_label = "programs" diff --git a/openedx/core/djangoapps/schedules/models.py b/openedx/core/djangoapps/schedules/models.py index 1659ab1464..1b28c68167 100644 --- a/openedx/core/djangoapps/schedules/models.py +++ b/openedx/core/djangoapps/schedules/models.py @@ -8,6 +8,10 @@ from config_models.models import ConfigurationModel class Schedule(TimeStampedModel): + """ + .. no_pii: + """ + enrollment = models.OneToOneField('student.CourseEnrollment', null=False, on_delete=models.CASCADE) active = models.BooleanField( default=True, @@ -36,6 +40,9 @@ class Schedule(TimeStampedModel): class ScheduleConfig(ConfigurationModel): + """ + .. no_pii: + """ KEY_FIELDS = ('site',) site = models.ForeignKey(Site, on_delete=models.CASCADE) @@ -50,6 +57,9 @@ class ScheduleConfig(ConfigurationModel): class ScheduleExperience(models.Model): + """ + .. no_pii: + """ EXPERIENCES = Choices( (0, 'default', 'Recurring Nudge and Upgrade Reminder'), (1, 'course_updates', 'Course Updates') diff --git a/openedx/core/djangoapps/self_paced/models.py b/openedx/core/djangoapps/self_paced/models.py index e01f8be8b2..1f7bf706d9 100644 --- a/openedx/core/djangoapps/self_paced/models.py +++ b/openedx/core/djangoapps/self_paced/models.py @@ -10,6 +10,8 @@ from django.utils.translation import ugettext_lazy as _ class SelfPacedConfiguration(ConfigurationModel): """ Configuration for self-paced courses. + + .. no_pii: """ enable_course_home_improvements = BooleanField( diff --git a/openedx/core/djangoapps/site_configuration/models.py b/openedx/core/djangoapps/site_configuration/models.py index 8f1ed67b69..8cc3fc0174 100644 --- a/openedx/core/djangoapps/site_configuration/models.py +++ b/openedx/core/djangoapps/site_configuration/models.py @@ -22,6 +22,8 @@ class SiteConfiguration(models.Model): Fields: site (OneToOneField): one to one field relating each configuration to a single site values (JSONField): json field to store configurations for a site + + .. no_pii: """ site = models.OneToOneField(Site, related_name='configuration', on_delete=models.CASCADE) enabled = models.BooleanField(default=False, verbose_name="Enabled") @@ -140,6 +142,8 @@ class SiteConfigurationHistory(TimeStampedModel): Fields: site (ForeignKey): foreign-key to django Site values (JSONField): json field to store configurations for a site + + .. no_pii: """ site = models.ForeignKey(Site, related_name='configuration_histories', on_delete=models.CASCADE) enabled = models.BooleanField(default=False, verbose_name="Enabled") diff --git a/openedx/core/djangoapps/theming/models.py b/openedx/core/djangoapps/theming/models.py index d06b9312b2..e77e2860a3 100644 --- a/openedx/core/djangoapps/theming/models.py +++ b/openedx/core/djangoapps/theming/models.py @@ -11,6 +11,8 @@ class SiteTheme(models.Model): `site` field is foreignkey to django Site model `theme_dir_name` contains directory name having Site's theme + + .. no_pii: """ site = models.ForeignKey(Site, related_name='themes', on_delete=models.CASCADE) theme_dir_name = models.CharField(max_length=255) diff --git a/openedx/core/djangoapps/user_api/accounts/views.py b/openedx/core/djangoapps/user_api/accounts/views.py index d2f1a684b2..d3a6a8457a 100644 --- a/openedx/core/djangoapps/user_api/accounts/views.py +++ b/openedx/core/djangoapps/user_api/accounts/views.py @@ -398,6 +398,7 @@ class DeactivateLogoutView(APIView): if verify_user_password_response.status_code != status.HTTP_204_NO_CONTENT: return verify_user_password_response with transaction.atomic(): + # Add user to retirement queue. UserRetirementStatus.create_retirement(request.user) # Unlink LMS social auth accounts UserSocialAuth.objects.filter(user_id=request.user.id).delete() @@ -406,10 +407,11 @@ class DeactivateLogoutView(APIView): request.user.email = get_retired_email_by_email(request.user.email) request.user.save() _set_unusable_password(request.user) + # TODO: Unlink social accounts & change password on each IDA. # Remove the activation keys sent by email to the user for account activation. Registration.objects.filter(user=request.user).delete() - # Add user to retirement queue. + # Delete OAuth tokens associated with the user. retire_dop_oauth2_models(request.user) retire_dot_oauth2_models(request.user) diff --git a/openedx/core/djangoapps/user_api/models.py b/openedx/core/djangoapps/user_api/models.py index 1b3f2f2ebf..1fb6e933f0 100644 --- a/openedx/core/djangoapps/user_api/models.py +++ b/openedx/core/djangoapps/user_api/models.py @@ -33,7 +33,11 @@ class RetirementStateError(Exception): class UserPreference(models.Model): - """A user's preference, stored as generic text to be processed by client""" + """ + A user's preference, stored as generic text to be processed by client + + .. no_pii: Stores arbitrary key/value pairs, currently none are PII. If that changes, update this annotation. + """ KEY_REGEX = r"[-_a-zA-Z0-9]+" user = models.ForeignKey(User, db_index=True, related_name="preferences", on_delete=models.CASCADE) key = models.CharField(max_length=255, db_index=True, validators=[RegexValidator(KEY_REGEX)]) @@ -112,6 +116,8 @@ class UserCourseTag(models.Model): """ Per-course user tags, to be used by various things that want to store tags about the user. Added initially to store assignment to experimental groups. + + .. no_pii: Stores arbitrary key/value pairs about users, but does not currently store any PII. This may change! """ user = models.ForeignKey(User, db_index=True, related_name="+", on_delete=models.CASCADE) key = models.CharField(max_length=255, db_index=True) @@ -128,6 +134,9 @@ class UserOrgTag(TimeStampedModel, DeletableByUserValue): # pylint: disable=mod Allows settings to be configured at an organization level. + .. pii: Does not strictly store PII, but maintains the email-optin flag and so is retired in AccountRetirementView. + .. pii_types: other + .. pii_retirement: local_api """ user = models.ForeignKey(User, db_index=True, related_name="+", on_delete=models.CASCADE) key = models.CharField(max_length=255, db_index=True) @@ -142,6 +151,8 @@ class RetirementState(models.Model): """ Stores the list and ordering of the steps of retirement, this should almost never change as updating it can break the retirement process of users already in the queue. + + .. no_pii: """ state_name = models.CharField(max_length=30, unique=True) state_execution_order = models.SmallIntegerField(unique=True) @@ -174,6 +185,10 @@ class UserRetirementPartnerReportingStatus(TimeStampedModel): and asynchronous, timeline than LMS retirement and only impacts a subset of learners so it maintains a queue. This queue is populated as part of the LMS retirement process. + + .. pii: Contains a retiring user's name, username, and email. Retired in AccountRetirementPartnerReportView. + .. pii_types: name, username, email_address + .. pii_retirement: local_api """ user = models.OneToOneField(User) original_username = models.CharField(max_length=150, db_index=True) @@ -197,6 +212,8 @@ class UserRetirementRequest(TimeStampedModel): Records and perists every user retirement request. Users that have requested to cancel their retirement before retirement begins can be removed. All other retired users persist in this table forever. + + .. no_pii: """ user = models.OneToOneField(User, on_delete=models.CASCADE) @@ -227,6 +244,10 @@ class UserRetirementRequest(TimeStampedModel): class UserRetirementStatus(TimeStampedModel): """ Tracks the progress of a user's retirement request + + .. pii: Contains a retiring user's name, username, and email. Retired in AccountRetirementStatusView.cleanup(). + .. pii_types: name, username, email_address + .. pii_retirement: local_api """ user = models.OneToOneField(User, on_delete=models.CASCADE) original_username = models.CharField(max_length=150, db_index=True) diff --git a/openedx/core/djangoapps/user_authn/views/login.py b/openedx/core/djangoapps/user_authn/views/login.py index e49bf0510e..2fefe772b1 100644 --- a/openedx/core/djangoapps/user_authn/views/login.py +++ b/openedx/core/djangoapps/user_authn/views/login.py @@ -260,6 +260,9 @@ def _track_user_login(user, request): """ Sends a tracking event for a successful login. """ + # .. pii: Username and email are sent to Segment here. Retired directly through Segment API call in Tubular. + # .. pii_types: email_address, username + # .. pii_retirement: third_party segment.identify( user.id, { diff --git a/openedx/core/djangoapps/user_authn/views/register.py b/openedx/core/djangoapps/user_authn/views/register.py index 79062ebc9a..f169d06496 100644 --- a/openedx/core/djangoapps/user_authn/views/register.py +++ b/openedx/core/djangoapps/user_authn/views/register.py @@ -335,6 +335,9 @@ def _track_user_registration(user, profile, params, third_party_provider): } }) + # .. pii: Many pieces of PII are sent to Segment here. Retired directly through Segment API call in Tubular. + # .. pii_types: email_address, username, name, birth_date, location, gender + # .. pii_retirement: third_party segment.identify(*identity_args) segment.track( user.id, diff --git a/openedx/core/djangoapps/verified_track_content/models.py b/openedx/core/djangoapps/verified_track_content/models.py index 0734bc13ed..a8cfbc59a8 100644 --- a/openedx/core/djangoapps/verified_track_content/models.py +++ b/openedx/core/djangoapps/verified_track_content/models.py @@ -92,6 +92,8 @@ def pre_save_callback(sender, instance, **kwargs): # pylint: disable=unused-arg class VerifiedTrackCohortedCourse(models.Model): """ Tracks which courses have verified track auto-cohorting enabled. + + .. no_pii: """ course_key = CourseKeyField( max_length=255, db_index=True, unique=True, @@ -154,6 +156,8 @@ def invalidate_verified_track_cache(sender, **kwargs): # pylint: disable=unuse class MigrateVerifiedTrackCohortsSetting(ConfigurationModel): """ Configuration for the swap_from_auto_track_cohorts management command. + + .. no_pii: """ class Meta(object): app_label = "verified_track_content" diff --git a/openedx/core/djangoapps/video_config/models.py b/openedx/core/djangoapps/video_config/models.py index 66b90aafec..11f8ccb25d 100644 --- a/openedx/core/djangoapps/video_config/models.py +++ b/openedx/core/djangoapps/video_config/models.py @@ -17,6 +17,8 @@ class HLSPlaybackEnabledFlag(ConfigurationModel): When this feature flag is set to true, individual courses must also have HLS Playback enabled for this feature to take effect. + + .. no_pii: """ # this field overrides course-specific settings enabled_for_all_courses = BooleanField(default=False) @@ -56,6 +58,8 @@ class CourseHLSPlaybackEnabledFlag(ConfigurationModel): """ Enables HLS Playback for a specific course. Global feature must be enabled for this to take effect. + + .. no_pii: """ KEY_FIELDS = ('course_id',) @@ -80,6 +84,8 @@ class VideoTranscriptEnabledFlag(ConfigurationModel): take effect. When this feature is enabled, 3rd party transcript integration functionality would be available accross all courses or some specific courses and S3 video transcript would be served (currently as a fallback). + + .. no_pii: """ # this field overrides course-specific settings enabled_for_all_courses = BooleanField(default=False) @@ -121,6 +127,8 @@ class CourseVideoTranscriptEnabledFlag(ConfigurationModel): enabled for this to take effect. When this feature is enabled, 3rd party transcript integration functionality would be available for the specific course and S3 video transcript would be served (currently as a fallback). + + .. no_pii: """ KEY_FIELDS = ('course_id',) @@ -140,6 +148,8 @@ class CourseVideoTranscriptEnabledFlag(ConfigurationModel): class TranscriptMigrationSetting(ConfigurationModel): """ Arguments for the Transcript Migration management command + + .. no_pii: """ def __unicode__(self): return ( @@ -181,6 +191,8 @@ class TranscriptMigrationSetting(ConfigurationModel): class MigrationEnqueuedCourse(TimeStampedModel): """ Temporary model to persist the course IDs who has been enqueued for transcripts migration to S3. + + .. no_pii: """ course_id = CourseKeyField(db_index=True, primary_key=True, max_length=255) command_run = PositiveIntegerField(default=0) @@ -194,6 +206,8 @@ class MigrationEnqueuedCourse(TimeStampedModel): class VideoThumbnailSetting(ConfigurationModel): """ Arguments for the Video Thumbnail management command + + .. no_pii: """ command_run = PositiveIntegerField(default=0) offset = PositiveIntegerField(default=0) @@ -234,6 +248,8 @@ class VideoThumbnailSetting(ConfigurationModel): class UpdatedCourseVideos(TimeStampedModel): """ Temporary model to persist the course videos which have been enqueued to update video thumbnails. + + .. no_pii: """ course_id = CourseKeyField(db_index=True, max_length=255) edx_video_id = models.CharField(max_length=100) diff --git a/openedx/core/djangoapps/video_pipeline/models.py b/openedx/core/djangoapps/video_pipeline/models.py index 9d35fe26bf..0572ad5491 100644 --- a/openedx/core/djangoapps/video_pipeline/models.py +++ b/openedx/core/djangoapps/video_pipeline/models.py @@ -11,6 +11,8 @@ from opaque_keys.edx.django.models import CourseKeyField class VideoPipelineIntegration(ConfigurationModel): """ Manages configuration for connecting to the edx-video-pipeline service and using its API. + + .. no_pii: """ client_name = models.CharField( max_length=100, @@ -43,6 +45,8 @@ class VideoPipelineIntegration(ConfigurationModel): class VideoUploadsEnabledByDefault(ConfigurationModel): """ Enables video uploads enabled By default feature across the platform. + + .. no_pii: """ # this field overrides course-specific settings enabled_for_all_courses = models.BooleanField(default=False) @@ -83,6 +87,8 @@ class CourseVideoUploadsEnabledByDefault(ConfigurationModel): """ Enables video uploads enabled by default feature for a specific course. Its global feature must be enabled for this to take effect. + + .. no_pii: """ KEY_FIELDS = ('course_id',) diff --git a/openedx/core/djangoapps/waffle_utils/models.py b/openedx/core/djangoapps/waffle_utils/models.py index 11a2d988a6..792b139d54 100644 --- a/openedx/core/djangoapps/waffle_utils/models.py +++ b/openedx/core/djangoapps/waffle_utils/models.py @@ -14,6 +14,8 @@ from openedx.core.lib.cache_utils import request_cached class WaffleFlagCourseOverrideModel(ConfigurationModel): """ Used to force a waffle flag on or off for a course. + + .. no_pii: """ OVERRIDE_CHOICES = Choices(('on', _('Force On')), ('off', _('Force Off'))) ALL_CHOICES = OVERRIDE_CHOICES + Choices('unset') diff --git a/openedx/features/content_type_gating/models.py b/openedx/features/content_type_gating/models.py index a3a3075827..f410294f0f 100644 --- a/openedx/features/content_type_gating/models.py +++ b/openedx/features/content_type_gating/models.py @@ -34,6 +34,8 @@ from xmodule.partitions.partitions import ENROLLMENT_TRACK_PARTITION_ID class ContentTypeGatingConfig(StackedConfigurationModel): """ A ConfigurationModel used to manage configuration for Content Type Gating (Feature Based Enrollments). + + .. no_pii: """ STACKABLE_FIELDS = ('enabled', 'enabled_as_of', 'studio_override_enabled') diff --git a/openedx/features/course_duration_limits/models.py b/openedx/features/course_duration_limits/models.py index 9a74ba4a02..a1e4eaadce 100644 --- a/openedx/features/course_duration_limits/models.py +++ b/openedx/features/course_duration_limits/models.py @@ -34,6 +34,8 @@ from xmodule.partitions.partitions import ENROLLMENT_TRACK_PARTITION_ID class CourseDurationLimitConfig(StackedConfigurationModel): """ Configuration to manage the Course Duration Limit facility. + + .. no_pii: """ STACKABLE_FIELDS = ('enabled', 'enabled_as_of') diff --git a/pavelib/quality.py b/pavelib/quality.py index 3ab6531bce..650cbde7d4 100644 --- a/pavelib/quality.py +++ b/pavelib/quality.py @@ -756,6 +756,31 @@ def _get_xsscommitlint_count(filename): return None +@task +@needs('pavelib.prereqs.install_python_prereqs') +@timed +def run_pii_check(options): # pylint: disable=unused-argument + """ + Guarantee that all Django models are PII-annotated. + """ + for env_name, env_settings_file in (("CMS", "cms.envs.test"), ("LMS", "lms.envs.test")): + try: + print() + print("Running {} PII Annotation check and report".format(env_name)) + print("-" * 45) + sh( + "export DJANGO_SETTINGS_MODULE={}; " + "code_annotations django_find_annotations " + "--config_file .pii_annotations.yml --report_path pii_report/ " + "--lint --report --coverage".format(env_settings_file) + ) + + except BuildFailure as error_message: + fail_quality('pii_check', 'FAILURE: {}'.format(error_message)) + + return True + + @task @needs('pavelib.prereqs.install_python_prereqs') @cmdopts([ diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 4e1d6062e8..71684a16f5 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -44,7 +44,6 @@ aniso8601==4.1.0 # via tincan anyjson==0.3.3 # via kombu appdirs==1.4.3 # via fs argh==0.26.2 -argparse==1.4.0 asn1crypto==0.24.0 attrs==17.4.0 babel==1.3 @@ -228,7 +227,7 @@ sorl-thumbnail==12.3 sortedcontainers==0.9.2 soupsieve==1.8 # via beautifulsoup4 sqlparse==0.2.4 -stevedore==1.10.0 +stevedore==1.30.0 sympy==0.7.1 tincan==0.0.5 # via edx-enterprise unicodecsv==0.14.1 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index a6b88c2220..0abe0ce330 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -71,6 +71,7 @@ cffi==1.12.1 chardet==3.0.4 click-log==0.1.8 click==7.0 +code-annotations==0.2.3 colorama==0.4.1 configparser==3.7.1 constantly==15.1.0 @@ -321,7 +322,7 @@ sphinx==1.8.4 sphinxcontrib-websupport==1.1.0 # via sphinx splinter==0.9.0 sqlparse==0.2.4 -stevedore==1.10.0 +stevedore==1.30.0 sure==1.4.11 sympy==0.7.1 testfixtures==6.5.2 diff --git a/requirements/edx/paver.in b/requirements/edx/paver.in index 36b1cb4054..72facb63d2 100644 --- a/requirements/edx/paver.in +++ b/requirements/edx/paver.in @@ -21,7 +21,7 @@ psutil==1.2.1 # Library for retrieving information on runn pymongo==2.9.1 # via edx-opaque-keys python-memcached==1.48 # Python interface to the memcached memory cache daemon requests # Simple interface for making HTTP requests -stevedore==1.10.0 # via edx-opaque-keys +stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins watchdog # Used in paver watch_assets wrapt==1.10.5 # Decorator utilities used in the @timed paver task decorator diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index 979a8e2427..aef06f419d 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -5,7 +5,6 @@ # make upgrade # argh==0.26.2 # via watchdog -argparse==1.4.0 # via stevedore certifi==2018.11.29 # via requests chardet==3.0.4 # via requests edx-opaque-keys==0.4.4 @@ -24,7 +23,7 @@ python-memcached==1.48 pyyaml==3.13 # via watchdog requests==2.21.0 six==1.11.0 -stevedore==1.10.0 +stevedore==1.30.0 urllib3==1.23 # via requests watchdog==0.9.0 wrapt==1.10.5 diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index 11fdc7d599..d4ce3d011d 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -21,6 +21,7 @@ beautifulsoup4 # Library for extracting data from HTML and XML files before_after # Syntactic sugar for mock, only used in one test case, not Python 3 compatible bok-choy # Framework for browser automation tests, based on selenium caniusepython3 # Library for checking the ability to upgrade to python3 +code-annotations # Perform code annotation checking, such as for PII annotations cssselect # Used to extract HTML fragments via CSS selectors in 2 test cases and pyquery ddt # Run a test case multiple times with different input; used in many, many of our tests edx-i18n-tools>=0.4.6 # Commands for developers and translators to extract, compile and validate translations diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index a327b11e60..05cc68beca 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -46,7 +46,7 @@ anyjson==0.3.3 apipkg==1.5 # via execnet appdirs==1.4.3 argh==0.26.2 -argparse==1.4.0 +argparse==1.4.0 # via unittest2 asn1crypto==0.24.0 astroid==1.5.3 # via edx-lint, pylint, pylint-celery atomicwrites==1.3.0 # via pytest @@ -69,6 +69,7 @@ cffi==1.12.1 chardet==3.0.4 click-log==0.1.8 # via edx-lint click==7.0 +code-annotations==0.2.3 colorama==0.4.1 # via radon configparser==3.7.1 # via entrypoints, flake8, pylint constantly==15.1.0 # via twisted @@ -308,7 +309,7 @@ sortedcontainers==0.9.2 soupsieve==1.8 splinter==0.9.0 sqlparse==0.2.4 -stevedore==1.10.0 +stevedore==1.30.0 sure==1.4.11 sympy==0.7.1 testfixtures==6.5.2