""" CSV processing and generation utilities for Teams LMS app. """ import csv from collections import Counter from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user from django.db.models import Prefetch from common.djangoapps.student.models import CourseEnrollment from lms.djangoapps.program_enrollments.models import ProgramCourseEnrollment, ProgramEnrollment from lms.djangoapps.teams.api import ( ORGANIZATION_PROTECTED_MODES, OrganizationProtectionStatus, user_organization_protection_status, user_protection_status_matches_team ) from lms.djangoapps.teams.models import CourseTeam, CourseTeamMembership from .utils import emit_team_event def load_team_membership_csv(course, response): """ Load a CSV detailing course membership. Arguments: course (CourseBlock): Course module for which CSV download has been requested. response (HttpResponse): Django response object to which the CSV content will be written. """ headers = _get_team_membership_csv_headers(course) writer = csv.DictWriter(response, fieldnames=headers, extrasaction="ignore") writer.writeheader() team_membership_data = _lookup_team_membership_data(course) writer.writerows(team_membership_data) def _get_team_membership_csv_headers(course): """ Get headers for team membership csv. ['user', 'mode', , ..., ,] """ headers = ['user', 'mode'] for teamset in sorted(course.teams_configuration.teamsets, key=lambda ts: ts.teamset_id): headers.append(teamset.teamset_id) return headers def _lookup_team_membership_data(course): """ Returns a list of dicts, in the following form: [ { 'user': If the user is enrolled in this course as a part of a program, this will be if the user has one, otherwise , 'mode': , : for each teamset in which the given user is on a team } for student in course ] """ # Get course enrollments and team memberships for the given course course_enrollments = _fetch_course_enrollments_with_related_models(course.id) course_team_memberships = CourseTeamMembership.objects.filter( team__course_id=course.id ).select_related('team', 'user').all() teamset_memberships_by_user = _group_teamset_memberships_by_user(course_team_memberships) team_membership_data = [] for course_enrollment in course_enrollments: # This dict contains all the user's team memberships keyed by teamset student_row = teamset_memberships_by_user.get(course_enrollment.user, dict()) student_row['user'] = _get_displayed_user_identifier(course_enrollment) student_row['mode'] = course_enrollment.mode team_membership_data.append(student_row) return team_membership_data def _fetch_course_enrollments_with_related_models(course_id): """ Look up active course enrollments for this course. Fetch the user. Fetch the ProgramCourseEnrollment and ProgramEnrollment if any of the CourseEnrollments are associated with a program enrollment (so we have access to an external_user_id if it exists). Order by the username of the enrolled user. Returns a QuerySet """ return CourseEnrollment.objects.filter( course_id=course_id, is_active=True ).prefetch_related( Prefetch( 'programcourseenrollment_set', queryset=ProgramCourseEnrollment.objects.select_related('program_enrollment') ) ).select_related( 'user' ).order_by('user__username') def _get_displayed_user_identifier(course_enrollment): """ If a user is enrolled in the course as a part of a program and the program identifies them with an external_user_key, use that as the value of the 'user' column. Otherwise, use the user's username. """ program_course_enrollments = course_enrollment.programcourseenrollment_set if program_course_enrollments.exists(): # A user should only have one or zero ProgramCourseEnrollments associated with a given CourseEnrollment program_course_enrollment = program_course_enrollments.all()[0] external_user_key = program_course_enrollment.program_enrollment.external_user_key if external_user_key: return external_user_key return course_enrollment.user.username def _group_teamset_memberships_by_user(course_team_memberships): """ Parameters: - course_team_memberships: a collection of CourseTeamMemberships. Returns: { : { : for CourseTeamMembership in input corresponding to } per user represented in input } """ teamset_memberships_by_user = dict() for team_membership in course_team_memberships: user = team_membership.user if user not in teamset_memberships_by_user: teamset_memberships_by_user[user] = dict() topic_id = team_membership.team.topic_id team_name = team_membership.team.name teamset_memberships_by_user[user][topic_id] = team_name return teamset_memberships_by_user class TeamMembershipImportManager: """ A manager class that is responsible the import process of csv file including validation and creation of team_courseteam and teams_courseteammembership objects. """ def __init__(self, course): self.validation_errors = [] self.teamset_ids = [] self.course = course self.max_errors = 0 self.existing_course_team_memberships = {} self.existing_course_teams = {} self.user_count_by_team = Counter() self.user_enrollment_by_team = {} self.number_of_learners_assigned = 0 self.user_to_actual_enrollment_mode = {} @property def import_succeeded(self): """ Helper wrapper that tells us the status of the import """ return not self.validation_errors def set_team_membership_from_csv(self, input_file): """ Parse an input CSV file and pass to `set_team_memberships` for processing """ csv_reader = csv.DictReader(line.decode('utf-8-sig').strip() for line in input_file.readlines()) return self.set_team_memberships(csv_reader) def set_team_memberships(self, csv_reader): """ Assigns team membership based on the data from an uploaded CSV file. Returns true if there were no issues. """ # File-level validation if not self.validate_header(csv_reader): return False if not self.validate_teamsets(csv_reader): return False self.teamset_ids = csv_reader.fieldnames[2:] row_dictionaries = [] csv_usernames = set() # Get existing team membership data self.load_course_team_memberships() self.load_course_teams() # process student rows: for row in csv_reader: if not self.validate_teams_have_matching_teamsets(row): return False username = row['user'] if not username: continue if not self.is_username_unique(username, csv_usernames): return False csv_usernames.add(username) user = self.get_user(username) if user is None: continue if not self.validate_user_enrollment_is_valid(user, row['mode']): row['user'] = None continue row['user'] = user if not self.validate_user_assignment_to_team_and_teamset(row): return False row_dictionaries.append(row) if not self.validate_team_sizes_not_exceeded(): return False if not self.validation_errors: for row in row_dictionaries: self.remove_user_from_team_for_reassignment(row) self.add_user_to_team(row) self.number_of_learners_assigned = len(row_dictionaries) return True else: return False def load_course_team_memberships(self): """ Caches existing team memberships by (user_id, teamset_id) """ for membership in CourseTeamMembership.get_memberships(course_ids=[self.course.id]): user_id = membership.user_id teamset_id = membership.team.topic_id self.existing_course_team_memberships[(user_id, teamset_id)] = membership.team def load_course_teams(self): """ Caches existing course teams by (team_name, topic_id) and existing membership counts by (topic_id, team_name) """ for team in CourseTeam.objects.filter(course_id=self.course.id).prefetch_related('users'): self.existing_course_teams[(team.name, team.topic_id)] = team self.user_count_by_team[(team.topic_id, team.name)] = team.users.count() def validate_header(self, csv_reader): """ Validates header row to ensure that it contains at a minimum columns called 'user', 'mode'. Teamset validation is handled separately """ header = csv_reader.fieldnames if 'user' not in header: self.validation_errors.append("Header must contain column 'user'.") return False if 'mode' not in header: self.validation_errors.append("Header must contain column 'mode'.") return False return True def validate_teamsets(self, csv_reader): """ Validates team set ids. Returns true if there are no errors. The following conditions result in errors: Teamset does not exist Teamset id is duplicated """ teamset_ids = csv_reader.fieldnames[2:] valid_teamset_ids = {ts.teamset_id for ts in self.course.teams_configuration.teamsets} dupe_set = set() for teamset_id in teamset_ids: if teamset_id in dupe_set: self.validation_errors.append("Teamset with id " + teamset_id + " is duplicated.") return False dupe_set.add(teamset_id) if teamset_id not in valid_teamset_ids: self.validation_errors.append("Teamset with id " + teamset_id + " does not exist.") return False return True def validate_user_enrollment_is_valid(self, user, supplied_enrollment): """ Invalid states: user not enrolled in course enrollment mode from csv doesn't match actual user enrollment """ actual_enrollment_mode, user_enrolled = CourseEnrollment.enrollment_mode_for_user(user, self.course.id) if not user_enrolled: self.validation_errors.append('User ' + user.username + ' is not enrolled in this course.') return False if actual_enrollment_mode != supplied_enrollment.strip(): self.validation_errors.append('User ' + user.username + ' enrollment mismatch.') return False self.user_to_actual_enrollment_mode[user.id] = actual_enrollment_mode return True def is_username_unique(self, username, usernames_found_so_far): """ Ensures that username exists only once in an input file """ if username in usernames_found_so_far: error_message = f'Username {username} listed more than once in file.' if self.add_error_and_check_if_max_exceeded(error_message): return False return True def validate_teams_have_matching_teamsets(self, row): """ It's possible for a user to create a row that has more team names in it than there are teamset ids provided in the header. In that case, `row` will have one or more null keys mapping to team names, for example: {'teamset-1': 'team-a', 'teamset-2': 'team-beta', None: 'team-37'} This method will add a validation error and return False if this is the case. """ if None in row: error_message = "Team(s) {} don't have matching teamsets.".format( row[None] ) if self.add_error_and_check_if_max_exceeded(error_message): return False return True def validate_user_assignment_to_team_and_teamset(self, row): """ Validates a user entry relative to an existing team. row is a dictionary where key is column name and value is the row value [andrew],masters,team1,,team3 [joe],masters,,team2,team3 """ user = row['user'] for teamset_id in self.teamset_ids: # See if the user is already on a team in the teamset if (user.id, teamset_id) in self.existing_course_team_memberships: current_team_name = self.existing_course_team_memberships[(user.id, teamset_id)].name else: current_team_name = None team_name = row[teamset_id] # We don't need to do anything if the user isn't moving to a different team if current_team_name == team_name: continue # If the user is on a team currently, remove them in from the updated count if current_team_name is not None: self.user_count_by_team[(teamset_id, current_team_name)] -= 1 # If we aren't moving them to a new team, we can go to the next team-set if not team_name: continue # Check that user enrollment mode is compatible for the target team if not self.validate_compatible_enrollment_modes(user, team_name, teamset_id): return False # Update proposed team counts, initializing the team count if it doesn't exist if (teamset_id, team_name) not in self.user_count_by_team: self.user_count_by_team[(teamset_id, team_name)] = 1 else: self.user_count_by_team[(teamset_id, team_name)] += 1 return True def validate_compatible_enrollment_modes(self, user, team_name, teamset_id): """ Validates that only students enrolled in a masters track are on a single team. Disallows mixing of masters with other enrollment modes on a single team. Masters track students can't be added to existing non-protected teams """ if(teamset_id, team_name) not in self.user_enrollment_by_team: self.user_enrollment_by_team[teamset_id, team_name] = set() self.user_enrollment_by_team[teamset_id, team_name].add(self.user_to_actual_enrollment_mode[user.id]) if self.is_FERPA_bubble_breached(teamset_id, team_name) or \ not self.is_enrollment_protection_for_existing_team_matches_user(user, team_name, teamset_id): error_message = \ f'Team {team_name} cannot have Master’s track users mixed with users in other tracks.' self.add_error_and_check_if_max_exceeded(error_message) return False return True def is_enrollment_protection_for_existing_team_matches_user(self, user, team_name, teamset_id): """ Applies only to existing teams. Returns True if no violations False if there is a mismatch """ try: team = self.existing_course_teams[(team_name, teamset_id)] return user_protection_status_matches_team(user, team) except KeyError: return True def is_FERPA_bubble_breached(self, teamset_id, team_name): """ Ensures that FERPA bubble is not breached. Checks that we are not trying to violate FERPA proctection by mixing masters track students with other enrollment tracks. """ team_enrollment_modes = self.user_enrollment_by_team[teamset_id, team_name] protected_modes = set(ORGANIZATION_PROTECTED_MODES) if team_enrollment_modes.isdisjoint(protected_modes): return False elif team_enrollment_modes.issubset(protected_modes): return False else: return True def validate_team_sizes_not_exceeded(self): """ Validates that the number of users we want to add to a team won't exceed maximum team size. """ for teamset_id in self.teamset_ids: # Get max size for team-set if self.course.teams_configuration.teamsets_by_id[teamset_id].max_team_size is None: max_team_size = self.course.teams_configuration.default_max_team_size else: max_team_size = self.course.teams_configuration.teamsets_by_id[teamset_id].max_team_size # Get teams in team-set team_names = [ teamset_to_team[1] for teamset_to_team in self.user_count_by_team if teamset_to_team[0] == teamset_id ] # Calculate proposed team size and return False if it exceeds capacity for team_name in team_names: key = (teamset_id, team_name) if self.user_count_by_team[key] > max_team_size: self.add_error_and_check_if_max_exceeded( 'New membership for team {} would exceed max size of {}.'.format( team_name, max_team_size ) ) return False return True def remove_user_from_team_for_reassignment(self, row): """ Remove a user from a team if: a. The user's current team is different from the team specified in csv for the same teamset (this user will then be assigned to a new team in 'add_user_to_team`. b. The team value in the CSV is blank - the user should be removed from the current team in teamset. Also, if there is no change in user's membership, the input row's team name will be nulled out so that no action will take place further in the processing chain. """ for ts_id in self.teamset_ids: if row[ts_id] is None: # remove this student from the teamset try: self._remove_user_from_teamset_and_emit_signal(row['user'].id, ts_id, self.course.id) except CourseTeamMembership.DoesNotExist: pass else: # reassignment happens only if proposed team membership is different from existing team membership if (row['user'].id, ts_id) in self.existing_course_team_memberships: current_user_teams_name = self.existing_course_team_memberships[row['user'].id, ts_id].name if current_user_teams_name != row[ts_id]: try: self._remove_user_from_teamset_and_emit_signal(row['user'].id, ts_id, self.course.id) del self.existing_course_team_memberships[row['user'].id, ts_id] except CourseTeamMembership.DoesNotExist: pass else: # the user will remain in the same team. In order to avoid validation/attempting # to readd the user, null out the team name row[ts_id] = None def _remove_user_from_teamset_and_emit_signal(self, user_id, ts_id, course_id): """ If a team membership exists for the specified user, in the specified course and teamset, delete it. This removes the user from the team. Then, emit an event. If the membership doesn't exist, don't emit the event and instead raise CourseTeamMembership.DoesNotExist """ membership = CourseTeamMembership.objects.select_related('team').get( user_id=user_id, team__topic_id=ts_id, team__course_id=course_id ) membership.delete() emit_team_event( 'edx.team.learner_removed', course_id, { 'team_id': membership.team.team_id, 'user_id': membership.user_id, 'remove_method': 'team_csv_import' } ) def add_error_and_check_if_max_exceeded(self, error_message): """ Adds an error to the error collection. :param error_message: :return: True if maximum error threshold is exceeded and processing must stop False if maximum error threshold is NOT exceeded and processing can continue """ self.validation_errors.append(error_message) return len(self.validation_errors) >= self.max_errors def add_user_to_team(self, user_row): """ Creates a CourseTeamMembership entry - i.e: a relationship between a user and a team. user_row is a dictionary where key is column name and value is the row value. {'mode': ' masters','topic_0': '','topic_1': 'team 2','topic_2': None,'user': } andrew,masters,team1,,team3 joe,masters,,team2,team3 """ user = user_row['user'] for teamset_id in self.teamset_ids: team_name = user_row[teamset_id] if not team_name: continue if (team_name, teamset_id) not in self.existing_course_teams: protection_status = user_organization_protection_status(user, self.course.id) team = CourseTeam.create( name=team_name, course_id=self.course.id, description='Import from csv', topic_id=teamset_id, organization_protected=protection_status == OrganizationProtectionStatus.protected ) team.save() self.existing_course_teams[(team_name, teamset_id)] = team else: team = self.existing_course_teams[(team_name, teamset_id)] team.add_user(user) emit_team_event( 'edx.team.learner_added', team.course_id, { 'team_id': team.team_id, 'user_id': user.id, 'add_method': 'team_csv_import' } ) def get_user(self, user_name): """ Gets the user object from user_name/email/locator user_name: the user_name/email/user locator """ try: return User.objects.get(username=user_name) except User.DoesNotExist: try: return User.objects.get(email=user_name) except User.DoesNotExist: try: user = ProgramEnrollment.objects.get(external_user_key=user_name).user if user is None: return None return user except ProgramEnrollment.DoesNotExist: self.validation_errors.append('User name/email/external key: ' + user_name + ' does not exist.') return None