05b8da5d93
When doing import, we ran into an issue where there was a learner with an external program key that was the same as an existing, completely unrelated edX account username. Rather than try to guess which learner we want or do the lookups backwards, it seemed that splitting these columns to avoid any ambiguity would be the most straightforward and simple approach
564 lines
23 KiB
Python
564 lines
23 KiB
Python
"""
|
||
CSV processing and generation utilities for Teams LMS app.
|
||
"""
|
||
|
||
import csv
|
||
from collections import Counter
|
||
|
||
from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user
|
||
from django.db.models import Prefetch
|
||
|
||
from common.djangoapps.student.models import CourseEnrollment
|
||
from lms.djangoapps.program_enrollments.models import ProgramCourseEnrollment
|
||
from lms.djangoapps.teams.api import (
|
||
ORGANIZATION_PROTECTED_MODES,
|
||
OrganizationProtectionStatus,
|
||
user_organization_protection_status,
|
||
user_protection_status_matches_team
|
||
)
|
||
from lms.djangoapps.teams.models import CourseTeam, CourseTeamMembership
|
||
|
||
from .utils import emit_team_event
|
||
|
||
|
||
def load_team_membership_csv(course, response):
|
||
"""
|
||
Load a CSV detailing course membership.
|
||
|
||
Arguments:
|
||
course (CourseBlock): Course module for which CSV
|
||
download has been requested.
|
||
response (HttpResponse): Django response object to which
|
||
the CSV content will be written.
|
||
"""
|
||
headers = _get_team_membership_csv_headers(course)
|
||
writer = csv.DictWriter(response, fieldnames=headers, extrasaction="ignore")
|
||
writer.writeheader()
|
||
team_membership_data = _lookup_team_membership_data(course)
|
||
writer.writerows(team_membership_data)
|
||
|
||
|
||
def _get_team_membership_csv_headers(course):
|
||
"""
|
||
Get headers for team membership csv.
|
||
['username', 'external_user_id', 'mode', <teamset_id_1>, ..., ,<teamset_id_n>]
|
||
"""
|
||
headers = ['username', 'external_user_id', 'mode']
|
||
for teamset in sorted(course.teams_configuration.teamsets, key=lambda ts: ts.teamset_id):
|
||
headers.append(teamset.teamset_id)
|
||
return headers
|
||
|
||
|
||
def _lookup_team_membership_data(course):
|
||
"""
|
||
Returns a list of dicts, in the following form:
|
||
[
|
||
{
|
||
'username': <edX User username>
|
||
'external_user_id': If the user is enrolled in this course as a part of a program,
|
||
this will be <external_user_id> if the user has one, otherwise, blank.
|
||
'mode': <student enrollment mode for the given course>,
|
||
<teamset id>: <team name> for each teamset in which the given user is on a team
|
||
}
|
||
for student in course
|
||
]
|
||
"""
|
||
# Get course enrollments and team memberships for the given course
|
||
course_enrollments = _fetch_course_enrollments_with_related_models(course.id)
|
||
course_team_memberships = CourseTeamMembership.objects.filter(
|
||
team__course_id=course.id
|
||
).select_related('team', 'user').all()
|
||
teamset_memberships_by_user = _group_teamset_memberships_by_user(course_team_memberships)
|
||
|
||
team_membership_data = []
|
||
for course_enrollment in course_enrollments:
|
||
# This dict contains all the user's team memberships keyed by teamset
|
||
student_row = teamset_memberships_by_user.get(course_enrollment.user, {})
|
||
student_row['username'] = course_enrollment.user.username
|
||
student_row['external_user_id'] = _get_external_user_key(course_enrollment)
|
||
student_row['mode'] = course_enrollment.mode
|
||
team_membership_data.append(student_row)
|
||
return team_membership_data
|
||
|
||
|
||
def _fetch_course_enrollments_with_related_models(course_id):
|
||
"""
|
||
Look up active course enrollments for this course. Fetch the user.
|
||
Fetch the ProgramCourseEnrollment and ProgramEnrollment if any of the CourseEnrollments are associated with
|
||
a program enrollment (so we have access to an external_user_id if it exists).
|
||
Order by the username of the enrolled user.
|
||
|
||
Returns a QuerySet
|
||
"""
|
||
return CourseEnrollment.objects.filter(
|
||
course_id=course_id,
|
||
is_active=True
|
||
).prefetch_related(
|
||
Prefetch(
|
||
'programcourseenrollment_set',
|
||
queryset=ProgramCourseEnrollment.objects.select_related('program_enrollment')
|
||
)
|
||
).select_related(
|
||
'user'
|
||
).order_by('user__username')
|
||
|
||
|
||
def _get_external_user_key(course_enrollment):
|
||
"""
|
||
If a user is enrolled in the course as a part of a program and the program identifies them
|
||
with an external_user_key, return that value for the 'external_user_key' column.
|
||
Otherwise, return None.
|
||
"""
|
||
program_course_enrollments = course_enrollment.programcourseenrollment_set
|
||
if program_course_enrollments.exists():
|
||
# A user should only have one or zero ProgramCourseEnrollments associated with a given CourseEnrollment
|
||
program_course_enrollment = program_course_enrollments.all()[0]
|
||
external_user_key = program_course_enrollment.program_enrollment.external_user_key
|
||
if external_user_key:
|
||
return external_user_key
|
||
return None
|
||
|
||
|
||
def _group_teamset_memberships_by_user(course_team_memberships):
|
||
"""
|
||
Parameters:
|
||
- course_team_memberships: a collection of CourseTeamMemberships.
|
||
|
||
Returns:
|
||
{
|
||
<User>: {
|
||
<teamset_id>: <team_name>
|
||
for CourseTeamMembership in input corresponding to <User>
|
||
}
|
||
per user represented in input
|
||
}
|
||
"""
|
||
teamset_memberships_by_user = {}
|
||
for team_membership in course_team_memberships:
|
||
user = team_membership.user
|
||
if user not in teamset_memberships_by_user:
|
||
teamset_memberships_by_user[user] = {}
|
||
topic_id = team_membership.team.topic_id
|
||
team_name = team_membership.team.name
|
||
teamset_memberships_by_user[user][topic_id] = team_name
|
||
return teamset_memberships_by_user
|
||
|
||
|
||
class TeamMembershipImportManager:
|
||
"""
|
||
A manager class that is responsible the import process of csv file including validation and creation of
|
||
team_courseteam and teams_courseteammembership objects.
|
||
"""
|
||
|
||
def __init__(self, course):
|
||
self.validation_errors = []
|
||
self.teamset_ids = []
|
||
self.course = course
|
||
self.max_errors = 0
|
||
self.existing_course_team_memberships = {}
|
||
self.existing_course_teams = {}
|
||
self.user_count_by_team = Counter()
|
||
self.user_enrollment_by_team = {}
|
||
self.number_of_learners_assigned = 0
|
||
self.user_to_actual_enrollment_mode = {}
|
||
|
||
@property
|
||
def import_succeeded(self):
|
||
"""
|
||
Helper wrapper that tells us the status of the import
|
||
"""
|
||
return not self.validation_errors
|
||
|
||
def set_team_membership_from_csv(self, input_file):
|
||
"""
|
||
Parse an input CSV file and pass to `set_team_memberships` for processing
|
||
"""
|
||
csv_reader = csv.DictReader(line.decode('utf-8-sig').strip() for line in input_file.readlines())
|
||
return self.set_team_memberships(csv_reader)
|
||
|
||
def set_team_memberships(self, csv_reader):
|
||
"""
|
||
Assigns team membership based on the data from an uploaded CSV file.
|
||
Returns true if there were no issues.
|
||
"""
|
||
# File-level validation
|
||
if not self.validate_header(csv_reader):
|
||
return False
|
||
if not self.validate_teamsets(csv_reader):
|
||
return False
|
||
|
||
self.teamset_ids = self.get_teamset_ids_from_reader(csv_reader)
|
||
row_dictionaries = []
|
||
csv_usernames = set()
|
||
|
||
# Get existing team membership data
|
||
self.load_course_team_memberships()
|
||
self.load_course_teams()
|
||
|
||
# process student rows:
|
||
for row in csv_reader:
|
||
if not self.validate_teams_have_matching_teamsets(row):
|
||
return False
|
||
username = row['username']
|
||
if not username:
|
||
continue
|
||
if not self.is_username_unique(username, csv_usernames):
|
||
return False
|
||
csv_usernames.add(username)
|
||
user = self.get_user(username)
|
||
if user is None:
|
||
continue
|
||
if not self.validate_user_enrollment_is_valid(user, row['mode']):
|
||
row['user_model'] = None
|
||
continue
|
||
row['user_model'] = user
|
||
if not self.validate_user_assignment_to_team_and_teamset(row):
|
||
return False
|
||
row_dictionaries.append(row)
|
||
|
||
if not self.validate_team_sizes_not_exceeded():
|
||
return False
|
||
|
||
if not self.validation_errors:
|
||
for row in row_dictionaries:
|
||
self.remove_user_from_team_for_reassignment(row)
|
||
self.add_user_to_team(row)
|
||
self.number_of_learners_assigned = len(row_dictionaries)
|
||
return True
|
||
else:
|
||
return False
|
||
|
||
def load_course_team_memberships(self):
|
||
"""
|
||
Caches existing team memberships by (user_id, teamset_id)
|
||
"""
|
||
for membership in CourseTeamMembership.get_memberships(course_ids=[self.course.id]):
|
||
user_id = membership.user_id
|
||
teamset_id = membership.team.topic_id
|
||
self.existing_course_team_memberships[(user_id, teamset_id)] = membership.team
|
||
|
||
def load_course_teams(self):
|
||
"""
|
||
Caches existing course teams by (team_name, topic_id)
|
||
and existing membership counts by (topic_id, team_name)
|
||
"""
|
||
for team in CourseTeam.objects.filter(course_id=self.course.id).prefetch_related('users'):
|
||
self.existing_course_teams[(team.name, team.topic_id)] = team
|
||
self.user_count_by_team[(team.topic_id, team.name)] = team.users.count()
|
||
|
||
def validate_header(self, csv_reader):
|
||
"""
|
||
Validates header row to ensure that it contains at a minimum columns called 'username', 'mode'.
|
||
Teamset validation is handled separately
|
||
"""
|
||
header = csv_reader.fieldnames
|
||
if 'username' not in header:
|
||
self.validation_errors.append("Header must contain column 'username'.")
|
||
return False
|
||
if 'mode' not in header:
|
||
self.validation_errors.append("Header must contain column 'mode'.")
|
||
return False
|
||
return True
|
||
|
||
def get_teamset_ids_from_reader(self, csv_reader):
|
||
"""
|
||
The teamsets currently will be directly after 'mode'
|
||
"""
|
||
mode_index = csv_reader.fieldnames.index('mode')
|
||
return csv_reader.fieldnames[mode_index + 1:]
|
||
|
||
def validate_teamsets(self, csv_reader):
|
||
"""
|
||
Validates team set ids. Returns true if there are no errors.
|
||
The following conditions result in errors:
|
||
Teamset does not exist
|
||
Teamset id is duplicated
|
||
"""
|
||
teamset_ids = self.get_teamset_ids_from_reader(csv_reader)
|
||
valid_teamset_ids = {ts.teamset_id for ts in self.course.teams_configuration.teamsets}
|
||
|
||
dupe_set = set()
|
||
for teamset_id in teamset_ids:
|
||
if teamset_id in dupe_set:
|
||
self.validation_errors.append("Teamset with id " + teamset_id + " is duplicated.")
|
||
return False
|
||
dupe_set.add(teamset_id)
|
||
if teamset_id not in valid_teamset_ids:
|
||
self.validation_errors.append("Teamset with id " + teamset_id + " does not exist.")
|
||
return False
|
||
return True
|
||
|
||
def validate_user_enrollment_is_valid(self, user, supplied_enrollment):
|
||
"""
|
||
Invalid states:
|
||
user not enrolled in course
|
||
enrollment mode from csv doesn't match actual user enrollment
|
||
"""
|
||
actual_enrollment_mode, user_enrolled = CourseEnrollment.enrollment_mode_for_user(user, self.course.id)
|
||
if not user_enrolled:
|
||
self.validation_errors.append('User ' + user.username + ' is not enrolled in this course.')
|
||
return False
|
||
if actual_enrollment_mode != supplied_enrollment.strip():
|
||
self.validation_errors.append('User ' + user.username + ' enrollment mismatch.')
|
||
return False
|
||
self.user_to_actual_enrollment_mode[user.id] = actual_enrollment_mode
|
||
return True
|
||
|
||
def is_username_unique(self, username, usernames_found_so_far):
|
||
"""
|
||
Ensures that username exists only once in an input file
|
||
"""
|
||
if username in usernames_found_so_far:
|
||
error_message = f'Username {username} listed more than once in file.'
|
||
if self.add_error_and_check_if_max_exceeded(error_message):
|
||
return False
|
||
return True
|
||
|
||
def validate_teams_have_matching_teamsets(self, row):
|
||
"""
|
||
It's possible for a user to create a row that has more team names in it
|
||
than there are teamset ids provided in the header.
|
||
In that case, `row` will have one or more null keys mapping to team names, for example:
|
||
{'teamset-1': 'team-a', 'teamset-2': 'team-beta', None: 'team-37'}
|
||
|
||
This method will add a validation error and return False if this is the case.
|
||
"""
|
||
if None in row:
|
||
error_message = "Team(s) {} don't have matching teamsets.".format(
|
||
row[None]
|
||
)
|
||
if self.add_error_and_check_if_max_exceeded(error_message):
|
||
return False
|
||
return True
|
||
|
||
def validate_user_assignment_to_team_and_teamset(self, row):
|
||
"""
|
||
Validates a user entry relative to an existing team.
|
||
row is a dictionary where key is column name and value is the row value
|
||
[andrew],masters,team1,,team3
|
||
[joe],masters,,team2,team3
|
||
"""
|
||
user = row['user_model']
|
||
for teamset_id in self.teamset_ids:
|
||
# See if the user is already on a team in the teamset
|
||
if (user.id, teamset_id) in self.existing_course_team_memberships:
|
||
current_team_name = self.existing_course_team_memberships[(user.id, teamset_id)].name
|
||
else:
|
||
current_team_name = None
|
||
|
||
team_name = row[teamset_id]
|
||
|
||
# We don't need to do anything if the user isn't moving to a different team
|
||
if current_team_name == team_name:
|
||
continue
|
||
|
||
# If the user is on a team currently, remove them in from the updated count
|
||
if current_team_name is not None:
|
||
self.user_count_by_team[(teamset_id, current_team_name)] -= 1
|
||
|
||
# If we aren't moving them to a new team, we can go to the next team-set
|
||
if not team_name:
|
||
continue
|
||
|
||
# Check that user enrollment mode is compatible for the target team
|
||
if not self.validate_compatible_enrollment_modes(user, team_name, teamset_id):
|
||
return False
|
||
|
||
# Update proposed team counts, initializing the team count if it doesn't exist
|
||
if (teamset_id, team_name) not in self.user_count_by_team:
|
||
self.user_count_by_team[(teamset_id, team_name)] = 1
|
||
else:
|
||
self.user_count_by_team[(teamset_id, team_name)] += 1
|
||
|
||
return True
|
||
|
||
def validate_compatible_enrollment_modes(self, user, team_name, teamset_id):
|
||
"""
|
||
Validates that only students enrolled in a masters track are on a single team. Disallows mixing of masters
|
||
with other enrollment modes on a single team.
|
||
Masters track students can't be added to existing non-protected teams
|
||
"""
|
||
if(teamset_id, team_name) not in self.user_enrollment_by_team:
|
||
self.user_enrollment_by_team[teamset_id, team_name] = set()
|
||
self.user_enrollment_by_team[teamset_id, team_name].add(self.user_to_actual_enrollment_mode[user.id])
|
||
if self.is_FERPA_bubble_breached(teamset_id, team_name) or \
|
||
not self.is_enrollment_protection_for_existing_team_matches_user(user, team_name, teamset_id):
|
||
error_message = \
|
||
f'Team {team_name} cannot have Master’s track users mixed with users in other tracks.'
|
||
self.add_error_and_check_if_max_exceeded(error_message)
|
||
return False
|
||
return True
|
||
|
||
def is_enrollment_protection_for_existing_team_matches_user(self, user, team_name, teamset_id):
|
||
"""
|
||
Applies only to existing teams.
|
||
Returns True if no violations
|
||
False if there is a mismatch
|
||
"""
|
||
try:
|
||
team = self.existing_course_teams[(team_name, teamset_id)]
|
||
return user_protection_status_matches_team(user, team)
|
||
except KeyError:
|
||
return True
|
||
|
||
def is_FERPA_bubble_breached(self, teamset_id, team_name):
|
||
"""
|
||
Ensures that FERPA bubble is not breached.
|
||
Checks that we are not trying to violate FERPA proctection by mixing masters
|
||
track students with other enrollment tracks.
|
||
"""
|
||
|
||
team_enrollment_modes = self.user_enrollment_by_team[teamset_id, team_name]
|
||
protected_modes = set(ORGANIZATION_PROTECTED_MODES)
|
||
|
||
if team_enrollment_modes.isdisjoint(protected_modes):
|
||
return False
|
||
elif team_enrollment_modes.issubset(protected_modes):
|
||
return False
|
||
else:
|
||
return True
|
||
|
||
def validate_team_sizes_not_exceeded(self):
|
||
"""
|
||
Validates that the number of users we want to add to a team won't exceed maximum team size.
|
||
"""
|
||
for teamset_id in self.teamset_ids:
|
||
# Get max size for team-set
|
||
if self.course.teams_configuration.teamsets_by_id[teamset_id].max_team_size is None:
|
||
max_team_size = self.course.teams_configuration.default_max_team_size
|
||
else:
|
||
max_team_size = self.course.teams_configuration.teamsets_by_id[teamset_id].max_team_size
|
||
|
||
# Get teams in team-set
|
||
team_names = [
|
||
teamset_to_team[1] for teamset_to_team in self.user_count_by_team
|
||
if teamset_to_team[0] == teamset_id
|
||
]
|
||
|
||
# Calculate proposed team size and return False if it exceeds capacity
|
||
for team_name in team_names:
|
||
key = (teamset_id, team_name)
|
||
if self.user_count_by_team[key] > max_team_size:
|
||
self.add_error_and_check_if_max_exceeded(
|
||
'New membership for team {} would exceed max size of {}.'.format(
|
||
team_name, max_team_size
|
||
)
|
||
)
|
||
return False
|
||
|
||
return True
|
||
|
||
def remove_user_from_team_for_reassignment(self, row):
|
||
"""
|
||
Remove a user from a team if:
|
||
a. The user's current team is different from the team specified in csv for the same teamset (this user will
|
||
then be assigned to a new team in 'add_user_to_team`.
|
||
b. The team value in the CSV is blank - the user should be removed from the current team in teamset.
|
||
Also, if there is no change in user's membership, the input row's team name will be nulled out so that no
|
||
action will take place further in the processing chain.
|
||
"""
|
||
user = row['user_model']
|
||
for ts_id in self.teamset_ids:
|
||
if row[ts_id] is None:
|
||
# remove this student from the teamset
|
||
try:
|
||
self._remove_user_from_teamset_and_emit_signal(user.id, ts_id, self.course.id)
|
||
except CourseTeamMembership.DoesNotExist:
|
||
pass
|
||
else:
|
||
# reassignment happens only if proposed team membership is different from existing team membership
|
||
if (user.id, ts_id) in self.existing_course_team_memberships:
|
||
current_user_teams_name = self.existing_course_team_memberships[user.id, ts_id].name
|
||
if current_user_teams_name != row[ts_id]:
|
||
try:
|
||
self._remove_user_from_teamset_and_emit_signal(user.id, ts_id, self.course.id)
|
||
del self.existing_course_team_memberships[user.id, ts_id]
|
||
except CourseTeamMembership.DoesNotExist:
|
||
pass
|
||
else:
|
||
# the user will remain in the same team. In order to avoid validation/attempting
|
||
# to readd the user, null out the team name
|
||
row[ts_id] = None
|
||
|
||
def _remove_user_from_teamset_and_emit_signal(self, user_id, ts_id, course_id):
|
||
"""
|
||
If a team membership exists for the specified user, in the specified course and teamset, delete it.
|
||
This removes the user from the team.
|
||
Then, emit an event.
|
||
|
||
If the membership doesn't exist, don't emit the event and instead raise CourseTeamMembership.DoesNotExist
|
||
"""
|
||
membership = CourseTeamMembership.objects.select_related('team').get(
|
||
user_id=user_id,
|
||
team__topic_id=ts_id,
|
||
team__course_id=course_id
|
||
)
|
||
membership.delete()
|
||
emit_team_event(
|
||
'edx.team.learner_removed',
|
||
course_id,
|
||
{
|
||
'team_id': membership.team.team_id,
|
||
'user_id': membership.user_id,
|
||
'remove_method': 'team_csv_import'
|
||
}
|
||
)
|
||
|
||
def add_error_and_check_if_max_exceeded(self, error_message):
|
||
"""
|
||
Adds an error to the error collection.
|
||
:param error_message:
|
||
:return: True if maximum error threshold is exceeded and processing must stop
|
||
False if maximum error threshold is NOT exceeded and processing can continue
|
||
"""
|
||
self.validation_errors.append(error_message)
|
||
return len(self.validation_errors) >= self.max_errors
|
||
|
||
def add_user_to_team(self, user_row):
|
||
"""
|
||
Creates a CourseTeamMembership entry - i.e: a relationship between a user and a team.
|
||
user_row is a dictionary where key is column name and value is the row value.
|
||
{'mode': ' masters','topic_0': '','topic_1': 'team 2','topic_2': None,'user_model': <user_obj>}
|
||
andrew,masters,team1,,team3
|
||
joe,masters,,team2,team3
|
||
"""
|
||
user = user_row['user_model']
|
||
for teamset_id in self.teamset_ids:
|
||
team_name = user_row[teamset_id]
|
||
if not team_name:
|
||
continue
|
||
if (team_name, teamset_id) not in self.existing_course_teams:
|
||
protection_status = user_organization_protection_status(user, self.course.id)
|
||
team = CourseTeam.create(
|
||
name=team_name,
|
||
course_id=self.course.id,
|
||
description='Import from csv',
|
||
topic_id=teamset_id,
|
||
organization_protected=protection_status == OrganizationProtectionStatus.protected
|
||
)
|
||
team.save()
|
||
self.existing_course_teams[(team_name, teamset_id)] = team
|
||
else:
|
||
team = self.existing_course_teams[(team_name, teamset_id)]
|
||
team.add_user(user)
|
||
emit_team_event(
|
||
'edx.team.learner_added',
|
||
team.course_id,
|
||
{
|
||
'team_id': team.team_id,
|
||
'user_id': user.id,
|
||
'add_method': 'team_csv_import'
|
||
}
|
||
)
|
||
|
||
def get_user(self, username):
|
||
"""
|
||
Gets the user object from user_name/email/locator
|
||
user_name: the user_name/email/user locator
|
||
"""
|
||
try:
|
||
return User.objects.get(username=username)
|
||
except User.DoesNotExist:
|
||
self.validation_errors.append('User ' + username + ' does not exist.')
|
||
return None
|