Move user retirement scripts code from the tubular repo (#34063)
* refactor: Migragte user retirement scripts code from the tubular repo
This commit is contained in:
Muhammad Farhan Khan
committed by
GitHub
GitHub
parent
20570ff417
commit
65ea55c8aa
522
scripts/user_retirement/utils/edx_api.py
Normal file
522
scripts/user_retirement/utils/edx_api.py
Normal file
@@ -0,0 +1,522 @@
|
||||
"""
|
||||
edX API classes which call edX service REST API endpoints using the edx-rest-api-client module.
|
||||
"""
|
||||
import logging
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import backoff
|
||||
import requests
|
||||
from edx_rest_api_client.auth import SuppliedJwtAuth
|
||||
from edx_rest_api_client.client import REQUEST_CONNECT_TIMEOUT, REQUEST_READ_TIMEOUT
|
||||
from requests.exceptions import ConnectionError, HTTPError, Timeout
|
||||
|
||||
from scripts.user_retirement.utils.exception import HttpDoesNotExistException
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
OAUTH_ACCESS_TOKEN_URL = "/oauth2/access_token"
|
||||
|
||||
|
||||
class EdxGatewayTimeoutError(Exception):
|
||||
"""
|
||||
Exception used to indicate a 504 server error was returned.
|
||||
Differentiates from other 5xx errors.
|
||||
"""
|
||||
|
||||
|
||||
class BaseApiClient:
|
||||
"""
|
||||
API client base class used to submit API requests to a particular web service.
|
||||
"""
|
||||
append_slash = True
|
||||
_access_token = None
|
||||
|
||||
def __init__(self, lms_base_url, api_base_url, client_id, client_secret):
|
||||
"""
|
||||
Retrieves OAuth access token from the LMS and creates REST API client instance.
|
||||
"""
|
||||
self.api_base_url = api_base_url
|
||||
self._access_token = self.get_access_token(lms_base_url, client_id, client_secret)
|
||||
|
||||
def get_api_url(self, path):
|
||||
"""
|
||||
Construct the full API URL using the api_base_url and path.
|
||||
|
||||
Args:
|
||||
path (str): API endpoint path.
|
||||
"""
|
||||
path = path.strip('/')
|
||||
if self.append_slash:
|
||||
path += '/'
|
||||
|
||||
return urljoin(f'{self.api_base_url}/', path)
|
||||
|
||||
def _request(self, method, url, log_404_as_error=True, **kwargs):
|
||||
if 'headers' not in kwargs:
|
||||
kwargs['headers'] = {'Content-type': 'application/json'}
|
||||
|
||||
try:
|
||||
response = requests.request(method, url, auth=SuppliedJwtAuth(self._access_token), **kwargs)
|
||||
response.raise_for_status()
|
||||
|
||||
if response.status_code != 204:
|
||||
return response.json()
|
||||
except HTTPError as exc:
|
||||
status_code = exc.response.status_code
|
||||
|
||||
if status_code == 404 and not log_404_as_error:
|
||||
# Immediately raise the error so that a 404 isn't logged as an API error in this case.
|
||||
raise HttpDoesNotExistException(str(exc))
|
||||
|
||||
LOG.error(f'API Error: {str(exc)} with status code: {status_code}')
|
||||
|
||||
if status_code == 504:
|
||||
# Differentiate gateway errors so different backoff can be used.
|
||||
raise EdxGatewayTimeoutError(str(exc))
|
||||
|
||||
if status_code == 404:
|
||||
raise HttpDoesNotExistException(str(exc))
|
||||
raise
|
||||
|
||||
except Timeout:
|
||||
LOG.error("The request is timed out.")
|
||||
raise
|
||||
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def get_access_token(oauth_base_url, client_id, client_secret):
|
||||
"""
|
||||
Returns an access token for this site's service user.
|
||||
|
||||
Returns:
|
||||
str: JWT access token
|
||||
"""
|
||||
oauth_access_token_url = urljoin(f'{oauth_base_url}/', OAUTH_ACCESS_TOKEN_URL)
|
||||
data = {
|
||||
'grant_type': 'client_credentials',
|
||||
'client_id': client_id,
|
||||
'client_secret': client_secret,
|
||||
'token_type': 'jwt',
|
||||
}
|
||||
try:
|
||||
response = requests.post(
|
||||
oauth_access_token_url,
|
||||
data=data,
|
||||
headers={
|
||||
'User-Agent': 'scripts.user_retirement',
|
||||
},
|
||||
timeout=(REQUEST_CONNECT_TIMEOUT, REQUEST_READ_TIMEOUT)
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()['access_token']
|
||||
except KeyError as exc:
|
||||
LOG.error(f'Failed to get token. {str(exc)} does not exist.')
|
||||
raise
|
||||
|
||||
except HTTPError as exc:
|
||||
LOG.error(
|
||||
f'API Error: {str(exc)} with status code: {exc.response.status_code} fetching access token: {client_id}'
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
def _backoff_handler(details):
|
||||
"""
|
||||
Simple logging handler for when timeout backoff occurs.
|
||||
"""
|
||||
LOG.info('Trying again in {wait:0.1f} seconds after {tries} tries calling {target}'.format(**details))
|
||||
|
||||
|
||||
def _wait_one_minute():
|
||||
"""
|
||||
Backoff generator that waits for 60 seconds.
|
||||
"""
|
||||
return backoff.constant(interval=60)
|
||||
|
||||
|
||||
def _giveup_on_unexpected_exception(exc):
|
||||
"""
|
||||
Giveup method that gives up backoff upon any unexpected exception.
|
||||
"""
|
||||
keep_retrying = (
|
||||
# Treat a ConnectionError as retryable.
|
||||
isinstance(exc, ConnectionError)
|
||||
# All 5xx status codes are retryable except for 504 Gateway Timeout.
|
||||
or (
|
||||
500 <= exc.response.status_code < 600
|
||||
and exc.response.status_code != 504 # Gateway Timeout
|
||||
)
|
||||
# Status code 104 is unreserved, but we must have added this because we observed retryable 104 responses.
|
||||
or exc.response.status_code == 104
|
||||
)
|
||||
return not keep_retrying
|
||||
|
||||
|
||||
def _retry_lms_api():
|
||||
"""
|
||||
Decorator which enables retries with sane backoff defaults for LMS APIs.
|
||||
"""
|
||||
|
||||
def inner(func): # pylint: disable=missing-docstring
|
||||
func_with_backoff = backoff.on_exception(
|
||||
backoff.expo,
|
||||
(HTTPError, ConnectionError),
|
||||
max_time=600, # 10 minutes
|
||||
giveup=_giveup_on_unexpected_exception,
|
||||
# Wrap the actual _backoff_handler so that we can patch the real one in unit tests. Otherwise, the func
|
||||
# will get decorated on import, embedding this handler as a python object reference, precluding our ability
|
||||
# to patch it in tests.
|
||||
on_backoff=lambda details: _backoff_handler(details) # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
func_with_timeout_backoff = backoff.on_exception(
|
||||
_wait_one_minute,
|
||||
EdxGatewayTimeoutError,
|
||||
max_tries=2,
|
||||
# Wrap the actual _backoff_handler so that we can patch the real one in unit tests. Otherwise, the func
|
||||
# will get decorated on import, embedding this handler as a python object reference, precluding our ability
|
||||
# to patch it in tests.
|
||||
on_backoff=lambda details: _backoff_handler(details) # pylint: disable=unnecessary-lambda
|
||||
)
|
||||
return func_with_backoff(func_with_timeout_backoff(func))
|
||||
|
||||
return inner
|
||||
|
||||
|
||||
class LmsApi(BaseApiClient):
|
||||
"""
|
||||
LMS API client with convenience methods for making API calls.
|
||||
"""
|
||||
|
||||
@_retry_lms_api()
|
||||
def learners_to_retire(self, states_to_request, cool_off_days=7, limit=None):
|
||||
"""
|
||||
Retrieves a list of learners awaiting retirement actions.
|
||||
"""
|
||||
params = {
|
||||
'cool_off_days': cool_off_days,
|
||||
'states': states_to_request
|
||||
}
|
||||
if limit:
|
||||
params['limit'] = limit
|
||||
api_url = self.get_api_url('api/user/v1/accounts/retirement_queue')
|
||||
return self._request('GET', api_url, params=params)
|
||||
|
||||
@_retry_lms_api()
|
||||
def get_learners_by_date_and_status(self, state_to_request, start_date, end_date):
|
||||
"""
|
||||
Retrieves a list of learners in the given retirement state that were
|
||||
created in the retirement queue between the dates given. Date range
|
||||
is inclusive, so to get one day you would set both dates to that day.
|
||||
|
||||
:param state_to_request: String LMS UserRetirementState state name (ex. COMPLETE)
|
||||
:param start_date: Date or Datetime object
|
||||
:param end_date: Date or Datetime
|
||||
"""
|
||||
params = {
|
||||
'start_date': start_date.strftime('%Y-%m-%d'),
|
||||
'end_date': end_date.strftime('%Y-%m-%d'),
|
||||
'state': state_to_request
|
||||
}
|
||||
api_url = self.get_api_url('api/user/v1/accounts/retirements_by_status_and_date')
|
||||
return self._request('GET', api_url, params=params)
|
||||
|
||||
@_retry_lms_api()
|
||||
def get_learner_retirement_state(self, username):
|
||||
"""
|
||||
Retrieves the given learner's retirement state.
|
||||
"""
|
||||
api_url = self.get_api_url(f'api/user/v1/accounts/{username}/retirement_status')
|
||||
return self._request('GET', api_url)
|
||||
|
||||
@_retry_lms_api()
|
||||
def update_learner_retirement_state(self, username, new_state_name, message, force=False):
|
||||
"""
|
||||
Updates the given learner's retirement state to the retirement state name new_string
|
||||
with the additional string information in message (for logging purposes).
|
||||
"""
|
||||
data = {
|
||||
'username': username,
|
||||
'new_state': new_state_name,
|
||||
'response': message
|
||||
}
|
||||
|
||||
if force:
|
||||
data['force'] = True
|
||||
|
||||
api_url = self.get_api_url('api/user/v1/accounts/update_retirement_status')
|
||||
return self._request('PATCH', api_url, json=data)
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_deactivate_logout(self, learner):
|
||||
"""
|
||||
Performs the user deactivation and forced logout step of learner retirement
|
||||
"""
|
||||
data = {'username': learner['original_username']}
|
||||
api_url = self.get_api_url('api/user/v1/accounts/deactivate_logout')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_retire_forum(self, learner):
|
||||
"""
|
||||
Performs the forum retirement step of learner retirement
|
||||
"""
|
||||
# api/discussion/
|
||||
data = {'username': learner['original_username']}
|
||||
try:
|
||||
api_url = self.get_api_url('api/discussion/v1/accounts/retire_forum')
|
||||
return self._request('POST', api_url, json=data)
|
||||
except HttpDoesNotExistException:
|
||||
LOG.info("No information about learner retirement")
|
||||
return True
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_retire_mailings(self, learner):
|
||||
"""
|
||||
Performs the email list retirement step of learner retirement
|
||||
"""
|
||||
data = {'username': learner['original_username']}
|
||||
api_url = self.get_api_url('api/user/v1/accounts/retire_mailings')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_unenroll(self, learner):
|
||||
"""
|
||||
Unenrolls the user from all courses
|
||||
"""
|
||||
data = {'username': learner['original_username']}
|
||||
api_url = self.get_api_url('api/enrollment/v1/unenroll')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
# This endpoint additionally returns 500 when the EdxNotes backend service is unavailable.
|
||||
@_retry_lms_api()
|
||||
def retirement_retire_notes(self, learner):
|
||||
"""
|
||||
Deletes all the user's notes (aka. annotations)
|
||||
"""
|
||||
data = {'username': learner['original_username']}
|
||||
api_url = self.get_api_url('api/edxnotes/v1/retire_user')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_lms_retire_misc(self, learner):
|
||||
"""
|
||||
Deletes, blanks, or one-way hashes personal information in LMS as
|
||||
defined in EDUCATOR-2802 and sub-tasks.
|
||||
"""
|
||||
data = {'username': learner['original_username']}
|
||||
api_url = self.get_api_url('api/user/v1/accounts/retire_misc')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_lms_retire(self, learner):
|
||||
"""
|
||||
Deletes, blanks, or one-way hashes all remaining personal information in LMS
|
||||
"""
|
||||
data = {'username': learner['original_username']}
|
||||
api_url = self.get_api_url('api/user/v1/accounts/retire')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_partner_queue(self, learner):
|
||||
"""
|
||||
Calls LMS to add the given user to the retirement reporting queue
|
||||
"""
|
||||
data = {'username': learner['original_username']}
|
||||
api_url = self.get_api_url('api/user/v1/accounts/retirement_partner_report')
|
||||
return self._request('PUT', api_url, json=data)
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_partner_report(self):
|
||||
"""
|
||||
Retrieves the list of users to create partner reports for and set their status to
|
||||
processing
|
||||
"""
|
||||
api_url = self.get_api_url('api/user/v1/accounts/retirement_partner_report')
|
||||
return self._request('POST', api_url)
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_partner_cleanup(self, usernames):
|
||||
"""
|
||||
Removes the given users from the partner reporting queue
|
||||
"""
|
||||
api_url = self.get_api_url('api/user/v1/accounts/retirement_partner_report_cleanup')
|
||||
return self._request('POST', api_url, json=usernames)
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_retire_proctoring_data(self, learner):
|
||||
"""
|
||||
Deletes or hashes learner data from edx-proctoring
|
||||
"""
|
||||
api_url = self.get_api_url(f"api/edx_proctoring/v1/retire_user/{learner['user']['id']}")
|
||||
return self._request('POST', api_url)
|
||||
|
||||
@_retry_lms_api()
|
||||
def retirement_retire_proctoring_backend_data(self, learner):
|
||||
"""
|
||||
Removes the given learner from 3rd party proctoring backends
|
||||
"""
|
||||
api_url = self.get_api_url(f"api/edx_proctoring/v1/retire_backend_user/{learner['user']['id']}")
|
||||
return self._request('POST', api_url)
|
||||
|
||||
@_retry_lms_api()
|
||||
def bulk_cleanup_retirements(self, usernames):
|
||||
"""
|
||||
Deletes the retirements for all given usernames
|
||||
"""
|
||||
data = {'usernames': usernames}
|
||||
api_url = self.get_api_url('api/user/v1/accounts/retirement_cleanup')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
def replace_lms_usernames(self, username_mappings):
|
||||
"""
|
||||
Calls LMS API to replace usernames.
|
||||
|
||||
Param:
|
||||
username_mappings: list of dicts where key is current username and value is new desired username
|
||||
[{current_un_1: desired_un_1}, {current_un_2: desired_un_2}]
|
||||
"""
|
||||
data = {"username_mappings": username_mappings}
|
||||
api_url = self.get_api_url('api/user/v1/accounts/replace_usernames')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
def replace_forums_usernames(self, username_mappings):
|
||||
"""
|
||||
Calls the discussion forums API inside of LMS to replace usernames.
|
||||
|
||||
Param:
|
||||
username_mappings: list of dicts where key is current username and value is new unique username
|
||||
[{current_un_1: new_un_1}, {current_un_2: new_un_2}]
|
||||
"""
|
||||
data = {"username_mappings": username_mappings}
|
||||
api_url = self.get_api_url('api/discussion/v1/accounts/replace_usernames')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
|
||||
class EcommerceApi(BaseApiClient):
|
||||
"""
|
||||
Ecommerce API client with convenience methods for making API calls.
|
||||
"""
|
||||
|
||||
@_retry_lms_api()
|
||||
def retire_learner(self, learner):
|
||||
"""
|
||||
Performs the learner retirement step for Ecommerce
|
||||
"""
|
||||
data = {'username': learner['original_username']}
|
||||
api_url = self.get_api_url('api/v2/user/retire')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
@_retry_lms_api()
|
||||
def get_tracking_key(self, learner):
|
||||
"""
|
||||
Fetches the ecommerce tracking id used for Segment tracking when
|
||||
ecommerce doesn't have access to the LMS user id.
|
||||
"""
|
||||
api_url = self.get_api_url(f"api/v2/retirement/tracking_id/{learner['original_username']}")
|
||||
return self._request('GET', api_url)['ecommerce_tracking_id']
|
||||
|
||||
def replace_usernames(self, username_mappings):
|
||||
"""
|
||||
Calls the ecommerce API to replace usernames.
|
||||
|
||||
Param:
|
||||
username_mappings: list of dicts where key is current username and value is new unique username
|
||||
[{current_un_1: new_un_1}, {current_un_2: new_un_2}]
|
||||
"""
|
||||
data = {"username_mappings": username_mappings}
|
||||
api_url = self.get_api_url('api/v2/user_management/replace_usernames')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
|
||||
class CredentialsApi(BaseApiClient):
|
||||
"""
|
||||
Credentials API client with convenience methods for making API calls.
|
||||
"""
|
||||
|
||||
@_retry_lms_api()
|
||||
def retire_learner(self, learner):
|
||||
"""
|
||||
Performs the learner retirement step for Credentials
|
||||
"""
|
||||
data = {'username': learner['original_username']}
|
||||
api_url = self.get_api_url('user/retire')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
def replace_usernames(self, username_mappings):
|
||||
"""
|
||||
Calls the credentials API to replace usernames.
|
||||
|
||||
Param:
|
||||
username_mappings: list of dicts where key is current username and value is new unique username
|
||||
[{current_un_1: new_un_1}, {current_un_2: new_un_2}]
|
||||
"""
|
||||
data = {"username_mappings": username_mappings}
|
||||
api_url = self.get_api_url('api/v2/replace_usernames')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
|
||||
class DiscoveryApi(BaseApiClient):
|
||||
"""
|
||||
Discovery API client with convenience methods for making API calls.
|
||||
"""
|
||||
|
||||
def replace_usernames(self, username_mappings):
|
||||
"""
|
||||
Calls the discovery API to replace usernames.
|
||||
|
||||
Param:
|
||||
username_mappings: list of dicts where key is current username and value is new unique username
|
||||
[{current_un_1: new_un_1}, {current_un_2: new_un_2}]
|
||||
"""
|
||||
data = {"username_mappings": username_mappings}
|
||||
api_url = self.get_api_url('api/v1/replace_usernames')
|
||||
return self._request('POST', api_url, json=data)
|
||||
|
||||
|
||||
class DemographicsApi(BaseApiClient):
|
||||
"""
|
||||
Demographics API client.
|
||||
"""
|
||||
|
||||
@_retry_lms_api()
|
||||
def retire_learner(self, learner):
|
||||
"""
|
||||
Performs the learner retirement step for Demographics. Passes the learner's LMS User Id instead of username.
|
||||
"""
|
||||
data = {'lms_user_id': learner['user']['id']}
|
||||
# If the user we are retiring has no data in the Demographics DB the request will return a 404. We
|
||||
# catch the HTTPError and return True in order to prevent this error getting raised and
|
||||
# incorrectly causing the learner to enter an ERROR state during retirement.
|
||||
try:
|
||||
api_url = self.get_api_url('demographics/api/v1/retire_demographics')
|
||||
return self._request('POST', api_url, log_404_as_error=False, json=data)
|
||||
except HttpDoesNotExistException:
|
||||
LOG.info("No demographics data found for user")
|
||||
return True
|
||||
|
||||
|
||||
class LicenseManagerApi(BaseApiClient):
|
||||
"""
|
||||
License Manager API client.
|
||||
"""
|
||||
|
||||
@_retry_lms_api()
|
||||
def retire_learner(self, learner):
|
||||
"""
|
||||
Performs the learner retirement step for License manager. Passes the learner's LMS User Id in addition to
|
||||
username.
|
||||
"""
|
||||
data = {
|
||||
'lms_user_id': learner['user']['id'],
|
||||
'original_username': learner['original_username'],
|
||||
}
|
||||
# If the user we are retiring has no data in the License Manager DB the request will return a 404. We
|
||||
# catch the HTTPError and return True in order to prevent this error getting raised and
|
||||
# incorrectly causing the learner to enter an ERROR state during retirement.
|
||||
try:
|
||||
api_url = self.get_api_url('api/v1/retire_user')
|
||||
return self._request('POST', api_url, log_404_as_error=False, json=data)
|
||||
except HttpDoesNotExistException:
|
||||
LOG.info("No license manager data found for user")
|
||||
return True
|
||||
Reference in New Issue
Block a user