"""
This file contains tasks that are designed to perform background operations on the
running state of a course.

"""


import csv
import logging
from collections import OrderedDict
from contextlib import contextmanager
from datetime import datetime
from tempfile import TemporaryFile
from time import time

from django.contrib.auth.models import User  # lint-amnesty, pylint: disable=imported-auth-user
from django.core.exceptions import ValidationError
from django.core.files.storage import DefaultStorage
from openassessment.data import OraAggregateData, OraDownloadData
from pytz import UTC

from common.djangoapps.student.models import unique_id_for_user, anonymous_id_for_user
from lms.djangoapps.instructor_analytics.basic import get_proctored_exam_results
from lms.djangoapps.instructor_analytics.csvs import format_dictlist
from lms.djangoapps.survey.models import SurveyAnswer
from openedx.core.djangoapps.course_groups.cohorts import add_user_to_cohort
from openedx.core.djangoapps.course_groups.models import CourseUserGroup

from .runner import TaskProgress
from .utils import (
    UPDATE_STATUS_FAILED,
    UPDATE_STATUS_SUCCEEDED,
    upload_csv_to_report_store,
    upload_zip_to_report_store,

)

# define different loggers for use within tasks and on client side
TASK_LOG = logging.getLogger('edx.celery.task')


def upload_course_survey_report(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):
    """
    For a given `course_id`, generate a html report containing the survey results for a course.
    """
    start_time = time()
    start_date = datetime.now(UTC)
    num_reports = 1
    task_progress = TaskProgress(action_name, num_reports, start_time)

    current_step = {'step': 'Gathering course survey report information'}
    task_progress.update_task_state(extra_meta=current_step)

    distinct_survey_fields_queryset = SurveyAnswer.objects.filter(course_key=course_id).values('field_name').distinct()
    survey_fields = []
    for unique_field_row in distinct_survey_fields_queryset:
        survey_fields.append(unique_field_row['field_name'])
    survey_fields.sort()

    user_survey_answers = OrderedDict()
    survey_answers_for_course = SurveyAnswer.objects.filter(course_key=course_id).select_related('user')

    for survey_field_record in survey_answers_for_course:
        user_id = survey_field_record.user.id
        if user_id not in list(user_survey_answers.keys()):
            user_survey_answers[user_id] = {
                'username': survey_field_record.user.username,
                'email': survey_field_record.user.email
            }

        user_survey_answers[user_id][survey_field_record.field_name] = survey_field_record.field_value

    header = ["User ID", "User Name", "Email"]
    header.extend(survey_fields)
    csv_rows = []

    for user_id in user_survey_answers.keys():
        row = []
        row.append(user_id)
        row.append(user_survey_answers[user_id].get('username', ''))
        row.append(user_survey_answers[user_id].get('email', ''))
        for survey_field in survey_fields:
            row.append(user_survey_answers[user_id].get(survey_field, ''))
        csv_rows.append(row)

    task_progress.attempted = task_progress.succeeded = len(csv_rows)
    task_progress.skipped = task_progress.total - task_progress.attempted

    csv_rows.insert(0, header)

    current_step = {'step': 'Uploading CSV'}
    task_progress.update_task_state(extra_meta=current_step)

    # Perform the upload
    upload_csv_to_report_store(csv_rows, 'course_survey_results', course_id, start_date)

    return task_progress.update_task_state(extra_meta=current_step)


def upload_proctored_exam_results_report(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name):
    """
    For a given `course_id`, generate a CSV file containing
    information about proctored exam results, and store using a `ReportStore`.
    """
    start_time = time()
    start_date = datetime.now(UTC)
    num_reports = 1
    task_progress = TaskProgress(action_name, num_reports, start_time)
    current_step = {'step': 'Calculating info about proctored exam results in a course'}
    task_progress.update_task_state(extra_meta=current_step)

    # Compute result table and format it
    query_features = [
        'course_id',
        'provider',
        'track',
        'exam_name',
        'username',
        'email',
        'attempt_code',
        'allowed_time_limit_mins',
        'is_sample_attempt',
        'started_at',
        'completed_at',
        'status',
        'review_status',
        'Suspicious Count',
        'Suspicious Comments',
        'Rules Violation Count',
        'Rules Violation Comments'
    ]

    student_data = get_proctored_exam_results(course_id, query_features)
    header, rows = format_dictlist(student_data, query_features)

    task_progress.attempted = task_progress.succeeded = len(rows)
    task_progress.skipped = task_progress.total - task_progress.attempted

    rows.insert(0, header)

    current_step = {'step': 'Uploading CSV'}
    task_progress.update_task_state(extra_meta=current_step)

    # Perform the upload
    upload_csv_to_report_store(rows, 'proctored_exam_results_report', course_id, start_date)

    return task_progress.update_task_state(extra_meta=current_step)


def _get_csv_file_content(csv_file):
    """
    returns appropriate csv file content based on input and output is
    compatible with python versions
    """
    if not isinstance(csv_file, str):
        content = csv_file.read()
    else:
        content = csv_file

    if isinstance(content, bytes):
        csv_content = content.decode('utf-8')
    else:
        csv_content = content

    return csv_content


def cohort_students_and_upload(_xmodule_instance_args, _entry_id, course_id, task_input, action_name):  # lint-amnesty, pylint: disable=too-many-statements
    """
    Within a given course, cohort students in bulk, then upload the results
    using a `ReportStore`.
    """
    start_time = time()
    start_date = datetime.now(UTC)

    # Iterate through rows to get total assignments for task progress
    with DefaultStorage().open(task_input['file_name']) as f:
        total_assignments = 0
        reader = csv.DictReader(_get_csv_file_content(f).splitlines())

        for _line in reader:
            total_assignments += 1

    task_progress = TaskProgress(action_name, total_assignments, start_time)
    current_step = {'step': 'Cohorting Students'}
    task_progress.update_task_state(extra_meta=current_step)

    # cohorts_status is a mapping from cohort_name to metadata about
    # that cohort.  The metadata will include information about users
    # successfully added to the cohort, users not found, Preassigned
    # users, and a cached reference to the corresponding cohort object
    # to prevent redundant cohort queries.
    cohorts_status = {}

    with DefaultStorage().open(task_input['file_name']) as f:

        reader = csv.DictReader(_get_csv_file_content(f).splitlines())

        for row in reader:
            # Try to use the 'email' field to identify the user.  If it's not present, use 'username'.
            username_or_email = row.get('email') or row.get('username')
            cohort_name = row.get('cohort') or ''
            task_progress.attempted += 1

            if not cohorts_status.get(cohort_name):
                cohorts_status[cohort_name] = {
                    'Cohort Name': cohort_name,
                    'Learners Added': 0,
                    'Learners Not Found': set(),
                    'Invalid Email Addresses': set(),
                    'Preassigned Learners': set()
                }
                try:
                    cohorts_status[cohort_name]['cohort'] = CourseUserGroup.objects.get(
                        course_id=course_id,
                        group_type=CourseUserGroup.COHORT,
                        name=cohort_name
                    )
                    cohorts_status[cohort_name]["Exists"] = True
                except CourseUserGroup.DoesNotExist:
                    cohorts_status[cohort_name]["Exists"] = False

            if not cohorts_status[cohort_name]['Exists']:
                task_progress.failed += 1
                continue

            try:
                # If add_user_to_cohort successfully adds a user, a user object is returned.
                # If a user is preassigned to a cohort, no user object is returned (we already have the email address).
                (user, previous_cohort, preassigned) = add_user_to_cohort(cohorts_status[cohort_name]['cohort'], username_or_email)  # lint-amnesty, pylint: disable=line-too-long, unused-variable
                if preassigned:
                    cohorts_status[cohort_name]['Preassigned Learners'].add(username_or_email)
                    task_progress.preassigned += 1
                else:
                    cohorts_status[cohort_name]['Learners Added'] += 1
                    task_progress.succeeded += 1
            except User.DoesNotExist:
                # Raised when a user with the username could not be found, and the email is not valid
                cohorts_status[cohort_name]['Learners Not Found'].add(username_or_email)
                task_progress.failed += 1
            except ValidationError:
                # Raised when a user with the username could not be found, and the email is not valid,
                # but the entered string contains an "@"
                # Since there is no way to know if the entered string is an invalid username or an invalid email,
                # assume that a string with the "@" symbol in it is an attempt at entering an email
                cohorts_status[cohort_name]['Invalid Email Addresses'].add(username_or_email)
                task_progress.failed += 1
            except ValueError:
                # Raised when the user is already in the given cohort
                task_progress.skipped += 1

            task_progress.update_task_state(extra_meta=current_step)

    current_step['step'] = 'Uploading CSV'
    task_progress.update_task_state(extra_meta=current_step)

    # Filter the output of `add_users_to_cohorts` in order to upload the result.
    output_header = ['Cohort Name', 'Exists', 'Learners Added', 'Learners Not Found', 'Invalid Email Addresses', 'Preassigned Learners']  # lint-amnesty, pylint: disable=line-too-long
    output_rows = [
        [
            ','.join(status_dict.get(column_name, '')) if (column_name == 'Learners Not Found'  # lint-amnesty, pylint: disable=consider-using-in
                                                           or column_name == 'Invalid Email Addresses'
                                                           or column_name == 'Preassigned Learners')
            else status_dict[column_name]
            for column_name in output_header
        ]
        for _cohort_name, status_dict in cohorts_status.items()
    ]
    output_rows.insert(0, output_header)
    upload_csv_to_report_store(output_rows, 'cohort_results', course_id, start_date)

    return task_progress.update_task_state(extra_meta=current_step)


def upload_ora2_data(
        _xmodule_instance_args, _entry_id, course_id, _task_input, action_name
):
    """
    Collect ora2 responses and upload them to S3 as a CSV
    """

    return _upload_ora2_data_common(
        _xmodule_instance_args, _entry_id, course_id, _task_input, action_name,
        'data', OraAggregateData.collect_ora2_data
    )


def upload_ora2_summary(
        _xmodule_instance_args, _entry_id, course_id, _task_input, action_name
):
    """
    Collect ora2/student summaries and upload them to file storage as a CSV
    """

    return _upload_ora2_data_common(
        _xmodule_instance_args, _entry_id, course_id, _task_input, action_name,
        'summary', OraAggregateData.collect_ora2_summary
    )


def _upload_ora2_data_common(
        _xmodule_instance_args, _entry_id, course_id, _task_input, action_name,
        report_name, csv_gen_func
):
    """
    Common code for uploading data or summary csv report.
    """
    start_date = datetime.now(UTC)
    start_time = time()

    num_attempted = 1
    num_total = 1

    fmt = 'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}'
    task_info_string = fmt.format(
        task_id=_xmodule_instance_args.get('task_id') if _xmodule_instance_args is not None else None,
        entry_id=_entry_id,
        course_id=course_id,
        task_input=_task_input
    )
    TASK_LOG.info('%s, Task type: %s, Starting task execution', task_info_string, action_name)

    task_progress = TaskProgress(action_name, num_total, start_time)
    task_progress.attempted = num_attempted

    curr_step = {'step': "Collecting responses"}
    TASK_LOG.info(
        '%s, Task type: %s, Current step: %s for all submissions',
        task_info_string,
        action_name,
        curr_step,
    )

    task_progress.update_task_state(extra_meta=curr_step)

    try:
        header, datarows = csv_gen_func(course_id)
        rows = [header]
        for row in datarows:
            rows.append(row)
    # Update progress to failed regardless of error type
    except Exception:  # pylint: disable=broad-except
        TASK_LOG.exception('Failed to get ORA data.')
        task_progress.failed = 1
        curr_step = {'step': "Error while collecting data"}

        task_progress.update_task_state(extra_meta=curr_step)

        return UPDATE_STATUS_FAILED

    task_progress.succeeded = 1
    curr_step = {'step': "Uploading CSV"}
    TASK_LOG.info(
        '%s, Task type: %s, Current step: %s',
        task_info_string,
        action_name,
        curr_step,
    )
    task_progress.update_task_state(extra_meta=curr_step)

    upload_csv_to_report_store(rows, 'ORA_{}'.format(report_name), course_id, start_date)

    curr_step = {'step': 'Finalizing ORA {} report'.format(report_name)}
    task_progress.update_task_state(extra_meta=curr_step)
    TASK_LOG.info('%s, Task type: %s, Upload complete.', task_info_string, action_name)

    return UPDATE_STATUS_SUCCEEDED


def _task_step(task_progress, task_info_string, action_name):
    """
    Returns a context manager, that logs error and updates TaskProgress
    filures counter in case inner block throws an exception.
    """

    @contextmanager
    def _step_context_manager(step_description, exception_text, step_error_description):
        curr_step = {'step': step_description}
        TASK_LOG.info(
            '%s, Task type: %s, Current step: %s',
            task_info_string,
            action_name,
            curr_step,
        )

        task_progress.update_task_state(extra_meta=curr_step)

        try:
            yield

        # Update progress to failed regardless of error type
        except Exception:  # pylint: disable=broad-except
            TASK_LOG.exception(exception_text)
            task_progress.failed = 1

            task_progress.update_task_state(extra_meta={'step': step_error_description})

    return _step_context_manager


def upload_ora2_submission_files(
    _xmodule_instance_args, _entry_id, course_id, _task_input, action_name
):
    """
    Creates zip archive with submission files in three steps:

    1. Collect all files information using ORA download helper.
    2. Download all submission attachments, put them in temporary zip
        file along with submission texts and csv downloads list.
    3. Upload zip file into reports storage.
    """

    start_time = time()
    start_date = datetime.now(UTC)

    num_attempted = 1
    num_total = 1

    fmt = 'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}'
    task_info_string = fmt.format(
        task_id=_xmodule_instance_args.get('task_id') if _xmodule_instance_args is not None else None,
        entry_id=_entry_id,
        course_id=course_id,
        task_input=_task_input
    )
    TASK_LOG.info('%s, Task type: %s, Starting task execution', task_info_string, action_name)

    task_progress = TaskProgress(action_name, num_total, start_time)
    task_progress.attempted = num_attempted

    step_manager = _task_step(task_progress, task_info_string, action_name)

    submission_files_data = None
    with step_manager(
        'Collecting attachments data',
        'Failed to get ORA submissions attachments data.',
        'Error while collecting data',
    ):
        submission_files_data = OraDownloadData.collect_ora2_submission_files(course_id)

    if submission_files_data is None:
        return UPDATE_STATUS_FAILED

    with TemporaryFile('rb+') as zip_file:
        compressed = None
        with step_manager(
            'Downloading and compressing attachments files',
            'Failed to download and compress submissions attachments.',
            'Error while downloading and compressing submissions attachments',
        ):
            compressed = OraDownloadData.create_zip_with_attachments(zip_file, course_id, submission_files_data)

        if compressed is None:
            return UPDATE_STATUS_FAILED

        zip_filename = None
        with step_manager(
            'Uploading zip file to storage',
            'Failed to upload zip file to storage.',
            'Error while uploading zip file to storage',
        ):
            zip_filename = upload_zip_to_report_store(zip_file, 'submission_files', course_id, start_date),  # lint-amnesty, pylint: disable=trailing-comma-tuple

        if not zip_filename:
            return UPDATE_STATUS_FAILED

    task_progress.succeeded = 1
    curr_step = {'step': 'Finalizing attachments extracting'}
    task_progress.update_task_state(extra_meta=curr_step)
    TASK_LOG.info('%s, Task type: %s, Upload complete.', task_info_string, action_name)

    return UPDATE_STATUS_SUCCEEDED


def generate_anonymous_ids(_xmodule_instance_args, _entry_id, course_id, task_input, action_name):  # lint-amnesty, pylint: disable=too-many-statements
    """
    Generate a 2-column CSV output of user-id, anonymized-user-id
    """
    def _log_and_update_progress(step):
        """
        Updates progress task and logs

        Arguments:
            step: current step task is on
        """
        TASK_LOG.info(
            '%s, Task type: %s, Current step: %s for all learners',
            task_info_string,
            action_name,
            step,
        )
        task_progress.update_task_state(extra_meta=step)
    TASK_LOG.info('ANONYMOUS_IDS_TASK: Starting task execution.')

    task_info_string_format = 'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}'
    task_info_string = task_info_string_format.format(
        task_id=_xmodule_instance_args.get('task_id') if _xmodule_instance_args is not None else None,
        entry_id=_entry_id,
        course_id=course_id,
        task_input=task_input
    )
    TASK_LOG.info('%s, Task type: %s, Starting task execution', task_info_string, action_name)

    start_time = time()
    start_date = datetime.now(UTC)

    students = User.objects.filter(
        courseenrollment__course_id=course_id,
    ).order_by('id')

    task_progress = TaskProgress(action_name, students.count, start_time)
    _log_and_update_progress({'step': "Compiling learner rows"})

    header = ['User ID', 'Anonymized User ID', 'Course Specific Anonymized User ID']
    rows = [[s.id, unique_id_for_user(s), anonymous_id_for_user(s, course_id)]
            for s in students]

    task_progress.attempted = students.count
    _log_and_update_progress({'step': "Finished compiling learner rows"})

    csv_name = 'anonymized_ids'
    upload_csv_to_report_store([header] + rows, csv_name, course_id, start_date)

    return UPDATE_STATUS_SUCCEEDED