From 79fe2e626046b470ceb2ddedf07d945fdb33ccf7 Mon Sep 17 00:00:00 2001 From: bmedx Date: Fri, 27 Oct 2017 10:23:05 -0400 Subject: [PATCH] CMS management command cleanup for Django 1.11 --- .../management/commands/clean_cert_name.py | 10 ++- .../management/commands/cleanup_assets.py | 12 +-- .../management/commands/clone_course.py | 28 ++++--- .../management/commands/create_course.py | 60 +++++++------- .../management/commands/delete_course.py | 5 +- .../management/commands/delete_orphans.py | 12 +-- .../management/commands/edit_course_tabs.py | 79 +++++++++++-------- .../commands/empty_asset_trashcan.py | 16 ++-- .../management/commands/export.py | 3 +- .../management/commands/export_all_courses.py | 40 +++++----- .../management/commands/export_olx.py | 7 +- .../management/commands/fix_not_found.py | 2 +- .../management/commands/force_publish.py | 18 ++--- .../management/commands/generate_courses.py | 3 +- .../management/commands/git_export.py | 31 +++----- .../management/commands/import.py | 2 +- .../management/commands/migrate_to_split.py | 37 ++++----- .../management/commands/populate_creators.py | 32 ++++---- .../management/commands/reindex_course.py | 46 +++++------ .../management/commands/reindex_library.py | 25 +++--- .../commands/restore_asset_from_trashcan.py | 8 +- .../commands/tests/test_reindex_courses.py | 19 ++--- .../contentstore/management/commands/xlint.py | 29 ++++--- 23 files changed, 267 insertions(+), 257 deletions(-) diff --git a/cms/djangoapps/contentstore/management/commands/clean_cert_name.py b/cms/djangoapps/contentstore/management/commands/clean_cert_name.py index 923e38c633..d291726eda 100644 --- a/cms/djangoapps/contentstore/management/commands/clean_cert_name.py +++ b/cms/djangoapps/contentstore/management/commands/clean_cert_name.py @@ -4,6 +4,8 @@ erroneous certificate names. """ from collections import namedtuple +from six.moves import input +from six import text_type from django.core.management.base import BaseCommand @@ -150,10 +152,10 @@ class Command(BaseCommand): """ headers = ["Course Key", "cert_name_short", "cert_name_short", "Should clean?"] col_widths = [ - max(len(unicode(result[col])) for result in results + [headers]) + max(len(text_type(result[col])) for result in results + [headers]) for col in range(len(results[0])) ] - id_format = "{{:>{}}} |".format(len(unicode(len(results)))) + id_format = "{{:>{}}} |".format(len(text_type(len(results)))) col_format = "| {{:>{}}} |" self.stdout.write(id_format.format(""), ending='') @@ -165,7 +167,7 @@ class Command(BaseCommand): for idx, result in enumerate(results): self.stdout.write(id_format.format(idx), ending='') for col, width in zip(result, col_widths): - self.stdout.write(col_format.format(width).format(unicode(col)), ending='') + self.stdout.write(col_format.format(width).format(text_type(col)), ending='') self.stdout.write("") def _commit(self, results): @@ -191,7 +193,7 @@ class Command(BaseCommand): while True: self._display(results) - command = raw_input("|commit|quit: ").strip() + command = input("|commit|quit: ").strip() if command == 'quit': return diff --git a/cms/djangoapps/contentstore/management/commands/cleanup_assets.py b/cms/djangoapps/contentstore/management/commands/cleanup_assets.py index c3524c51c6..2044fde7a3 100644 --- a/cms/djangoapps/contentstore/management/commands/cleanup_assets.py +++ b/cms/djangoapps/contentstore/management/commands/cleanup_assets.py @@ -24,17 +24,17 @@ class Command(BaseCommand): content_store = contentstore() success = False - log.info(u"-" * 80) - log.info(u"Cleaning up assets for all courses") + log.info("-" * 80) + log.info("Cleaning up assets for all courses") try: # Remove all redundant Mac OS metadata files assets_deleted = content_store.remove_redundant_content_for_courses() success = True except Exception as err: - log.info(u"=" * 30 + u"> failed to cleanup") - log.info(u"Error:") + log.info("=" * 30 + u"> failed to cleanup") + log.info("Error:") log.info(err) if success: - log.info(u"=" * 80) - log.info(u"Total number of assets deleted: {0}".format(assets_deleted)) + log.info("=" * 80) + log.info("Total number of assets deleted: {0}".format(assets_deleted)) diff --git a/cms/djangoapps/contentstore/management/commands/clone_course.py b/cms/djangoapps/contentstore/management/commands/clone_course.py index b8a8c9a752..fb7a39c9ba 100644 --- a/cms/djangoapps/contentstore/management/commands/clone_course.py +++ b/cms/djangoapps/contentstore/management/commands/clone_course.py @@ -1,7 +1,9 @@ """ Script for cloning a course """ -from django.core.management.base import BaseCommand, CommandError +from __future__ import print_function + +from django.core.management.base import BaseCommand from opaque_keys.edx.keys import CourseKey from student.roles import CourseInstructorRole, CourseStaffRole @@ -13,24 +15,30 @@ from xmodule.modulestore.django import modulestore # To run from command line: ./manage.py cms clone_course --settings=dev master/300/cough edx/111/foo # class Command(BaseCommand): - """Clone a MongoDB-backed course to another location""" + """ + Clone a MongoDB-backed course to another location + """ help = 'Clone a MongoDB backed course to another location' - def handle(self, *args, **options): - "Execute the command" - if len(args) != 2: - raise CommandError("clone requires 2 arguments: ") + def add_arguments(self, parser): + parser.add_argument('source_course_id', help='Course ID to copy from') + parser.add_argument('dest_course_id', help='Course ID to copy to') - source_course_id = CourseKey.from_string(args[0]) - dest_course_id = CourseKey.from_string(args[1]) + def handle(self, *args, **options): + """ + Execute the command + """ + + source_course_id = CourseKey.from_string(options['source_course_id']) + dest_course_id = CourseKey.from_string(options['dest_course_id']) mstore = modulestore() - print "Cloning course {0} to {1}".format(source_course_id, dest_course_id) + print("Cloning course {0} to {1}".format(source_course_id, dest_course_id)) with mstore.bulk_operations(dest_course_id): if mstore.clone_course(source_course_id, dest_course_id, ModuleStoreEnum.UserID.mgmt_command): - print "copying User permissions..." + print("copying User permissions...") # purposely avoids auth.add_user b/c it doesn't have a caller to authorize CourseInstructorRole(dest_course_id).add_users( *CourseInstructorRole(source_course_id).users_with_role() diff --git a/cms/djangoapps/contentstore/management/commands/create_course.py b/cms/djangoapps/contentstore/management/commands/create_course.py index 5908990a09..b1ac53e085 100644 --- a/cms/djangoapps/contentstore/management/commands/create_course.py +++ b/cms/djangoapps/contentstore/management/commands/create_course.py @@ -1,6 +1,8 @@ """ Django management command to create a course in a specific modulestore """ +from six import text_type + from django.contrib.auth.models import User from django.core.management.base import BaseCommand, CommandError @@ -9,6 +11,9 @@ from contentstore.views.course import create_new_course_in_store from xmodule.modulestore import ModuleStoreEnum +MODULESTORE_CHOICES = (ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split) + + class Command(BaseCommand): """ Create a course in a specific modulestore. @@ -16,45 +21,36 @@ class Command(BaseCommand): # can this query modulestore for the list of write accessible stores or does that violate command pattern? help = "Create a course in one of {}".format([ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split]) - args = "modulestore user org course run" - def parse_args(self, *args): + def add_arguments(self, parser): + parser.add_argument('modulestore', + choices=MODULESTORE_CHOICES, + help="Modulestore must be one of {}".format(MODULESTORE_CHOICES)) + parser.add_argument('user', + help="The instructor's email address or integer ID.") + parser.add_argument('org', + help="The organization to create the course within.") + parser.add_argument('course', + help="The name of the course.") + parser.add_argument('run', + help="The name of the course run.") + + def parse_args(self, **options): """ Return a tuple of passed in values for (modulestore, user, org, course, run). """ - if len(args) != 5: - raise CommandError( - "create_course requires 5 arguments: " - "a modulestore, user, org, course, run. Modulestore is one of {}".format( - [ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split] - ) - ) - - if args[0] not in [ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split]: - raise CommandError( - "Modulestore (first arg) must be one of {}".format( - [ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split] - ) - ) - storetype = args[0] - try: - user = user_from_str(args[1]) + user = user_from_str(options['user']) except User.DoesNotExist: - raise CommandError( - "No user {user} found: expected args are {args}".format( - user=args[1], - args=self.args, - ), - ) + raise CommandError("No user {user} found.".format(user=options['user'])) - org = args[2] - course = args[3] - run = args[4] - - return storetype, user, org, course, run + return options['modulestore'], user, options['org'], options['course'], options['run'] def handle(self, *args, **options): - storetype, user, org, course, run = self.parse_args(*args) + storetype, user, org, course, run = self.parse_args(**options) + + if storetype == ModuleStoreEnum.Type.mongo: + self.stderr.write("WARNING: The 'Old Mongo' store is deprecated. New courses should be added to split.") + new_course = create_new_course_in_store(storetype, user, org, course, run, {}) - self.stdout.write(u"Created {}".format(unicode(new_course.id))) + self.stdout.write(u"Created {}".format(text_type(new_course.id))) diff --git a/cms/djangoapps/contentstore/management/commands/delete_course.py b/cms/djangoapps/contentstore/management/commands/delete_course.py index c3502d5467..61f99b1dab 100644 --- a/cms/djangoapps/contentstore/management/commands/delete_course.py +++ b/cms/djangoapps/contentstore/management/commands/delete_course.py @@ -1,3 +1,6 @@ +from __future__ import print_function +from six import text_type + from django.core.management.base import BaseCommand, CommandError from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey @@ -54,7 +57,7 @@ class Command(BaseCommand): def handle(self, *args, **options): try: # a course key may have unicode chars in it - course_key = unicode(options['course_key'], 'utf8') + course_key = text_type(options['course_key'], 'utf8') course_key = CourseKey.from_string(course_key) except InvalidKeyError: raise CommandError('Invalid course_key: {}'.format(options['course_key'])) diff --git a/cms/djangoapps/contentstore/management/commands/delete_orphans.py b/cms/djangoapps/contentstore/management/commands/delete_orphans.py index c765cda23a..146f283ab4 100644 --- a/cms/djangoapps/contentstore/management/commands/delete_orphans.py +++ b/cms/djangoapps/contentstore/management/commands/delete_orphans.py @@ -1,4 +1,6 @@ """Script for deleting orphans""" +from __future__ import print_function + from django.core.management.base import BaseCommand, CommandError from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey @@ -26,15 +28,15 @@ class Command(BaseCommand): raise CommandError("Invalid course key.") if options['commit']: - print 'Deleting orphans from the course:' + print('Deleting orphans from the course:') deleted_items = _delete_orphans( course_key, ModuleStoreEnum.UserID.mgmt_command, options['commit'] ) - print "Success! Deleted the following orphans from the course:" - print "\n".join(deleted_items) + print("Success! Deleted the following orphans from the course:") + print("\n".join(deleted_items)) else: - print 'Dry run. The following orphans would have been deleted from the course:' + print('Dry run. The following orphans would have been deleted from the course:') deleted_items = _delete_orphans( course_key, ModuleStoreEnum.UserID.mgmt_command, options['commit'] ) - print "\n".join(deleted_items) + print("\n".join(deleted_items)) diff --git a/cms/djangoapps/contentstore/management/commands/edit_course_tabs.py b/cms/djangoapps/contentstore/management/commands/edit_course_tabs.py index 9d3891f7f7..04cd22fb0b 100644 --- a/cms/djangoapps/contentstore/management/commands/edit_course_tabs.py +++ b/cms/djangoapps/contentstore/management/commands/edit_course_tabs.py @@ -6,7 +6,8 @@ # Run it this way: # ./manage.py cms --settings dev edit_course_tabs --course Stanford/CS99/2013_spring # -from optparse import make_option +from __future__ import print_function + from django.core.management.base import BaseCommand, CommandError from opaque_keys.edx.keys import CourseKey @@ -18,10 +19,16 @@ from .prompt import query_yes_no def print_course(course): "Prints out the course id and a numbered list of tabs." - print course.id - print 'num type name' - for index, item in enumerate(course.tabs): - print index + 1, '"' + item.get('type') + '"', '"' + item.get('name', '') + '"' + try: + print(course.id) + print('num type name') + for index, item in enumerate(course.tabs): + print(index + 1, '"' + item.get('type') + '"', '"' + item.get('name', '') + '"') + # If a course is bad we will get an error descriptor here, dump it and die instead of + # just sending up the error that .id doesn't exist. + except AttributeError: + print(course) + raise # course.tabs looks like this @@ -42,48 +49,50 @@ As a first step, run the command with a courseid like this: This will print the existing tabs types and names. Then run the command again, adding --insert or --delete to edit the list. """ - # Making these option objects separately, so can refer to their .help below - course_option = make_option('--course', - action='store', - dest='course', - default=False, - help='--course required, e.g. Stanford/CS99/2013_spring') - delete_option = make_option('--delete', - action='store_true', - dest='delete', - default=False, - help='--delete ') - insert_option = make_option('--insert', - action='store_true', - dest='insert', - default=False, - help='--insert , e.g. 2 "course_info" "Course Info"') - option_list = BaseCommand.option_list + (course_option, delete_option, insert_option) + course_help = '--course required, e.g. Stanford/CS99/2013_spring' + delete_help = '--delete ' + insert_help = '--insert , e.g. 4 "course_info" "Course Info"' + + def add_arguments(self, parser): + parser.add_argument('--course', + dest='course', + default=False, + required=True, + help=self.course_help) + parser.add_argument('--delete', + dest='delete', + default=False, + nargs=1, + help=self.delete_help) + parser.add_argument('--insert', + dest='insert', + default=False, + nargs=3, + help=self.insert_help, + ) def handle(self, *args, **options): - if not options['course']: - raise CommandError(Command.course_option.help) - course = get_course_by_id(CourseKey.from_string(options['course'])) - print 'Warning: this command directly edits the list of course tabs in mongo.' - print 'Tabs before any changes:' + print('Warning: this command directly edits the list of course tabs in mongo.') + print('Tabs before any changes:') print_course(course) try: if options['delete']: - if len(args) != 1: - raise CommandError(Command.delete_option.help) - num = int(args[0]) + num = int(options['delete'][0]) + if num < 3: + raise CommandError("Tabs 1 and 2 cannot be changed.") + if query_yes_no('Deleting tab {0} Confirm?'.format(num), default='no'): tabs.primitive_delete(course, num - 1) # -1 for 0-based indexing elif options['insert']: - if len(args) != 3: - raise CommandError(Command.insert_option.help) - num = int(args[0]) - tab_type = args[1] - name = args[2] + num, tab_type, name = options['insert'] + num = int(num) + if num < 3: + raise CommandError("Tabs 1 and 2 cannot be changed.") + if query_yes_no('Inserting tab {0} "{1}" "{2}" Confirm?'.format(num, tab_type, name), default='no'): tabs.primitive_insert(course, num - 1, tab_type, name) # -1 as above except ValueError as e: diff --git a/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py b/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py index 3c7288552c..952164fb0a 100644 --- a/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py +++ b/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py @@ -8,16 +8,18 @@ from .prompt import query_yes_no class Command(BaseCommand): - help = '''Empty the trashcan. Can pass an optional course_id to limit the damage.''' + help = 'Empty the trashcan. Can pass an optional course_id to limit the damage.' + + def add_arguments(self, parser): + parser.add_argument('course_id', + help='Course ID to empty, leave off to empty for all courses', + nargs='?') def handle(self, *args, **options): - if len(args) != 1 and len(args) != 0: - raise CommandError("empty_asset_trashcan requires one or no arguments: ||") - - if len(args) == 1: - course_ids = [CourseKey.from_string(args[0])] + if options['course_id']: + course_ids = [CourseKey.from_string(options['course_id'])] else: course_ids = [course.id for course in modulestore().get_courses()] - if query_yes_no("Emptying trashcan. Confirm?", default="no"): + if query_yes_no("Emptying {} trashcan(s). Confirm?".format(len(course_ids)), default="no"): empty_asset_trashcan(course_ids) diff --git a/cms/djangoapps/contentstore/management/commands/export.py b/cms/djangoapps/contentstore/management/commands/export.py index ea351658ee..72dba96c76 100644 --- a/cms/djangoapps/contentstore/management/commands/export.py +++ b/cms/djangoapps/contentstore/management/commands/export.py @@ -1,6 +1,7 @@ """ Script for exporting courseware from Mongo to a tar.gz file """ +from __future__ import print_function import os from django.core.management.base import BaseCommand, CommandError @@ -37,7 +38,7 @@ class Command(BaseCommand): output_path = options['output_path'] - print "Exporting course id = {0} to {1}".format(course_key, output_path) + print("Exporting course id = {0} to {1}".format(course_key, output_path)) if not output_path.endswith('/'): output_path += '/' diff --git a/cms/djangoapps/contentstore/management/commands/export_all_courses.py b/cms/djangoapps/contentstore/management/commands/export_all_courses.py index 54f6b11f03..6f77ec7a99 100644 --- a/cms/djangoapps/contentstore/management/commands/export_all_courses.py +++ b/cms/djangoapps/contentstore/management/commands/export_all_courses.py @@ -1,7 +1,10 @@ """ Script for exporting all courseware from Mongo to a directory and listing the courses which failed to export """ -from django.core.management.base import BaseCommand, CommandError +from __future__ import print_function +from six import text_type + +from django.core.management.base import BaseCommand from xmodule.contentstore.django import contentstore from xmodule.modulestore.django import modulestore @@ -14,23 +17,22 @@ class Command(BaseCommand): """ help = 'Export all courses from mongo to the specified data directory and list the courses which failed to export' + def add_arguments(self, parser): + parser.add_argument('output_path') + def handle(self, *args, **options): """ Execute the command """ - if len(args) != 1: - raise CommandError("export requires one argument: ") + courses, failed_export_courses = export_courses_to_output_path(options['output_path']) - output_path = args[0] - courses, failed_export_courses = export_courses_to_output_path(output_path) - - print "=" * 80 - print u"=" * 30 + u"> Export summary" - print u"Total number of courses to export: {0}".format(len(courses)) - print u"Total number of courses which failed to export: {0}".format(len(failed_export_courses)) - print u"List of export failed courses ids:" - print u"\n".join(failed_export_courses) - print "=" * 80 + print("=" * 80) + print("=" * 30 + "> Export summary") + print("Total number of courses to export: {0}".format(len(courses))) + print("Total number of courses which failed to export: {0}".format(len(failed_export_courses))) + print("List of export failed courses ids:") + print("\n".join(failed_export_courses)) + print("=" * 80) def export_courses_to_output_path(output_path): @@ -46,15 +48,15 @@ def export_courses_to_output_path(output_path): failed_export_courses = [] for course_id in course_ids: - print u"-" * 80 - print u"Exporting course id = {0} to {1}".format(course_id, output_path) + print("-" * 80) + print("Exporting course id = {0} to {1}".format(course_id, output_path)) try: course_dir = course_id.to_deprecated_string().replace('/', '...') export_course_to_xml(module_store, content_store, course_id, root_dir, course_dir) except Exception as err: # pylint: disable=broad-except - failed_export_courses.append(unicode(course_id)) - print u"=" * 30 + u"> Oops, failed to export {0}".format(course_id) - print u"Error:" - print err + failed_export_courses.append(text_type(course_id)) + print("=" * 30 + "> Oops, failed to export {0}".format(course_id)) + print("Error:") + print(err) return courses, failed_export_courses diff --git a/cms/djangoapps/contentstore/management/commands/export_olx.py b/cms/djangoapps/contentstore/management/commands/export_olx.py index 72ac1f1f26..4d02aeefd7 100644 --- a/cms/djangoapps/contentstore/management/commands/export_olx.py +++ b/cms/djangoapps/contentstore/management/commands/export_olx.py @@ -12,7 +12,6 @@ At present, it differs from Studio exports in several ways: * The top-level directory in the resulting tarball is a "safe" (i.e. ascii) version of the course_key, rather than the word "course". * It only supports the export of courses. It does not export libraries. - """ import os @@ -34,17 +33,16 @@ from xmodule.modulestore.xml_exporter import export_course_to_xml class Command(BaseCommand): """ Export a course to XML. The output is compressed as a tar.gz file. - """ help = dedent(__doc__).strip() def add_arguments(self, parser): parser.add_argument('course_id') - parser.add_argument('--output', default=None) + parser.add_argument('--output') def handle(self, *args, **options): - course_id = options['course_id'] + try: course_key = CourseKey.from_string(course_id) except InvalidKeyError: @@ -54,6 +52,7 @@ class Command(BaseCommand): filename = options['output'] pipe_results = False + if filename is None: filename = mktemp() pipe_results = True diff --git a/cms/djangoapps/contentstore/management/commands/fix_not_found.py b/cms/djangoapps/contentstore/management/commands/fix_not_found.py index ddf00f9c97..7c41230417 100644 --- a/cms/djangoapps/contentstore/management/commands/fix_not_found.py +++ b/cms/djangoapps/contentstore/management/commands/fix_not_found.py @@ -20,7 +20,7 @@ class Command(BaseCommand): def handle(self, *args, **options): """Execute the command""" - course_id = options.get('course_id', None) + course_id = options['course_id'] course_key = CourseKey.from_string(course_id) # for now only support on split mongo diff --git a/cms/djangoapps/contentstore/management/commands/force_publish.py b/cms/djangoapps/contentstore/management/commands/force_publish.py index 6dcea6f95f..642e87db21 100644 --- a/cms/djangoapps/contentstore/management/commands/force_publish.py +++ b/cms/djangoapps/contentstore/management/commands/force_publish.py @@ -44,7 +44,7 @@ class Command(BaseCommand): owning_store = modulestore()._get_modulestore_for_courselike(course_key) # pylint: disable=protected-access if hasattr(owning_store, 'force_publish_course'): versions = get_course_versions(options['course_key']) - print "Course versions : {0}".format(versions) + print("Course versions : {0}".format(versions)) if options['commit']: if query_yes_no("Are you sure to publish the {0} course forcefully?".format(course_key), default="no"): @@ -55,20 +55,20 @@ class Command(BaseCommand): if updated_versions: # if publish and draft were different if versions['published-branch'] != versions['draft-branch']: - print "Success! Published the course '{0}' forcefully.".format(course_key) - print "Updated course versions : \n{0}".format(updated_versions) + print("Success! Published the course '{0}' forcefully.".format(course_key)) + print("Updated course versions : \n{0}".format(updated_versions)) else: - print "Course '{0}' is already in published state.".format(course_key) + print("Course '{0}' is already in published state.".format(course_key)) else: - print "Error! Could not publish course {0}.".format(course_key) + print("Error! Could not publish course {0}.".format(course_key)) else: # if publish and draft were different if versions['published-branch'] != versions['draft-branch']: - print "Dry run. Following would have been changed : " - print "Published branch version {0} changed to draft branch version {1}".format( - versions['published-branch'], versions['draft-branch'] + print("Dry run. Following would have been changed : ") + print("Published branch version {0} changed to draft branch version {1}".format( + versions['published-branch'], versions['draft-branch']) ) else: - print "Dry run. Course '{0}' is already in published state.".format(course_key) + print("Dry run. Course '{0}' is already in published state.".format(course_key)) else: raise CommandError("The owning modulestore does not support this command.") diff --git a/cms/djangoapps/contentstore/management/commands/generate_courses.py b/cms/djangoapps/contentstore/management/commands/generate_courses.py index 475e22801d..19bd29858f 100644 --- a/cms/djangoapps/contentstore/management/commands/generate_courses.py +++ b/cms/djangoapps/contentstore/management/commands/generate_courses.py @@ -3,6 +3,7 @@ Django management command to generate a test course from a course config json """ import json import logging +from six import text_type from django.contrib.auth.models import User from django.core.management.base import BaseCommand, CommandError @@ -57,7 +58,7 @@ class Command(BaseCommand): # Create the course try: new_course = create_new_course_in_store("split", user, org, num, run, fields) - logger.info("Created {}".format(unicode(new_course.id))) + logger.info("Created {}".format(text_type(new_course.id))) except DuplicateCourseError: logger.warning("Course already exists for %s, %s, %s", org, num, run) diff --git a/cms/djangoapps/contentstore/management/commands/git_export.py b/cms/djangoapps/contentstore/management/commands/git_export.py index bbf9d5a2a9..1fe60068fb 100644 --- a/cms/djangoapps/contentstore/management/commands/git_export.py +++ b/cms/djangoapps/contentstore/management/commands/git_export.py @@ -14,7 +14,7 @@ attribute is set and the FEATURE['ENABLE_EXPORT_GIT'] is set. """ import logging -from optparse import make_option +from six import text_type from django.core.management.base import BaseCommand, CommandError from django.utils.translation import ugettext as _ @@ -31,41 +31,34 @@ class Command(BaseCommand): """ Take a course from studio and export it to a git repository. """ - - option_list = BaseCommand.option_list + ( - make_option('--username', '-u', dest='user', - help=('Specify a username from LMS/Studio to be used ' - 'as the commit author.')), - make_option('--repo_dir', '-r', dest='repo', - help='Specify existing git repo directory.'), - ) - help = _('Take the specified course and attempt to ' 'export it to a git repository\n. Course directory ' 'must already be a git repository. Usage: ' ' git_export ') + def add_arguments(self, parser): + parser.add_argument('course_loc') + parser.add_argument('git_url') + parser.add_argument('--username', '-u', dest='user', + help='Specify a username from LMS/Studio to be used as the commit author.') + parser.add_argument('--repo_dir', '-r', dest='repo', help='Specify existing git repo directory.') + def handle(self, *args, **options): """ Checks arguments and runs export function if they are good """ - - if len(args) != 2: - raise CommandError('This script requires exactly two arguments: ' - 'course_loc and git_url') - # Rethrow GitExportError as CommandError for SystemExit try: - course_key = CourseKey.from_string(args[0]) + course_key = CourseKey.from_string(options['course_loc']) except InvalidKeyError: - raise CommandError(unicode(GitExportError.BAD_COURSE)) + raise CommandError(text_type(GitExportError.BAD_COURSE)) try: git_export_utils.export_to_git( course_key, - args[1], + options['git_url'], options.get('user', ''), options.get('rdir', None) ) except git_export_utils.GitExportError as ex: - raise CommandError(unicode(ex.message)) + raise CommandError(text_type(ex.message)) diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index 9cd6467033..123c9c4173 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -3,7 +3,7 @@ Script for importing courseware from XML format """ from optparse import make_option -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from django_comment_common.utils import are_permissions_roles_seeded, seed_permissions_roles from xmodule.contentstore.django import contentstore diff --git a/cms/djangoapps/contentstore/management/commands/migrate_to_split.py b/cms/djangoapps/contentstore/management/commands/migrate_to_split.py index 3613a26c5e..2a3909edb3 100644 --- a/cms/djangoapps/contentstore/management/commands/migrate_to_split.py +++ b/cms/djangoapps/contentstore/management/commands/migrate_to_split.py @@ -18,41 +18,34 @@ class Command(BaseCommand): Migrate a course from old-Mongo to split-Mongo. It reuses the old course id except where overridden. """ - help = "Migrate a course from old-Mongo to split-Mongo. The new org, course, and run will default to the old one unless overridden" - args = "course_key email " + help = "Migrate a course from old-Mongo to split-Mongo. The new org, course, and run will " \ + "default to the old one unless overridden." - def parse_args(self, *args): + def add_arguments(self, parser): + parser.add_argument('course_key') + parser.add_argument('email') + parser.add_argument('--org', help='New org to migrate to.') + parser.add_argument('--course', help='New course key to migrate to.') + parser.add_argument('--run', help='New run to migrate to.') + + def parse_args(self, **options): """ Return a 5-tuple of passed in values for (course_key, user, org, course, run). """ - if len(args) < 2: - raise CommandError( - "migrate_to_split requires at least two arguments: " - "a course_key and a user identifier (email or ID)" - ) - try: - course_key = CourseKey.from_string(args[0]) + course_key = CourseKey.from_string(options['course_key']) except InvalidKeyError: raise CommandError("Invalid location string") try: - user = user_from_str(args[1]) + user = user_from_str(options['email']) except User.DoesNotExist: - raise CommandError("No user found identified by {}".format(args[1])) + raise CommandError("No user found identified by {}".format(options['email'])) - org = course = run = None - try: - org = args[2] - course = args[3] - run = args[4] - except IndexError: - pass - - return course_key, user.id, org, course, run + return course_key, user.id, options['org'], options['course'], options['run'] def handle(self, *args, **options): - course_key, user, org, course, run = self.parse_args(*args) + course_key, user, org, course, run = self.parse_args(**options) migrator = SplitMigrator( source_modulestore=modulestore(), diff --git a/cms/djangoapps/contentstore/management/commands/populate_creators.py b/cms/djangoapps/contentstore/management/commands/populate_creators.py index fa5a848028..4822f9089f 100644 --- a/cms/djangoapps/contentstore/management/commands/populate_creators.py +++ b/cms/djangoapps/contentstore/management/commands/populate_creators.py @@ -2,6 +2,8 @@ Script for granting existing course instructors course creator privileges. This script is only intended to be run once on a given environment. + +To run: ./manage.py cms populate_creators --settings=dev """ from django.contrib.auth.models import User from django.core.management.base import BaseCommand @@ -11,9 +13,6 @@ from course_creators.views import add_user_with_status_granted, add_user_with_st from student.roles import CourseInstructorRole, CourseStaffRole -#------------ to run: ./manage.py cms populate_creators --settings=dev - - class Command(BaseCommand): """ Script for granting existing course instructors course creator privileges. @@ -35,23 +34,24 @@ class Command(BaseCommand): # the admin user will already exist. admin = User.objects.get(username=username, email=email) - for user in get_users_with_role(CourseInstructorRole.ROLE): - add_user_with_status_granted(admin, user) + try: + for user in get_users_with_role(CourseInstructorRole.ROLE): + add_user_with_status_granted(admin, user) - # Some users will be both staff and instructors. Those folks have been - # added with status granted above, and add_user_with_status_unrequested - # will not try to add them again if they already exist in the course creator database. - for user in get_users_with_role(CourseStaffRole.ROLE): - add_user_with_status_unrequested(user) + # Some users will be both staff and instructors. Those folks have been + # added with status granted above, and add_user_with_status_unrequested + # will not try to add them again if they already exist in the course creator database. + for user in get_users_with_role(CourseStaffRole.ROLE): + add_user_with_status_unrequested(user) - # There could be users who are not in either staff or instructor (they've - # never actually done anything in Studio). I plan to add those as unrequested - # when they first go to their dashboard. - - admin.delete() + # There could be users who are not in either staff or instructor (they've + # never actually done anything in Studio). I plan to add those as unrequested + # when they first go to their dashboard. + finally: + # Let's not leave this lying around. + admin.delete() -#============================================================================================================= # Because these are expensive and far-reaching, I moved them here def get_users_with_role(role_prefix): """ diff --git a/cms/djangoapps/contentstore/management/commands/reindex_course.py b/cms/djangoapps/contentstore/management/commands/reindex_course.py index 796783ca57..4328f2e968 100644 --- a/cms/djangoapps/contentstore/management/commands/reindex_course.py +++ b/cms/djangoapps/contentstore/management/commands/reindex_course.py @@ -1,6 +1,5 @@ """ Management command to update courses' search index """ import logging -from optparse import make_option from textwrap import dedent from django.core.management import BaseCommand, CommandError @@ -22,32 +21,25 @@ class Command(BaseCommand): Examples: - ./manage.py reindex_course - reindexes courses with keys course_id_1 and course_id_2 + ./manage.py reindex_course ... - reindexes courses with provided keys ./manage.py reindex_course --all - reindexes all available courses ./manage.py reindex_course --setup - reindexes all courses for devstack setup """ help = dedent(__doc__) - can_import_settings = True - - args = "" - - all_option = make_option('--all', - action='store_true', - dest='all', - default=False, - help='Reindex all courses') - - setup_option = make_option('--setup', - action='store_true', - dest='setup', - default=False, - help='Reindex all courses on developers stack setup') - - option_list = BaseCommand.option_list + (all_option, setup_option) - CONFIRMATION_PROMPT = u"Re-indexing all courses might be a time consuming operation. Do you want to continue?" + def add_arguments(self, parser): + parser.add_argument('course_ids', + nargs='*', + metavar='course_id') + parser.add_argument('--all', + action='store_true', + help='Reindex all courses') + parser.add_argument('--setup', + action='store_true', + help='Reindex all courses on developers stack setup') + def _parse_course_key(self, raw_value): """ Parses course key from string """ try: @@ -65,12 +57,14 @@ class Command(BaseCommand): By convention set by Django developers, this method actually executes command's actions. So, there could be no better docstring than emphasize this once again. """ - all_option = options.get('all', False) - setup_option = options.get('setup', False) + course_ids = options['course_ids'] + all_option = options['all'] + setup_option = options['setup'] index_all_courses_option = all_option or setup_option - if len(args) == 0 and not index_all_courses_option: - raise CommandError(u"reindex_course requires one or more arguments: ") + if (not len(course_ids) and not index_all_courses_option) or \ + (len(course_ids) and index_all_courses_option): + raise CommandError("reindex_course requires one or more s OR the --all or --setup flags.") store = modulestore() @@ -82,7 +76,7 @@ class Command(BaseCommand): # try getting the ElasticSearch engine searcher = SearchEngine.get_search_engine(index_name) except exceptions.ElasticsearchException as exc: - logging.exception('Search Engine error - %s', unicode(exc)) + logging.exception('Search Engine error - %s', exc) return index_exists = searcher._es.indices.exists(index=index_name) # pylint: disable=protected-access @@ -108,7 +102,7 @@ class Command(BaseCommand): return else: # in case course keys are provided as arguments - course_keys = map(self._parse_course_key, args) + course_keys = map(self._parse_course_key, course_ids) for course_key in course_keys: CoursewareSearchIndexer.do_course_reindex(store, course_key) diff --git a/cms/djangoapps/contentstore/management/commands/reindex_library.py b/cms/djangoapps/contentstore/management/commands/reindex_library.py index 596373ffea..50d7a70d0c 100644 --- a/cms/djangoapps/contentstore/management/commands/reindex_library.py +++ b/cms/djangoapps/contentstore/management/commands/reindex_library.py @@ -1,5 +1,5 @@ """ Management command to update libraries' search index """ -from optparse import make_option +from __future__ import print_function from textwrap import dedent from django.core.management import BaseCommand, CommandError @@ -22,21 +22,17 @@ class Command(BaseCommand): ./manage.py reindex_library --all - reindexes all available libraries """ help = dedent(__doc__) - can_import_settings = True + CONFIRMATION_PROMPT = u"Reindexing all libraries might be a time consuming operation. Do you want to continue?" - args = "" - - option_list = BaseCommand.option_list + ( - make_option( + def add_arguments(self, parser): + parser.add_argument('library_ids', nargs='*') + parser.add_argument( '--all', action='store_true', dest='all', - default=False, help='Reindex all libraries' - ),) - - CONFIRMATION_PROMPT = u"Reindexing all libraries might be a time consuming operation. Do you want to continue?" + ) def _parse_library_key(self, raw_value): """ Parses library key from string """ @@ -52,18 +48,19 @@ class Command(BaseCommand): By convention set by django developers, this method actually executes command's actions. So, there could be no better docstring than emphasize this once again. """ - if len(args) == 0 and not options.get('all', False): - raise CommandError(u"reindex_library requires one or more arguments: ") + if (not options['library_ids'] and not options['all']) or (options['library_ids'] and options['all']): + raise CommandError(u"reindex_library requires one or more s or the --all flag.") store = modulestore() - if options.get('all', False): + if options['all']: if query_yes_no(self.CONFIRMATION_PROMPT, default="no"): library_keys = [library.location.library_key.replace(branch=None) for library in store.get_libraries()] else: return else: - library_keys = map(self._parse_library_key, args) + library_keys = map(self._parse_library_key, options['library_ids']) for library_key in library_keys: + print("Indexing library {}".format(library_key)) LibrarySearchIndexer.do_library_reindex(store, library_key) diff --git a/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py b/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py index fa314ddbd3..ca8de0ceb7 100644 --- a/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py +++ b/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py @@ -6,8 +6,8 @@ from xmodule.contentstore.utils import restore_asset_from_trashcan class Command(BaseCommand): help = '''Restore a deleted asset from the trashcan back to it's original course''' - def handle(self, *args, **options): - if len(args) != 1 and len(args) != 0: - raise CommandError("restore_asset_from_trashcan requires one argument: ") + def add_arguments(self, parser): + parser.add_argument('location') - restore_asset_from_trashcan(args[0]) + def handle(self, *args, **options): + restore_asset_from_trashcan(options['location']) diff --git a/cms/djangoapps/contentstore/management/commands/tests/test_reindex_courses.py b/cms/djangoapps/contentstore/management/commands/tests/test_reindex_courses.py index 71db8f4f07..3d3d99b91a 100644 --- a/cms/djangoapps/contentstore/management/commands/tests/test_reindex_courses.py +++ b/cms/djangoapps/contentstore/management/commands/tests/test_reindex_courses.py @@ -2,6 +2,7 @@ import ddt from django.core.management import call_command, CommandError import mock +from six import text_type from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore @@ -60,34 +61,34 @@ class TestReindexCourse(ModuleStoreTestCase): def test_given_library_key_raises_command_error(self): """ Test that raises CommandError if library key is passed """ with self.assertRaisesRegexp(CommandError, ".* is not a course key"): - call_command('reindex_course', unicode(self._get_lib_key(self.first_lib))) + call_command('reindex_course', text_type(self._get_lib_key(self.first_lib))) with self.assertRaisesRegexp(CommandError, ".* is not a course key"): - call_command('reindex_course', unicode(self._get_lib_key(self.second_lib))) + call_command('reindex_course', text_type(self._get_lib_key(self.second_lib))) with self.assertRaisesRegexp(CommandError, ".* is not a course key"): call_command( 'reindex_course', - unicode(self.second_course.id), - unicode(self._get_lib_key(self.first_lib)) + text_type(self.second_course.id), + text_type(self._get_lib_key(self.first_lib)) ) def test_given_id_list_indexes_courses(self): """ Test that reindexes courses when given single course key or a list of course keys """ with mock.patch(self.REINDEX_PATH_LOCATION) as patched_index, \ mock.patch(self.MODULESTORE_PATCH_LOCATION, mock.Mock(return_value=self.store)): - call_command('reindex_course', unicode(self.first_course.id)) + call_command('reindex_course', text_type(self.first_course.id)) self.assertEqual(patched_index.mock_calls, self._build_calls(self.first_course)) patched_index.reset_mock() - call_command('reindex_course', unicode(self.second_course.id)) + call_command('reindex_course', text_type(self.second_course.id)) self.assertEqual(patched_index.mock_calls, self._build_calls(self.second_course)) patched_index.reset_mock() call_command( 'reindex_course', - unicode(self.first_course.id), - unicode(self.second_course.id) + text_type(self.first_course.id), + text_type(self.second_course.id) ) expected_calls = self._build_calls(self.first_course, self.second_course) self.assertEqual(patched_index.mock_calls, expected_calls) @@ -121,4 +122,4 @@ class TestReindexCourse(ModuleStoreTestCase): patched_index.side_effect = SearchIndexingError("message", []) with self.assertRaises(SearchIndexingError): - call_command('reindex_course', unicode(self.second_course.id)) + call_command('reindex_course', text_type(self.second_course.id)) diff --git a/cms/djangoapps/contentstore/management/commands/xlint.py b/cms/djangoapps/contentstore/management/commands/xlint.py index afb7c73980..4eac818538 100644 --- a/cms/djangoapps/contentstore/management/commands/xlint.py +++ b/cms/djangoapps/contentstore/management/commands/xlint.py @@ -1,26 +1,33 @@ """ Verify the structure of courseware as to it's suitability for import """ -from django.core.management.base import BaseCommand, CommandError +from __future__ import print_function +from argparse import REMAINDER + +from django.core.management.base import BaseCommand from xmodule.modulestore.xml_importer import perform_xlint class Command(BaseCommand): - """Verify the structure of courseware as to it's suitability for import""" - help = "Verify the structure of courseware as to it's suitability for import" + """Verify the structure of courseware as to its suitability for import""" + help = """ + Verify the structure of courseware as to its suitability for import. + To run: manage.py cms [...] + """ + + def add_arguments(self, parser): + parser.add_argument('data_dir') + parser.add_argument('source_dirs', nargs=REMAINDER) def handle(self, *args, **options): - "Execute the command" - if len(args) == 0: - raise CommandError("import requires at least one argument: [...]") + """Execute the command""" + + data_dir = options['data_dir'] + source_dirs = options['source_dirs'] - data_dir = args[0] - if len(args) > 1: - source_dirs = args[1:] - else: - source_dirs = None print("Importing. Data_dir={data}, source_dirs={courses}".format( data=data_dir, courses=source_dirs)) + perform_xlint(data_dir, source_dirs, load_error_modules=False)