Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[BD-14] Migrate all environments to use database-backed organizations (#25153)

Browse Source 
* Install `organizations` app into LMS and Studio non-optionally.
* Add toggle `ORGANIZATIONS_AUTOCREATE` to Studio.
* Remove the `FEATURES["ORGANIZATIONS_APP"]` toggle.
* Use the new `organizations.api.ensure_organization` function to
  either validate or get-or-create organizations, depending
  on the value of `ORGANIZATIONS_AUTOCREATE`,
  when creating course runs and V2 content libraries.
  We'll soon use it for V1 content libraries as well.
* Remove the `util.organizations_helpers` wrapper layer
  that had to exist because `organizations` was an optional app.
* Add `.get_library_keys()` method to the Split modulestore.
* Add Studio management command for backfilling organizations tables
  (`backfill_orgs_and_org_courses`).

For full details, see
https://github.com/edx/edx-organizations/blob/master/docs/decisions/0001-phase-in-db-backed-organizations-to-all.rst

TNL-7646
This commit is contained in:
Kyle McCormick
2020-12-02 13:58:40 -05:00
committed by GitHub
parent f096f5d685
commit 4dda73d797
30 changed files with 475 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cms/djangoapps/api/v1/tests/test_views/test_course_runs.py
View File

@@ -6,16 +6,16 @@ import datetime
import ddt
import pytz
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import RequestFactory
from django.test import RequestFactory, override_settings
from django.urls import reverse
from mock import patch
from opaque_keys.edx.keys import CourseKey
from organizations.api import add_organization, get_course_organizations
from rest_framework.test import APIClient
from openedx.core.lib.courses import course_image_url
from common.djangoapps.student.models import CourseAccessRole
from common.djangoapps.student.tests.factories import TEST_PASSWORD, AdminFactory, UserFactory
from common.djangoapps.util.organizations_helpers import add_organization, get_course_organizations
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import NotFoundError
@@ -321,7 +321,7 @@ class CourseRunViewSetTests(ModuleStoreTestCase):
# There should now be an image stored
contentstore().find(content_key)
@patch.dict('django.conf.settings.FEATURES', {'ORGANIZATIONS_APP': True})
@override_settings(ORGANIZATIONS_AUTOCREATE=False)
@ddt.data(
('instructor_paced', False, 'NotOriginalNumber1x'),
('self_paced', True, None),

162
cms/djangoapps/contentstore/management/commands/backfill_orgs_and_org_courses.py Normal file
View File

@@ -0,0 +1,162 @@
"""
A backfill command to migrate Open edX instances to the new world of
"organizations are enabled everywhere".
For full context, see:
https://github.com/edx/edx-organizations/blob/master/docs/decisions/0001-phase-in-db-backed-organizations-to-all.rst
"""
from django.core.management import BaseCommand, CommandError
from organizations import api as organizations_api
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
from xmodule.modulestore.django import modulestore
class Command(BaseCommand):
"""
Back-populate edx-organizations models from existing course runs & content libraries.
Before the Lilac open release, Open edX instances by default did not make
use of the models in edx-organizations.
In Lilac and beyond, the edx-organizations models are enabled globally.
This command exists to migrate pre-Lilac instances that did not enable
`FEATURES['ORGANIZATIONS_APP']`.
It automatically creates all missing Organization and OrganizationCourse
instances based on the course runs in the system (loaded from CourseOverview)
and the V1 content libraries in the system (loaded from the Modulestore).
Organizations created by this command will have their `short_name` and
`name` equal to the `org` part of the library/course key that triggered
their creation. For example, given an Open edX instance with the course run
`course-v1:myOrg+myCourse+myRun` but no such Organization with the short name
"myOrg" (case-insensitive), this command will create the following
organization:
> Organization(
> short_name='myOrg',
> name='myOrg',
> description=None,
> logo=None,
> active=True,
> )
"""
# Make help message the first line of docstring.
# I'd like to include the entire docstring but Django omits the newlines,
# so it looks pretty bad.
help = __doc__.strip().splitlines()[0]
def add_arguments(self, parser):
parser.add_argument(
'--apply',
action='store_true',
help="Apply backfill to database without prompting for confirmation."
)
parser.add_argument(
'--dry',
action='store_true',
help="Show backfill, but do not apply changes to database."
)
def handle(self, *args, **options):
"""
Handle the backfill command.
"""
orgslug_coursekey_pairs = find_orgslug_coursekey_pairs()
orgslug_library_pairs = find_orgslug_library_pairs()
orgslugs = (
{orgslug for orgslug, _ in orgslug_coursekey_pairs} |
{orgslug for orgslug, _ in orgslug_library_pairs}
)
# Note: the `organizations.api.bulk_add_*` code will handle:
# * not overwriting existing organizations, and
# * skipping duplicates, based on the short name (case-insensiive),
# so we don't have to worry about those here.
orgs = [
{"short_name": orgslug, "name": orgslug}
# The `sorted` calls aren't strictly necessary, but they'll help make this
# function more deterministic in case something goes wrong.
for orgslug in sorted(orgslugs)
]
org_coursekey_pairs = [
({"short_name": orgslug}, coursekey)
for orgslug, coursekey in sorted(orgslug_coursekey_pairs)
]
if not confirm_changes(options, orgs, org_coursekey_pairs):
print("No changes applied.")
return
print("Applying changes...")
organizations_api.bulk_add_organizations(orgs, dry_run=False)
organizations_api.bulk_add_organization_courses(org_coursekey_pairs, dry_run=False)
print("Changes applied successfully.")
def confirm_changes(options, orgs, org_coursekey_pairs):
"""
Should we apply the changes to the database?
If `--apply`, this just returns True.
If `--dry`, this does a dry run and then returns False.
Otherwise, it does a dry run and then prompts the user.
Arguments:
options (dict[str]): command-line arguments.
orgs (list[dict]): list of org data dictionaries to bulk-add.
org_coursekey_pairs (list[tuple[dict, CourseKey]]):
list of (org data dictionary, course key) links to bulk-add.
Returns: bool
"""
if options.get('apply') and options.get('dry'):
raise CommandError("Only one of 'apply' and 'dry' may be specified")
if options.get('apply'):
return True
organizations_api.bulk_add_organizations(orgs, dry_run=True)
organizations_api.bulk_add_organization_courses(org_coursekey_pairs, dry_run=True)
if options.get('dry'):
return False
answer = ""
while answer.lower() not in {'y', 'yes', 'n', 'no'}:
answer = input('Commit changes shown above to the database [y/n]? ')
return answer.lower().startswith('y')
def find_orgslug_coursekey_pairs():
"""
Returns the unique pairs of (organization short name, course run key)
from the CourseOverviews table, which should contain all course runs in the
system.
Returns: set[tuple[str, CourseKey]]
"""
# Using a set comprehension removes any duplicate (org, course) pairs.
return {
(course_key.org, course_key)
for course_key
# Worth noting: This will load all CourseOverviews, no matter their VERSION.
# This is intentional: there may be course runs that haven't updated
# their CourseOverviews entry since the last schema change; we still want
# capture those course runs.
in CourseOverview.objects.all().values_list("id", flat=True)
}
def find_orgslug_library_pairs():
"""
Returns the unique pairs of (organization short name, content library key)
from the modulestore.
Note that this only considers "version 1" (aka "legacy" or "modulestore-based")
content libraries.
We do not consider "version 2" (aka "blockstore-based") content libraries,
because those require a database-level link to their authoring organization,
and thus would not need backfilling via this command.
Returns: set[tuple[str, LibraryLocator]]
"""
# Using a set comprehension removes any duplicate (org, library) pairs.
return {
(library_key.org, library_key)
for library_key
in modulestore().get_library_keys()
}

186
cms/djangoapps/contentstore/management/commands/tests/test_backfill_orgs_and_org_courses.py Normal file
View File

@@ -0,0 +1,186 @@
"""
Tests for `backfill_orgs_and_org_courses` CMS management command.
"""
from unittest.mock import patch
import ddt
from django.core.management import CommandError, call_command
from organizations import api as organizations_api
from organizations.api import (
add_organization,
add_organization_course,
get_organization_by_short_name,
get_organization_courses,
get_organizations
)
from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.tests.factories import LibraryFactory
from .. import backfill_orgs_and_org_courses
@ddt.ddt
class BackfillOrgsAndOrgCoursesTest(SharedModuleStoreTestCase):
"""
Test `backfill_orgs_and_org_courses`.
We test:
* That one happy path of the command works.
* That the command line args are processed correctly.
* That the confirmation prompt works.
We don't test:
* Specifics/edge cases around fetching course run keys, content library keys,
or the actual application of the backfill. Those are handled by tests within
`course_overviews`, `modulestore`, and `organizations`, respectively.
"""
def test_end_to_end(self):
"""
Test the happy path of the backfill command without any mocking.
"""
# org_A: already existing, with courses and a library.
org_a = add_organization({"short_name": "org_A", "name": "Org A"})
course_a1_key = CourseOverviewFactory(org="org_A", run="1").id
CourseOverviewFactory(org="org_A", run="2")
LibraryFactory(org="org_A")
# Write linkage for org_a->course_a1.
# (Linkage for org_a->course_a2 is purposefully left out here;
# it should be created by the backfill).
add_organization_course(org_a, course_a1_key)
# org_B: already existing, but has no content.
add_organization({"short_name": "org_B", "name": "Org B"})
# org_C: has a couple courses; should be created.
CourseOverviewFactory(org="org_C", run="1")
CourseOverviewFactory(org="org_C", run="2")
# org_D: has both a course and a library; should be created.
CourseOverviewFactory(org="org_D", run="1")
LibraryFactory(org="org_D")
# org_E: just has a library; should be created.
LibraryFactory(org="org_E")
# Confirm starting condition:
# Only orgs are org_A and org_B, and only linkage is org_a->course_a1.
assert set(
org["short_name"] for org in get_organizations()
) == {
"org_A", "org_B"
}
assert len(get_organization_courses(get_organization_by_short_name('org_A'))) == 1
assert len(get_organization_courses(get_organization_by_short_name('org_B'))) == 0
# Run the backfill.
call_command("backfill_orgs_and_org_courses", "--apply")
# Confirm ending condition:
# All five orgs present. Each org a has expected number of org-course linkages.
assert set(
org["short_name"] for org in get_organizations()
) == {
"org_A", "org_B", "org_C", "org_D", "org_E"
}
assert len(get_organization_courses(get_organization_by_short_name('org_A'))) == 2
assert len(get_organization_courses(get_organization_by_short_name('org_B'))) == 0
assert len(get_organization_courses(get_organization_by_short_name('org_C'))) == 2
assert len(get_organization_courses(get_organization_by_short_name('org_D'))) == 1
assert len(get_organization_courses(get_organization_by_short_name('org_E'))) == 0
@ddt.data(
{
"command_line_args": [],
"user_inputs": ["n"],
"should_apply_changes": False,
},
{
"command_line_args": [],
"user_inputs": ["x", "N"],
"should_apply_changes": False,
},
{
"command_line_args": [],
"user_inputs": ["", "", "YeS"],
"should_apply_changes": True,
},
{
"command_line_args": ["--dry"],
"user_inputs": [],
"should_apply_changes": False,
},
{
"command_line_args": ["--apply"],
"user_inputs": [],
"should_apply_changes": True,
},
)
@ddt.unpack
@patch.object(organizations_api, 'bulk_add_organizations')
@patch.object(organizations_api, 'bulk_add_organization_courses')
def test_arguments_and_input(
self,
mock_add_orgs,
mock_add_org_courses,
command_line_args,
user_inputs,
should_apply_changes,
):
"""
Test that the command-line arguments and user input processing works as
expected.
Given a list of `command_line_args` and a sequence of `user_inputs`
that will be supplied, we expect that:
* the user will be prompted a number of times equal to the length of `user_inputs`, and
* the command will/won't apply changes according to `should_apply_changes`.
"""
with patch.object(
backfill_orgs_and_org_courses, "input", side_effect=user_inputs
) as mock_input:
call_command("backfill_orgs_and_org_courses", *command_line_args)
# Make sure user was prompted the number of times we expected.
assert mock_input.call_count == len(user_inputs)
if should_apply_changes and user_inputs:
# If we DID apply changes and the user WAS prompted first,
# then we expect one DRY bulk-add run *and* one REAL bulk-add run.
assert mock_add_orgs.call_count == 2
assert mock_add_org_courses.call_count == 2
assert mock_add_orgs.call_args_list[0].kwargs == {"dry_run": True}
assert mock_add_org_courses.call_args_list[0].kwargs == {"dry_run": True}
assert mock_add_orgs.call_args_list[1].kwargs == {"dry_run": False}
assert mock_add_org_courses.call_args_list[1].kwargs == {"dry_run": False}
elif should_apply_changes:
# If DID apply changes but the user WASN'T prompted,
# then we expect just one REAL bulk-add run.
assert mock_add_orgs.call_count == 1
assert mock_add_org_courses.call_count == 1
assert mock_add_orgs.call_args.kwargs == {"dry_run": False}
assert mock_add_org_courses.call_args.kwargs == {"dry_run": False}
elif user_inputs:
# If we DIDN'T apply changes but the user WAS prompted
# then we expect just one DRY bulk-add run.
assert mock_add_orgs.call_count == 1
assert mock_add_org_courses.call_count == 1
assert mock_add_orgs.call_args.kwargs == {"dry_run": True}
assert mock_add_org_courses.call_args.kwargs == {"dry_run": True}
else:
# Similarly, if we DIDN'T apply changes and the user WASN'T prompted
# then we expect just one DRY bulk-add run.
assert mock_add_orgs.call_count == 1
assert mock_add_org_courses.call_count == 1
assert mock_add_orgs.call_args.kwargs == {"dry_run": True}
assert mock_add_org_courses.call_args.kwargs == {"dry_run": True}
def test_conflicting_arguments(self):
"""
Test that calling the command with both "--dry" and "--apply" raises an exception.
"""
with self.assertRaises(CommandError):
call_command("backfill_orgs_and_org_courses", "--dry", "--apply")

4
cms/djangoapps/contentstore/tasks.py
View File

@@ -25,6 +25,7 @@ from django.utils.translation import ugettext as _
from edx_django_utils.monitoring import set_code_owner_attribute, set_code_owner_attribute_from_module
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryLocator
from organizations.api import add_organization_course, ensure_organization
from organizations.models import OrganizationCourse
from path import Path as path
from pytz import UTC
@@ -44,7 +45,6 @@ from common.djangoapps.course_action_state.models import CourseRerunState
from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse
from openedx.core.lib.extract_tar import safetar_extractall
from common.djangoapps.student.auth import has_course_author_access
from common.djangoapps.util.organizations_helpers import add_organization_course, get_organization_by_short_name
from xmodule.contentstore.django import contentstore
from xmodule.course_module import CourseFields
from xmodule.exceptions import SerializationError
@@ -128,7 +128,7 @@ def rerun_course(source_course_key_string, destination_course_key_string, user_i
for country_access_rule in country_access_rules:
clone_instance(country_access_rule, {'restricted_course': new_restricted_course})
org_data = get_organization_by_short_name(source_course_key.org)
org_data = ensure_organization(source_course_key.org)
add_organization_course(org_data, destination_course_key)
return "succeeded"

41
cms/djangoapps/contentstore/tests/test_course_create_rerun.py
View File

@@ -7,15 +7,20 @@ import datetime
import ddt
import six
from django.test import override_settings
from django.test.client import RequestFactory
from django.urls import reverse
from mock import patch
from opaque_keys.edx.keys import CourseKey
from organizations.api import (
add_organization,
get_organization_by_short_name,
get_course_organizations
)
from organizations.exceptions import InvalidOrganizationException
from cms.djangoapps.contentstore.tests.utils import AjaxEnabledTestClient, parse_json
from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole
from common.djangoapps.student.tests.factories import UserFactory
from common.djangoapps.util.organizations_helpers import add_organization, get_course_organizations
from xmodule.course_module import CourseFields
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
@@ -66,7 +71,6 @@ class TestCourseListing(ModuleStoreTestCase):
self.client.logout()
ModuleStoreTestCase.tearDown(self)
@patch.dict('django.conf.settings.FEATURES', {'ORGANIZATIONS_APP': True})
def test_rerun(self):
"""
Just testing the functionality the view handler adds over the tasks tested in test_clone_course
@@ -114,13 +118,15 @@ class TestCourseListing(ModuleStoreTestCase):
course = self.store.get_course(new_course_key)
self.assertTrue(course.cert_html_view_enabled)
@patch.dict('django.conf.settings.FEATURES', {'ORGANIZATIONS_APP': False})
@ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
def test_course_creation_without_org_app_enabled(self, store):
def test_course_creation_for_unknown_organization_relaxed(self, store):
"""
Tests course creation workflow should not create course to org
link if organizations_app is not enabled.
Tests that when ORGANIZATIONS_AUTOCREATE is True,
creating a course-run with an unknown org slug will create an organization
and organization-course linkage in the system.
"""
with self.assertRaises(InvalidOrganizationException):
get_organization_by_short_name("orgX")
with modulestore().default_store(store):
response = self.client.ajax_post(self.course_create_rerun_url, {
'org': 'orgX',
@@ -129,17 +135,19 @@ class TestCourseListing(ModuleStoreTestCase):
'run': '2015_T2'
})
self.assertEqual(response.status_code, 200)
self.assertIsNotNone(get_organization_by_short_name("orgX"))
data = parse_json(response)
new_course_key = CourseKey.from_string(data['course_key'])
course_orgs = get_course_organizations(new_course_key)
self.assertEqual(course_orgs, [])
self.assertEqual(len(course_orgs), 1)
self.assertEqual(course_orgs[0]['short_name'], 'orgX')
@patch.dict('django.conf.settings.FEATURES', {'ORGANIZATIONS_APP': True})
@ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
def test_course_creation_with_org_not_in_system(self, store):
@override_settings(ORGANIZATIONS_AUTOCREATE=False)
def test_course_creation_for_unknown_organization_strict(self, store):
"""
Tests course creation workflow when course organization does not exist
in system.
Tests that when ORGANIZATIONS_AUTOCREATE is False,
creating a course-run with an unknown org slug will raise a validation error.
"""
with modulestore().default_store(store):
response = self.client.ajax_post(self.course_create_rerun_url, {
@@ -149,12 +157,13 @@ class TestCourseListing(ModuleStoreTestCase):
'run': '2015_T2'
})
self.assertEqual(response.status_code, 400)
with self.assertRaises(InvalidOrganizationException):
get_organization_by_short_name("orgX")
data = parse_json(response)
self.assertIn(u'Organization you selected does not exist in the system', data['error'])
@patch.dict('django.conf.settings.FEATURES', {'ORGANIZATIONS_APP': True})
@ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
def test_course_creation_with_org_in_system(self, store):
@ddt.data(True, False)
def test_course_creation_for_known_organization(self, organizations_autocreate):
"""
Tests course creation workflow when course organization exist in system.
"""
@@ -163,7 +172,7 @@ class TestCourseListing(ModuleStoreTestCase):
'short_name': 'orgX',
'description': 'Testing Organization Description',
})
with modulestore().default_store(store):
with override_settings(ORGANIZATIONS_AUTOCREATE=organizations_autocreate):
response = self.client.ajax_post(self.course_create_rerun_url, {
'org': 'orgX',
'number': 'CS101',

2
cms/djangoapps/contentstore/tests/test_tasks.py
View File

@@ -127,7 +127,7 @@ class RerunCourseTaskTestCase(CourseTestCase):
old_course_id = str(old_course_key)
new_course_id = str(new_course_key)
organization = OrganizationFactory()
organization = OrganizationFactory(short_name=old_course_key.org)
OrganizationCourse.objects.create(course_id=old_course_id, organization=organization)
restricted_course = RestrictedCourse.objects.create(course_key=self.course.id)

16
cms/djangoapps/contentstore/views/course.py
View File

@@ -28,6 +28,8 @@ from milestones import api as milestones_api
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import BlockUsageLocator
from organizations.api import add_organization_course, ensure_organization
from organizations.exceptions import InvalidOrganizationException
from six import text_type
from six.moves import filter
@@ -66,9 +68,6 @@ from common.djangoapps.util.milestones_helpers import (
set_prerequisite_courses
)
from openedx.core import toggles as core_toggles
from common.djangoapps.util.organizations_helpers import (
add_organization_course, get_organization_by_short_name, organizations_enabled
)
from common.djangoapps.util.string_utils import _has_non_ascii_characters
from common.djangoapps.xblock_django.api import deprecated_xblocks
from xmodule.contentstore.content import StaticContent
@@ -887,10 +886,13 @@ def create_new_course(user, org, number, run, fields):
Raises:
DuplicateCourseError: Course run already exists.
"""
org_data = get_organization_by_short_name(org)
if not org_data and organizations_enabled():
raise ValidationError(_('You must link this course to an organization in order to continue. Organization '
'you selected does not exist in the system, you will need to add it to the system'))
try:
org_data = ensure_organization(org)
except InvalidOrganizationException:
raise ValidationError(_(
'You must link this course to an organization in order to continue. Organization '
'you selected does not exist in the system, you will need to add it to the system'
))
store_for_new_course = modulestore().default_modulestore.get_modulestore_type()
new_course = create_new_course_in_store(store_for_new_course, user, org, number, run, fields)
add_organization_course(org_data, new_course.id)

2
cms/djangoapps/contentstore/views/organization.py
View File

@@ -5,9 +5,9 @@ from django.contrib.auth.decorators import login_required
from django.http import HttpResponse
from django.utils.decorators import method_decorator
from django.views.generic import View
from organizations.api import get_organizations
from openedx.core.djangolib.js_utils import dump_js_escaped_json
from common.djangoapps.util.organizations_helpers import get_organizations
class OrganizationListView(View):

5
cms/djangoapps/contentstore/views/tests/test_organizations.py
View File

@@ -5,16 +5,13 @@ import json
from django.test import TestCase
from django.urls import reverse
from mock import patch
from organizations.api import add_organization
from common.djangoapps.student.tests.factories import UserFactory
from common.djangoapps.util.organizations_helpers import add_organization
@patch.dict('django.conf.settings.FEATURES', {'ORGANIZATIONS_APP': True})
class TestOrganizationListing(TestCase):
"""Verify Organization listing behavior."""
@patch.dict('django.conf.settings.FEATURES', {'ORGANIZATIONS_APP': True})
def setUp(self):
super(TestOrganizationListing, self).setUp()
self.staff = UserFactory(is_staff=True)

3
cms/envs/bok_choy.py
View File

@@ -147,7 +147,8 @@ FEATURES['ENABLE_COURSEWARE_INDEX'] = True
FEATURES['ENABLE_LIBRARY_INDEX'] = True
FEATURES['ENABLE_CONTENT_LIBRARY_INDEX'] = False
FEATURES['ORGANIZATIONS_APP'] = True
ORGANIZATIONS_AUTOCREATE = False
SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
# Path at which to store the mock index
MOCK_SEARCH_BACKING_FILE = (

25
cms/envs/common.py
View File

@@ -293,8 +293,6 @@ FEATURES = {
# Special Exams, aka Timed and Proctored Exams
'ENABLE_SPECIAL_EXAMS': False,
'ORGANIZATIONS_APP': False,
# Show the language selector in the header
'SHOW_HEADER_LANGUAGE_SELECTOR': False,
@@ -1513,6 +1511,9 @@ INSTALLED_APPS = [
'openedx.core.djangoapps.content.learning_sequences.apps.LearningSequencesConfig',
'ratelimitbackend',
# Database-backed Organizations App (http://github.com/edx/edx-organizations)
'organizations',
]
@@ -1641,9 +1642,6 @@ OPTIONAL_APPS = (
# edxval
('edxval', 'openedx.core.djangoapps.content.course_overviews.apps.CourseOverviewsConfig'),
# Organizations App (http://github.com/edx/edx-organizations)
('organizations', None),
# Enterprise App (http://github.com/edx/edx-enterprise)
('enterprise', None),
('consent', None),
@@ -2294,3 +2292,20 @@ VERIFY_STUDENT = {
# The variable represents the window within which a verification is considered to be "expiring soon."
"EXPIRING_SOON_WINDOW": 28,
}
######################## Organizations ########################
# .. toggle_name: ORGANIZATIONS_AUTOCREATE
# .. toggle_implementation: DjangoSetting
# .. toggle_default: True
# .. toggle_description: When enabled, creating a course run or content library with
# an "org slug" that does not map to an Organization in the database will trigger the
# creation of a new Organization, with its name and short_name set to said org slug.
# When disabled, creation of such content with an unknown org slug will instead
# result in a validation error.
# If you want the Organization table to be an authoritative information source in
# Studio, then disable this; however, if you want the table to just be a reflection of
# the orgs referenced in Studio content, then leave it enabled.
# .. toggle_use_cases: open_edx
# .. toggle_creation_date: 2020-11-02
ORGANIZATIONS_AUTOCREATE = True

11
cms/envs/devstack.py
View File

@@ -118,12 +118,11 @@ def should_show_debug_toolbar(request):
FEATURES['MILESTONES_APP'] = True
########################### ORGANIZATIONS #################################
# This is disabled for Devstack Studio for developer convenience.
# If it were enabled, then users would not be able to create course runs
# with any arbritrary org slug -- they would have to first make sure that
# the organization exists in the Organization table.
# Note that some production environments (such as studio.edx.org) do enable this flag.
FEATURES['ORGANIZATIONS_APP'] = False
# Although production studio.edx.org disables `ORGANIZATIONS_AUTOCREATE`,
# we purposefully leave auto-creation enabled in Devstack Studio for developer
# convenience, allowing devs to create test courses for any organization
# without having to first manually create said organizations in the admin panel.
ORGANIZATIONS_AUTOCREATE = True
################################ ENTRANCE EXAMS ################################
FEATURES['ENTRANCE_EXAMS'] = True

106
common/djangoapps/util/organizations_helpers.py
View File

@@ -1,106 +0,0 @@
"""
Utility library for working with the edx-organizations app
"""
from django.conf import settings
from django.db.utils import DatabaseError
def add_organization(organization_data):
"""
Client API operation adapter/wrapper
"""
if not organizations_enabled():
return None
from organizations import api as organizations_api
return organizations_api.add_organization(organization_data=organization_data)
def add_organization_course(organization_data, course_id):
"""
Client API operation adapter/wrapper
"""
if not organizations_enabled():
return None
from organizations import api as organizations_api
return organizations_api.add_organization_course(organization_data=organization_data, course_key=course_id)
def get_organization(organization_id):
"""
Client API operation adapter/wrapper
"""
if not organizations_enabled():
return []
from organizations import api as organizations_api
return organizations_api.get_organization(organization_id)
def get_organization_by_short_name(organization_short_name):
"""
Client API operation adapter/wrapper
"""
if not organizations_enabled():
return None
from organizations import api as organizations_api
from organizations.exceptions import InvalidOrganizationException
try:
return organizations_api.get_organization_by_short_name(organization_short_name)
except InvalidOrganizationException:
return None
def get_organizations():
"""
Client API operation adapter/wrapper
"""
if not organizations_enabled():
return []
from organizations import api as organizations_api
# Due to the way unit tests run for edx-platform, models are not yet available at the time
# of Django admin form instantiation. This unfortunately results in an invocation of the following
# workflow, because the test configuration is (correctly) configured to exercise the application
# The good news is that this case does not manifest in the Real World, because migrations have
# been run ahead of application instantiation and the flag set only when that is truly the case.
try:
return organizations_api.get_organizations()
except DatabaseError:
return []
def get_organization_courses(organization_id):
"""
Client API operation adapter/wrapper
"""
if not organizations_enabled():
return []
from organizations import api as organizations_api
return organizations_api.get_organization_courses(organization_id)
def get_course_organizations(course_id):
"""
Client API operation adapter/wrapper
"""
if not organizations_enabled():
return []
from organizations import api as organizations_api
return organizations_api.get_course_organizations(course_id)
def get_course_organization_id(course_id):
"""
Returns organization id for course or None if the course is not linked to an org
"""
course_organization = get_course_organizations(course_id)
if course_organization:
return course_organization[0]['id']
return None
def organizations_enabled():
"""
Returns boolean indication if organizations app is enabled on not.
"""
return settings.FEATURES.get('ORGANIZATIONS_APP', False)

79
common/djangoapps/util/tests/test_organizations_helpers.py
View File

@@ -1,79 +0,0 @@
"""
Tests for the organizations helpers library, which is the integration point for the edx-organizations API
"""
import six
from mock import patch
from common.djangoapps.util import organizations_helpers
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
@patch.dict('django.conf.settings.FEATURES', {'ORGANIZATIONS_APP': False})
class OrganizationsHelpersTestCase(ModuleStoreTestCase):
"""
Main test suite for Organizations API client library
"""
CREATE_USER = False
def setUp(self):
"""
Test case scaffolding
"""
super(OrganizationsHelpersTestCase, self).setUp()
self.course = CourseFactory.create()
self.organization = {
'name': 'Test Organization',
'short_name': 'Orgx',
'description': 'Testing Organization Helpers Library',
}
def test_get_organization_returns_none_when_app_disabled(self):
response = organizations_helpers.get_organization(1)
self.assertEqual(len(response), 0)
def test_get_organizations_returns_none_when_app_disabled(self):
response = organizations_helpers.get_organizations()
self.assertEqual(len(response), 0)
def test_get_organization_courses_returns_none_when_app_disabled(self):
response = organizations_helpers.get_organization_courses(1)
self.assertEqual(len(response), 0)
def test_get_course_organizations_returns_none_when_app_disabled(self):
response = organizations_helpers.get_course_organizations(six.text_type(self.course.id))
self.assertEqual(len(response), 0)
def test_add_organization_returns_none_when_app_disabled(self):
response = organizations_helpers.add_organization(organization_data=self.organization)
self.assertIsNone(response)
def test_add_organization_course_returns_none_when_app_disabled(self):
response = organizations_helpers.add_organization_course(self.organization, self.course.id)
self.assertIsNone(response)
def test_get_organization_by_short_name_when_app_disabled(self):
"""
Tests get_organization_by_short_name api when app is disabled.
"""
response = organizations_helpers.get_organization_by_short_name(self.organization['short_name'])
self.assertIsNone(response)
@patch.dict('django.conf.settings.FEATURES', {'ORGANIZATIONS_APP': True})
def test_get_organization_by_short_name_when_app_enabled(self):
"""
Tests get_organization_by_short_name api when app is enabled.
"""
response = organizations_helpers.add_organization(organization_data=self.organization)
self.assertIsNotNone(response['id'])
response = organizations_helpers.get_organization_by_short_name(self.organization['short_name'])
self.assertIsNotNone(response['id'])
# fetch non existing org
response = organizations_helpers.get_organization_by_short_name('non_existing')
self.assertIsNone(response)

17
common/lib/xmodule/xmodule/modulestore/mixed.py
View File

@@ -327,6 +327,23 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
courses[course_id] = course
return list(courses.values())
def get_library_keys(self):
"""
Returns a list of all unique content library keys in the mixed
modulestore.
Returns: list[LibraryLocator]
"""
all_library_keys = set()
for store in self.modulestores:
if not hasattr(store, 'get_library_keys'):
continue
all_library_keys |= set(
self._clean_locator_for_mapping(library_key)
for library_key in store.get_library_keys()
)
return list(all_library_keys)
@strip_key
def get_library_summaries(self, **kwargs):
"""

13
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
View File

@@ -1073,6 +1073,19 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
)
return courses_summaries
def get_library_keys(self):
"""
Returns a list of all unique content library keys in the Split
modulestore.
Returns: list[LibraryLocator]
"""
return list({
self._create_library_locator(library_index, branch=None)
for library_index
in self.find_matching_course_indexes(branch="library")
})
@autoretry_read()
def get_library_summaries(self, **kwargs):
"""

7
common/lib/xmodule/xmodule/modulestore/tests/test_libraries.py
View File

@@ -145,6 +145,13 @@ class TestLibraries(MixedSplitTestCase):
result = self.store.get_library(LibraryLocator("non", "existent"))
self.assertEqual(result, None)
def test_get_library_keys(self):
""" Test get_library_keys() """
libraries = [LibraryFactory.create(modulestore=self.store) for _ in range(3)]
lib_keys_expected = {lib.location.library_key for lib in libraries}
lib_keys_actual = set(self.store.get_library_keys())
assert lib_keys_expected == lib_keys_actual
def test_get_libraries(self):
""" Test get_libraries() """
libraries = [LibraryFactory.create(modulestore=self.store) for _ in range(3)]

8
import_shims/lms/util/organizations_helpers.py
View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('util.organizations_helpers', 'common.djangoapps.util.organizations_helpers')
from common.djangoapps.util.organizations_helpers import *

8
import_shims/lms/util/tests/test_organizations_helpers.py
View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('util.tests.test_organizations_helpers', 'common.djangoapps.util.tests.test_organizations_helpers')
from common.djangoapps.util.tests.test_organizations_helpers import *

8
import_shims/studio/util/organizations_helpers.py
View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('util.organizations_helpers', 'common.djangoapps.util.organizations_helpers')
from common.djangoapps.util.organizations_helpers import *

8
import_shims/studio/util/tests/test_organizations_helpers.py
View File

@@ -1,8 +0,0 @@
"""Deprecated import support. Auto-generated by import_shims/generate_shims.sh."""
# pylint: disable=redefined-builtin,wrong-import-position,wildcard-import,useless-suppression,line-too-long
from import_shims.warn import warn_deprecated_import
warn_deprecated_import('util.tests.test_organizations_helpers', 'common.djangoapps.util.tests.test_organizations_helpers')
from common.djangoapps.util.tests.test_organizations_helpers import *

2
lms/djangoapps/certificates/admin.py
View File

@@ -10,6 +10,7 @@ from django import forms
from django.conf import settings
from django.contrib import admin
from django.utils.safestring import mark_safe
from organizations.api import get_organizations
from lms.djangoapps.certificates.models import (
CertificateGenerationConfiguration,
@@ -19,7 +20,6 @@ from lms.djangoapps.certificates.models import (
CertificateTemplateAsset,
GeneratedCertificate
)
from common.djangoapps.util.organizations_helpers import get_organizations
class CertificateTemplateForm(forms.ModelForm):

2
lms/djangoapps/certificates/api.py
View File

@@ -15,6 +15,7 @@ from django.urls import reverse
from eventtracking import tracker
from opaque_keys.edx.django.models import CourseKeyField
from opaque_keys.edx.keys import CourseKey
from organizations.api import get_course_organization_id
from lms.djangoapps.branding import api as branding_api
from lms.djangoapps.certificates.models import (
@@ -32,7 +33,6 @@ from lms.djangoapps.certificates.queue import XQueueCertInterface
from lms.djangoapps.instructor.access import list_with_level
from openedx.core.djangoapps.certificates.api import certificates_viewable_for_course
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
from common.djangoapps.util.organizations_helpers import get_course_organization_id
from xmodule.modulestore.django import modulestore
log = logging.getLogger("edx.certificate")

8
lms/djangoapps/certificates/tests/test_webview_views.py
View File

@@ -5,6 +5,7 @@
import datetime
import json
from collections import OrderedDict
from urllib.parse import urlencode
from uuid import uuid4
import ddt
@@ -14,7 +15,7 @@ from django.test.client import Client, RequestFactory
from django.test.utils import override_settings
from django.urls import reverse
from mock import patch
from urllib.parse import urlencode
from organizations import api as organizations_api
from common.djangoapps.course_modes.models import CourseMode
from edx_toggles.toggles import WaffleSwitch
@@ -53,7 +54,6 @@ from openedx.core.lib.tests.assertions.events import assert_event_matches
from common.djangoapps.student.roles import CourseStaffRole
from common.djangoapps.student.tests.factories import CourseEnrollmentFactory, UserFactory
from common.djangoapps.track.tests import EventTrackingTestCase
from common.djangoapps.util import organizations_helpers as organizations_api
from common.djangoapps.util.date_utils import strftime_localized
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
@@ -413,7 +413,7 @@ class CertificatesViewsTests(CommonCertificatesTestCase, CacheIsolationTestCase)
'logo': '/logo_test1.png/'
}
test_org = organizations_api.add_organization(organization_data=test_organization_data)
organizations_api.add_organization_course(organization_data=test_org, course_id=six.text_type(self.course.id))
organizations_api.add_organization_course(organization_data=test_org, course_key=six.text_type(self.course.id))
self._add_course_certificates(count=1, signatory_count=1, is_active=True)
test_url = get_certificate_url(
user_id=self.user.id,
@@ -483,7 +483,7 @@ class CertificatesViewsTests(CommonCertificatesTestCase, CacheIsolationTestCase)
'logo': '/logo_test1.png'
}
test_org = organizations_api.add_organization(organization_data=test_organization_data)
organizations_api.add_organization_course(organization_data=test_org, course_id=six.text_type(self.course.id))
organizations_api.add_organization_course(organization_data=test_org, course_key=six.text_type(self.course.id))
self._add_course_certificates(count=1, signatory_count=1, is_active=True)
badge_class = get_completion_badge(course_id=self.course_id, user=self.user)
BadgeAssertionFactory.create(

4
lms/djangoapps/certificates/views/webview.py
View File

@@ -20,6 +20,7 @@ from django.utils.encoding import smart_str
from eventtracking import tracker
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from organizations import api as organizations_api
from lms.djangoapps.badges.events.course_complete import get_completion_badge
from lms.djangoapps.badges.utils import badges_enabled
@@ -48,7 +49,6 @@ from openedx.core.djangoapps.lang_pref.api import get_closest_released_language
from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
from openedx.core.lib.courses import course_image_url
from common.djangoapps.student.models import LinkedInAddToProfileConfiguration
from common.djangoapps.util import organizations_helpers as organization_api
from common.djangoapps.util.date_utils import strftime_localized
from common.djangoapps.util.views import handle_500
@@ -428,7 +428,7 @@ def _update_organization_context(context, course):
"""
partner_long_name, organization_logo = None, None
partner_short_name = course.display_organization if course.display_organization else course.org
organizations = organization_api.get_course_organizations(course_id=course.id)
organizations = organizations_api.get_course_organizations(course_key=course.id)
if organizations:
#TODO Need to add support for multiple organizations, Currently we are interested in the first one.
organization = organizations[0]

9
lms/envs/common.py
View File

@@ -442,9 +442,6 @@ FEATURES = {
# Milestones application flag
'MILESTONES_APP': False,
# Organizations application flag
'ORGANIZATIONS_APP': False,
# Prerequisite courses feature flag
'ENABLE_PREREQUISITE_COURSES': False,
@@ -2735,6 +2732,9 @@ INSTALLED_APPS = [
'openedx.core.djangoapps.content.learning_sequences.apps.LearningSequencesConfig',
'ratelimitbackend',
# Database-backed Organizations App (http://github.com/edx/edx-organizations)
'organizations',
]
######################### CSRF #########################################
@@ -3349,9 +3349,6 @@ OPTIONAL_APPS = [
# edxval
('edxval', 'openedx.core.djangoapps.content.course_overviews.apps.CourseOverviewsConfig'),
# Organizations App (http://github.com/edx/edx-organizations)
('organizations', None),
# Enterprise Apps (http://github.com/edx/edx-enterprise)
('enterprise', None),
('consent', None),

3
lms/envs/devstack.py
View File

@@ -151,9 +151,6 @@ FEATURES['PREVENT_CONCURRENT_LOGINS'] = False
########################### Milestones #################################
FEATURES['MILESTONES_APP'] = True
########################### Organizations #################################
FEATURES['ORGANIZATIONS_APP'] = True
########################### Entrance Exams #################################
FEATURES['ENTRANCE_EXAMS'] = True

3
lms/envs/devstack_decentralized.py
View File

@@ -117,9 +117,6 @@ FEATURES['PREVENT_CONCURRENT_LOGINS'] = False
########################### Milestones #################################
FEATURES['MILESTONES_APP'] = True
########################### Milestones #################################
FEATURES['ORGANIZATIONS_APP'] = True
########################### Entrance Exams #################################
FEATURES['ENTRANCE_EXAMS'] = True

3
lms/envs/test.py
View File

@@ -468,9 +468,6 @@ FEATURES['ENABLE_LTI_PROVIDER'] = True
INSTALLED_APPS.append('lms.djangoapps.lti_provider.apps.LtiProviderConfig')
AUTHENTICATION_BACKENDS.append('lms.djangoapps.lti_provider.users.LtiBackend')
# ORGANIZATIONS
FEATURES['ORGANIZATIONS_APP'] = True
# Financial assistance page
FEATURES['ENABLE_FINANCIAL_ASSISTANCE_FORM'] = True

15
openedx/core/djangoapps/content_libraries/views.py
View File

@@ -10,6 +10,8 @@ from django.shortcuts import get_object_or_404
from django.utils.translation import ugettext as _
import edx_api_doc_tools as apidocs
from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2
from organizations.api import ensure_organization
from organizations.exceptions import InvalidOrganizationException
from organizations.models import Organization
from rest_framework import status
from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError
@@ -156,14 +158,17 @@ class LibraryRootView(APIView):
# definitions elsewhere.
data['library_type'] = data.pop('type')
data['library_license'] = data.pop('license')
# Get the organization short_name out of the "key.org" pseudo-field that the serializer added:
org_name = data["key"]["org"]
# Move "slug" out of the "key.slug" pseudo-field that the serializer added:
data["slug"] = data.pop("key")["slug"]
# Get the organization short_name out of the "key.org" pseudo-field that the serializer added:
org_name = data["key"]["org"]
try:
org = Organization.objects.get(short_name=org_name)
except Organization.DoesNotExist:
raise ValidationError(detail={"org": "No such organization '{}' found.".format(org_name)})
ensure_organization(org_name)
except InvalidOrganizationException:
raise ValidationError(
detail={"org": "No such organization '{}' found.".format(org_name)}
)
org = Organization.objects.get(short_name=org_name)
try:
result = api.create_library(org=org, **data)
except api.LibraryAlreadyExists: