From b7b5400c3ebc1eb4fd0f004c99f495de59c381dc Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Thu, 18 Sep 2014 10:04:31 -0400 Subject: [PATCH] Upgrade pymongo LMS-11437 --- cms/djangoapps/contentstore/tests/utils.py | 4 ++-- .../xmodule/xmodule/modulestore/mongo/base.py | 10 ++++------ .../xmodule/xmodule/modulestore/mongo/draft.py | 6 +++++- .../modulestore/split_mongo/mongo_connection.py | 1 - .../xmodule/modulestore/split_mongo/split.py | 1 + .../xmodule/modulestore/tests/factories.py | 3 ++- .../modulestore/tests/test_mixed_modulestore.py | 16 ++++++++-------- .../xmodule/modulestore/tests/test_mongo.py | 1 + .../xmodule/modulestore/tests/test_publish.py | 5 ++--- requirements/edx/base.txt | 2 +- 10 files changed, 26 insertions(+), 23 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/utils.py b/cms/djangoapps/contentstore/tests/utils.py index 7417ad54a5..cc1b4b4717 100644 --- a/cms/djangoapps/contentstore/tests/utils.py +++ b/cms/djangoapps/contentstore/tests/utils.py @@ -72,10 +72,10 @@ class CourseTestCase(ModuleStoreTestCase): will be cleared out before each test case execution and deleted afterwards. """ - user_password = super(CourseTestCase, self).setUp() + self.user_password = super(CourseTestCase, self).setUp() self.client = AjaxEnabledTestClient() - self.client.login(username=self.user.username, password=user_password) + self.client.login(username=self.user.username, password=self.user_password) self.course = CourseFactory.create() diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 5568054bb6..6db5d42169 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -1137,7 +1137,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo ''' return self.get_course(location.course_key, depth) - def _update_single_item(self, location, update): + def _update_single_item(self, location, update, allow_not_found=False): """ Set update on the specified item, and raises ItemNotFoundError if the location doesn't exist @@ -1150,10 +1150,8 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo {'_id': location.to_deprecated_son()}, {'$set': update}, multi=False, - upsert=True, - # Must include this to avoid the django debug toolbar (which defines the deprecated "safe=False") - # from overriding our default value set in the init method. - safe=self.collection.safe + upsert=allow_not_found, + w=1, # wait until primary commits ) if result['n'] == 0: raise ItemNotFoundError(location) @@ -1205,7 +1203,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo if xblock.has_children: children = self._serialize_scope(xblock, Scope.children) payload.update({'definition.children': children['children']}) - self._update_single_item(xblock.scope_ids.usage_id, payload) + self._update_single_item(xblock.scope_ids.usage_id, payload, allow_not_found=allow_not_found) # update subtree edited info for ancestors # don't update the subtree info for descendants of the publish root for efficiency diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py index f72e36625e..cc904f5f6a 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py @@ -677,7 +677,11 @@ class DraftModuleStore(MongoModuleStore): # So, do not delete the child. It will be published when the new parent is published. pass - super(DraftModuleStore, self).update_item(item, user_id, isPublish=True, is_publish_root=is_root) + # update the published (not draft) item (ignoring that item is "draft"). The published + # may not exist; (if original_published is None); so, allow_not_found + super(DraftModuleStore, self).update_item( + item, user_id, isPublish=True, is_publish_root=is_root, allow_not_found=True + ) to_be_deleted.append(as_draft(item_location).to_deprecated_son()) # verify input conditions diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py index c21bc0f422..f5e8bd0105 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py @@ -303,4 +303,3 @@ class MongoConnection(object): ], unique=True ) - diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index 912c3b6a0a..39148ea516 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -196,6 +196,7 @@ class SplitBulkWriteMixin(BulkOperationsMixin): course_key.replace(org=None, course=None, run=None, branch=None) ] + # handle ignore case and general use return super(SplitBulkWriteMixin, self)._get_bulk_ops_record( course_key.replace(branch=None, version_guid=None), ignore_case ) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/factories.py b/common/lib/xmodule/xmodule/modulestore/tests/factories.py index b3dde849d3..b311e62c3a 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/factories.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/factories.py @@ -299,7 +299,8 @@ def check_mongo_calls(num_finds=0, num_sends=None): if num_sends is not None: with check_sum_of_calls( pymongo.message, - ['insert', 'update', 'delete'], + # mongo < 2.6 uses insert, update, delete and _do_batched_insert. >= 2.6 _do_batched_write + ['insert', 'update', 'delete', '_do_batched_write_command', '_do_batched_insert', ], num_sends, num_sends ): diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py index 70982a105c..c7e84876a6 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -660,12 +660,12 @@ class TestMixedModuleStore(CourseComparisonTest): # Draft # Find: find parents (definition.children query), get parent, get course (fill in run?), # find parents of the parent (course), get inheritance items, - # get errors, get item (to delete subtree), get inheritance again. + # get item (to delete subtree), get inheritance again. # Sends: delete item, update parent # Split # Find: active_versions, 2 structures (published & draft), definition (unnecessary) # Sends: updated draft and published structures and active_versions - @ddt.data(('draft', 8, 2), ('split', 4, 3)) + @ddt.data(('draft', 7, 2), ('split', 4, 3)) @ddt.unpack def test_delete_item(self, default_ms, max_find, max_send): """ @@ -690,12 +690,12 @@ class TestMixedModuleStore(CourseComparisonTest): # Draft: # queries: find parent (definition.children), count versions of item, get parent, count grandparents, - # inheritance items, draft item, draft child, get errors, inheritance + # inheritance items, draft item, draft child, inheritance # sends: delete draft vertical and update parent # Split: # queries: active_versions, draft and published structures, definition (unnecessary) # sends: update published (why?), draft, and active_versions - @ddt.data(('draft', 9, 2), ('split', 4, 3)) + @ddt.data(('draft', 8, 2), ('split', 4, 3)) @ddt.unpack def test_delete_private_vertical(self, default_ms, max_find, max_send): """ @@ -741,12 +741,12 @@ class TestMixedModuleStore(CourseComparisonTest): self.assertNotIn(vert_loc, course.children) # Draft: - # find: find parent (definition.children) 2x, find draft item, check error state, get inheritance items + # find: find parent (definition.children) 2x, find draft item, get inheritance items # send: one delete query for specific item # Split: # find: active_version & structure # send: update structure and active_versions - @ddt.data(('draft', 5, 1), ('split', 2, 2)) + @ddt.data(('draft', 4, 1), ('split', 2, 2)) @ddt.unpack def test_delete_draft_vertical(self, default_ms, max_find, max_send): """ @@ -1294,7 +1294,7 @@ class TestMixedModuleStore(CourseComparisonTest): self.assertEqual(len(self.store.get_courses_for_wiki('no_such_wiki')), 0) # Draft: - # Find: find vertical, find children, get last error + # Find: find vertical, find children # Sends: # 1. delete all of the published nodes in subtree # 2. insert vertical as published (deleted in step 1) w/ the deleted problems as children @@ -1303,7 +1303,7 @@ class TestMixedModuleStore(CourseComparisonTest): # Sends: # - insert structure # - write index entry - @ddt.data(('draft', 3, 6), ('split', 3, 2)) + @ddt.data(('draft', 2, 6), ('split', 3, 2)) @ddt.unpack def test_unpublish(self, default_ms, max_find, max_send): """ diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py index 031fd358a7..45e9537d7c 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py @@ -573,6 +573,7 @@ class TestMongoModuleStore(unittest.TestCase): 'published_by': published_by, }, }, + allow_not_found=True, ) # Retrieve the block and verify its fields diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py b/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py index 2dc83dd006..3f0a249631 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_publish.py @@ -98,13 +98,12 @@ class TestPublish(SplitWMongoCourseBoostrapper): # 12-15 get each ancestor (count then get): (2 x 2), # 16 then fail count of course parent (1) # 17 compute inheritance - # 18 get last error - # 19-20 get draft and published vert + # 18-19 get draft and published vert # Sends: # delete the subtree of drafts (1 call), # update the published version of each node in subtree (4 calls), # update the ancestors up to course (2 calls) - with check_mongo_calls(20, 7): + with check_mongo_calls(19, 7): self.draft_mongo.publish(item.location, self.user_id) # verify status diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 288f063876..71d8ba6abd 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -61,7 +61,7 @@ polib==1.0.3 pycrypto>=2.6 pygments==1.6 pygraphviz==1.1 -pymongo==2.4.1 +pymongo==2.7.2 pyparsing==2.0.1 python-memcached==1.48 python-openid==2.2.5