From 9f42b5edfef08d8d9e2ef755ba70b4c0aac67feb Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 6 Feb 2020 16:55:18 -0500 Subject: [PATCH] BOM-939 Update pickle procotol version. Updating to the current highest number instead of making it the default which is highest know by the language. This is beacuse when we update between python versions if there is a new proctocol version, we don't want that to cause cache thrashing as we try to load things from the various caches. We can get into a situation where new machines pickle things with a newer versions of the procotocl, put them in the cache and then break older machines that don't know about the new protocol version. --- .../modulestore/split_mongo/mongo_connection.py | 2 +- openedx/core/lib/cache_utils.py | 2 +- openedx/core/lib/session_serializers.py | 17 +++++------------ 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py index 8ab776f53d..01ac12d58e 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py @@ -256,7 +256,7 @@ class CourseStructureCache(object): return None with TIMER.timer("CourseStructureCache.set", course_context) as tagger: - pickled_data = pickle.dumps(structure, 2) # Protocol can't be incremented until cache is cleared + pickled_data = pickle.dumps(structure, 4) # Protocol can't be incremented until cache is cleared tagger.measure('uncompressed_size', len(pickled_data)) # 1 = Fastest (slightly larger results) diff --git a/openedx/core/lib/cache_utils.py b/openedx/core/lib/cache_utils.py index b7d016fee7..10029cfc1a 100644 --- a/openedx/core/lib/cache_utils.py +++ b/openedx/core/lib/cache_utils.py @@ -153,7 +153,7 @@ class process_cached(object): # pylint: disable=invalid-name def zpickle(data): """Given any data structure, returns a zlib compressed pickled serialization.""" - return zlib.compress(pickle.dumps(data, 2)) # Keep this constant as we upgrade from python 2 to 3. + return zlib.compress(pickle.dumps(data, 4)) # Keep this constant as we upgrade from python 2 to 3. def zunpickle(zdata): diff --git a/openedx/core/lib/session_serializers.py b/openedx/core/lib/session_serializers.py index c1ff9ae51d..9a401c6152 100644 --- a/openedx/core/lib/session_serializers.py +++ b/openedx/core/lib/session_serializers.py @@ -7,16 +7,15 @@ import six class PickleV2Serializer(object): """ - Lock the pickle serializer to version 2 of the protocol - because we don't want python 2 to be able to read session - data written by python3 while both are running at the same - time in production. + Set the pickle protocol version explicitly because we don't want + to have session thrashing when we upgrade to newer versions of + python. Based on the PickleSerializer built into django: https://github.com/django/django/blob/master/django/contrib/sessions/serializers.py """ - protocol = 2 + protocol = 4 def dumps(self, obj): """ @@ -28,10 +27,4 @@ class PickleV2Serializer(object): """ Return a python object from pickled data. """ - if six.PY2: - # Params used below don't exist in python 2 - return pickle.loads(data) - else: - # See notes here about pickling python2 objects in python3 - # https://docs.python.org/3/library/pickle.html#pickle.Unpickler - return pickle.loads(data, encoding='latin1') # pylint: disable=unexpected-keyword-arg + return pickle.loads(data)