From 14f90eba96e81b771f43c8a425694cc5946d8265 Mon Sep 17 00:00:00 2001 From: Yagnesh Date: Sat, 17 Jun 2023 02:31:07 +0530 Subject: [PATCH 1/7] feat: BLOCK_STRUCTURES_SETTINGS['PRUNING_ACTIVE'] toggle removed depr31 --- cms/envs/common.py | 3 --- cms/envs/devstack-experimental.yml | 1 - cms/envs/test.py | 2 -- lms/envs/common.py | 16 ---------------- lms/envs/devstack-experimental.yml | 1 - lms/envs/test.py | 2 -- .../djangoapps/content/block_structure/models.py | 3 --- .../content/block_structure/tests/test_models.py | 1 - 8 files changed, 29 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index 41ef3f60c2..d3660edc63 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -155,9 +155,6 @@ BLOCK_STRUCTURES_SETTINGS = dict( # Maximum number of retries per task. TASK_MAX_RETRIES=5, - - # Backend storage options - PRUNING_ACTIVE=False, ) ############################ FEATURE CONFIGURATION ############################# diff --git a/cms/envs/devstack-experimental.yml b/cms/envs/devstack-experimental.yml index a25a28debc..46e6af3894 100644 --- a/cms/envs/devstack-experimental.yml +++ b/cms/envs/devstack-experimental.yml @@ -43,7 +43,6 @@ BLOCKSTORE_API_URL: http://localhost:18250/api/v1 BLOCKSTORE_PUBLIC_URL_ROOT: http://localhost:18250 BLOCK_STRUCTURES_SETTINGS: COURSE_PUBLISH_TASK_DELAY: 30 - PRUNING_ACTIVE: false TASK_DEFAULT_RETRY_DELAY: 30 TASK_MAX_RETRIES: 5 BRANCH_IO_KEY: '' diff --git a/cms/envs/test.py b/cms/envs/test.py index e856034d08..118b3f1b0c 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -95,8 +95,6 @@ STATICFILES_DIRS += [ STATICFILES_STORAGE = 'pipeline.storage.NonPackagingPipelineStorage' STATIC_URL = "/static/" -BLOCK_STRUCTURES_SETTINGS['PRUNING_ACTIVE'] = True - # Update module store settings per defaults for tests update_module_store_settings( MODULESTORE, diff --git a/lms/envs/common.py b/lms/envs/common.py index af6c93f519..e4c213e0da 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -2867,22 +2867,6 @@ BLOCK_STRUCTURES_SETTINGS = dict( # For more information, check https://github.com/openedx/edx-platform/pull/13388 and # https://github.com/openedx/edx-platform/pull/14571. TASK_MAX_RETRIES=5, - - # .. toggle_name: BLOCK_STRUCTURES_SETTINGS['PRUNING_ACTIVE'] - # .. toggle_implementation: DjangoSetting - # .. toggle_default: False - # .. toggle_description: When `True`, only a specified number of versions of block structure - # files are kept for each structure, and the rest are cleaned up. The number of versions that - # are kept can be specified in the `BlockStructureConfiguration`, which can be edited in - # Django Admin. The default number of versions that are kept is `5`. - # .. toggle_warning: This toggle will likely be deprecated and removed. - # .. toggle_use_cases: temporary - # .. toggle_creation_date: 2018-03-22 - # .. toggle_target_removal_date: 2018-06-22 - # .. toggle_tickets: https://github.com/openedx/edx-platform/pull/14571, - # https://github.com/openedx/edx-platform/pull/17760, - # https://openedx.atlassian.net/browse/DEPR-146 - PRUNING_ACTIVE=False, ) ################################ Bulk Email ################################### diff --git a/lms/envs/devstack-experimental.yml b/lms/envs/devstack-experimental.yml index 0e554cdd9a..71d66b2af4 100644 --- a/lms/envs/devstack-experimental.yml +++ b/lms/envs/devstack-experimental.yml @@ -61,7 +61,6 @@ BLOCKSTORE_API_URL: http://localhost:18250/api/v1 BLOCKSTORE_PUBLIC_URL_ROOT: http://localhost:18250 BLOCK_STRUCTURES_SETTINGS: COURSE_PUBLISH_TASK_DELAY: 30 - PRUNING_ACTIVE: false TASK_DEFAULT_RETRY_DELAY: 30 TASK_MAX_RETRIES: 5 BRANCH_IO_KEY: '' diff --git a/lms/envs/test.py b/lms/envs/test.py index cb55cfcfb8..4cc08fdadd 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -327,8 +327,6 @@ FILE_UPLOAD_HANDLERS = [ 'django.core.files.uploadhandler.TemporaryFileUploadHandler', ] -BLOCK_STRUCTURES_SETTINGS['PRUNING_ACTIVE'] = True - ########################### Server Ports ################################### # These ports are carefully chosen so that if the browser needs to diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index b183bb5949..8fda8ad360 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -255,9 +255,6 @@ class BlockStructureModel(TimeStampedModel): """ Deletes previous file versions for data_usage_key. """ - if not settings.BLOCK_STRUCTURES_SETTINGS.get('PRUNING_ACTIVE', False): - return - if num_to_keep is None: num_to_keep = config.num_versions_to_keep() diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_models.py b/openedx/core/djangoapps/content/block_structure/tests/test_models.py index 08916151e7..443c722aed 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_models.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_models.py @@ -80,7 +80,6 @@ class BlockStructureModelTestCase(TestCase): return bsm @patch('openedx.core.djangoapps.content.block_structure.models.log') - @patch.dict(settings.BLOCK_STRUCTURES_SETTINGS, {'PRUNING_ACTIVE': False}) def test_update_or_create(self, mock_log): serialized_data = 'initial data' From f0112deb2c692890583bcba97c79c484e65e59ca Mon Sep 17 00:00:00 2001 From: Yagnesh1998 <127923546+Yagnesh1998@users.noreply.github.com> Date: Thu, 22 Jun 2023 15:17:09 +0530 Subject: [PATCH 2/7] feat: Update test_models.py --- .../core/djangoapps/content/block_structure/tests/test_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_models.py b/openedx/core/djangoapps/content/block_structure/tests/test_models.py index 443c722aed..39bb68b094 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_models.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_models.py @@ -9,7 +9,6 @@ from unittest.mock import Mock, patch import ddt import pytest -from django.conf import settings from django.core.exceptions import SuspiciousOperation from django.test import TestCase from django.utils.timezone import now From ba01ca9b2ad380f75978bbe16e7c0ecc5d81bb63 Mon Sep 17 00:00:00 2001 From: Yagnesh1998 <127923546+Yagnesh1998@users.noreply.github.com> Date: Fri, 4 Aug 2023 10:28:28 +0530 Subject: [PATCH 3/7] feat: Update models.py --- .../core/djangoapps/content/block_structure/models.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index 8fda8ad360..9925701a63 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -262,14 +262,7 @@ class BlockStructureModel(TimeStampedModel): all_files_by_date = sorted(cls._get_all_files(data_usage_key)) files_to_delete = all_files_by_date[:-num_to_keep] if num_to_keep > 0 else all_files_by_date # lint-amnesty, pylint: disable=invalid-unary-operand-type cls._delete_files(files_to_delete) - log.info( - 'BlockStructure: Deleted %d out of total %d files in store; data_usage_key: %s, num_to_keep: %d.', - len(files_to_delete), - len(all_files_by_date), - data_usage_key, - num_to_keep, - ) - + except Exception: # pylint: disable=broad-except log.exception('BlockStructure: Exception when deleting old files; data_usage_key: %s.', data_usage_key) From e7e69d5cd7043a9ef00c98a37f9734b2887f2078 Mon Sep 17 00:00:00 2001 From: Yagnesh1998 <127923546+Yagnesh1998@users.noreply.github.com> Date: Fri, 4 Aug 2023 10:33:49 +0530 Subject: [PATCH 4/7] feat: Update models.py remove white space. --- openedx/core/djangoapps/content/block_structure/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index 9925701a63..285deafa7d 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -262,7 +262,6 @@ class BlockStructureModel(TimeStampedModel): all_files_by_date = sorted(cls._get_all_files(data_usage_key)) files_to_delete = all_files_by_date[:-num_to_keep] if num_to_keep > 0 else all_files_by_date # lint-amnesty, pylint: disable=invalid-unary-operand-type cls._delete_files(files_to_delete) - except Exception: # pylint: disable=broad-except log.exception('BlockStructure: Exception when deleting old files; data_usage_key: %s.', data_usage_key) From b2286f4b5d7e8e8eebf74313965ae46bc8b31414 Mon Sep 17 00:00:00 2001 From: Yagnesh1998 <127923546+Yagnesh1998@users.noreply.github.com> Date: Tue, 8 Aug 2023 17:37:04 +0530 Subject: [PATCH 5/7] feat: Update models.py --- .../djangoapps/content/block_structure/models.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index 285deafa7d..6bb1810839 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -234,11 +234,11 @@ class BlockStructureModel(TimeStampedModel): with _storage_error_handling(bs_model, operation): bs_model.data.save('', ContentFile(serialized_data)) - cls._log(bs_model, operation, serialized_data) - if not created: cls._prune_files(data_usage_key) - + + cls._log(bs_model, operation, serialized_data) + return bs_model, created def __str__(self): @@ -262,6 +262,14 @@ class BlockStructureModel(TimeStampedModel): all_files_by_date = sorted(cls._get_all_files(data_usage_key)) files_to_delete = all_files_by_date[:-num_to_keep] if num_to_keep > 0 else all_files_by_date # lint-amnesty, pylint: disable=invalid-unary-operand-type cls._delete_files(files_to_delete) + log.info( + 'BlockStructure: Deleted %d out of total %d files in store; data_usage_key: %s, num_to_keep: %d.', + len(files_to_delete), + len(all_files_by_date), + data_usage_key, + num_to_keep, + ) + except Exception: # pylint: disable=broad-except log.exception('BlockStructure: Exception when deleting old files; data_usage_key: %s.', data_usage_key) From 0fae4c975bd78b93bd12e5e7f8998fb229b530fd Mon Sep 17 00:00:00 2001 From: Yagnesh1998 <127923546+Yagnesh1998@users.noreply.github.com> Date: Tue, 8 Aug 2023 17:50:12 +0530 Subject: [PATCH 6/7] feat: Update models.py removed white space --- openedx/core/djangoapps/content/block_structure/models.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index 6bb1810839..4be1092339 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -236,9 +236,7 @@ class BlockStructureModel(TimeStampedModel): if not created: cls._prune_files(data_usage_key) - cls._log(bs_model, operation, serialized_data) - return bs_model, created def __str__(self): @@ -268,8 +266,7 @@ class BlockStructureModel(TimeStampedModel): len(all_files_by_date), data_usage_key, num_to_keep, - ) - + ) except Exception: # pylint: disable=broad-except log.exception('BlockStructure: Exception when deleting old files; data_usage_key: %s.', data_usage_key) From dd191e94004ab80966d9b2de85d231d1396c78d3 Mon Sep 17 00:00:00 2001 From: Yagnesh1998 <127923546+Yagnesh1998@users.noreply.github.com> Date: Tue, 8 Aug 2023 18:12:01 +0530 Subject: [PATCH 7/7] feat: Update models.py --- openedx/core/djangoapps/content/block_structure/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index 4be1092339..4872e09346 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -266,7 +266,7 @@ class BlockStructureModel(TimeStampedModel): len(all_files_by_date), data_usage_key, num_to_keep, - ) + ) except Exception: # pylint: disable=broad-except log.exception('BlockStructure: Exception when deleting old files; data_usage_key: %s.', data_usage_key)