fix issues with migrate schema response task deletion

Author: Johnetordoff

api_tests/users/views/test_user_settings.py

@@ -267,10 +267,10 @@ def test_post_invalid_password(self, app, url, user_one, csrf_token):
         res = app.post_json_api(url, payload, expect_errors=True, headers={'X-THROTTLE-TOKEN': 'test-token', 'X-CSRFToken': csrf_token})
         assert res.status_code == 400
 
-    def test_throrrle(self, app, url, user_one):
+    def test_throttle(self, app, url, user_one):
         encoded_email = urllib.parse.quote(user_one.email)
         url = f'{url}?email={encoded_email}'
-        res = app.get(url)
+        app.get(url)
         user_one.reload()
         payload = {
             'data': {

osf/management/commands/migrate_notifications.py

@@ -4,7 +4,7 @@
 from osf.models.notifications import NotificationSubscriptionLegacy
 from django.core.management.base import BaseCommand
 from django.db import transaction
-from osf.management.commands.local_setup.populate_notification_types import populate_notification_types
+from osf.management.commands.populate_notification_types import populate_notification_types
 
 logger = logging.getLogger(__name__)
 

osf/management/commands/migrate_registration_responses.py

@@ -0,0 +1,173 @@
+import datetime
+import logging
+
+from django.core.management.base import BaseCommand
+from django.apps import apps
+from tqdm import tqdm
+
+from framework.celery_tasks import app as celery_app
+from framework import sentry
+
+from osf.exceptions import SchemaBlockConversionError
+from osf.utils.registrations import flatten_registration_metadata
+
+logger = logging.getLogger(__name__)
+
+# because Registrations and DraftRegistrations are different
+def get_nested_responses(registration_or_draft, schema_id):
+    nested_responses = getattr(
+        registration_or_draft,
+        'registration_metadata',
+        None,
+    )
+    if nested_responses is None:
+        registered_meta = registration_or_draft.registered_meta or {}
+        nested_responses = registered_meta.get(schema_id, None)
+    return nested_responses
+
+# because Registrations and DraftRegistrations are different
+def get_registration_schema(registration_or_draft):
+    schema = getattr(registration_or_draft, 'registration_schema', None)
+    if schema is None:
+        schema = registration_or_draft.registered_schema.first()
+    return schema
+
+def migrate_registrations(dry_run, rows='all', AbstractNodeModel=None):
+    """
+    Loops through registrations whose registration_responses have not been migrated,
+    and pulls this information from the "registered_meta" and flattens it, with
+    keys being the "registration_response_key"s and values being the most deeply
+    nested user response in registered_meta
+    """
+    if AbstractNodeModel is None:
+        AbstractNodeModel = apps.get_model('osf', 'abstractnode')
+
+    registrations = AbstractNodeModel.objects.filter(
+        type='osf.registration',
+    ).exclude(
+        registration_responses_migrated=True,
+    )
+    return migrate_responses(AbstractNodeModel, registrations, 'registrations', dry_run, rows)
+
+def migrate_draft_registrations(dry_run, rows='all', DraftRegistrationModel=None):
+    """
+    Populates a subset of draft_registration.registration_responses, and corresponding
+    draft_registration.registration_responses_migrated.
+    :params dry_run
+    :params rows
+    """
+    if DraftRegistrationModel is None:
+        DraftRegistrationModel = apps.get_model('osf', 'draftregistration')
+
+    draft_registrations = DraftRegistrationModel.objects.exclude(
+        registration_responses_migrated=True
+    )
+    return migrate_responses(DraftRegistrationModel, draft_registrations, 'draft registrations', dry_run, rows)
+
+
+def migrate_responses(model, resources, resource_name, dry_run=False, rows='all'):
+    """
+    DRY method to be used to migrate both DraftRegistration.registration_responses
+    and Registration.registration_responses.
+    """
+    progress_bar = None
+    if rows == 'all':
+        logger.info(f'Migrating all {resource_name}.')
+    else:
+        resources = resources[:rows]
+        logger.info(f'Migrating up to {rows} {resource_name}.')
+        progress_bar = tqdm(total=rows)
+
+    successes_to_save = []
+    errors_to_save = []
+    for resource in resources:
+        try:
+            schema = get_registration_schema(resource)
+            resource.registration_responses = flatten_registration_metadata(
+                schema,
+                get_nested_responses(resource, schema._id),
+            )
+            resource.registration_responses_migrated = True
+            successes_to_save.append(resource)
+        except SchemaBlockConversionError as e:
+            resource.registration_responses_migrated = False
+            errors_to_save.append(resource)
+            logger.error(f'Unexpected/invalid nested data in resource: {resource} with error {e}')
+        if progress_bar:
+            progress_bar.update()
+
+    if progress_bar:
+        progress_bar.close()
+
+    success_count = len(successes_to_save)
+    error_count = len(errors_to_save)
+    total_count = success_count + error_count
+
+    if total_count == 0:
+        logger.info(f'No {resource_name} left to migrate.')
+        return total_count
+
+    logger.info(f'Successfully migrated {success_count} out of {total_count} {resource_name}.')
+    if error_count:
+        logger.warning(f'Encountered errors on {error_count} out of {total_count} {resource_name}.')
+        if not success_count:
+            sentry.log_message(f'`migrate_registration_responses` has only errors left ({error_count} errors)')
+
+    if dry_run:
+        logger.info('DRY RUN; discarding changes.')
+    else:
+        logger.info('Saving changes...')
+        model.objects.bulk_update(successes_to_save, fields=['registration_responses', 'registration_responses_migrated'])
+        model.objects.bulk_update(errors_to_save, fields=['registration_responses_migrated'])
+
+    return total_count
+
+
+@celery_app.task(name='management.commands.migrate_registration_responses')
+def migrate_registration_responses(dry_run=False, rows=5000):
+    script_start_time = datetime.datetime.now()
+    logger.info(f'Script started time: {script_start_time}')
+
+    draft_count = migrate_draft_registrations(dry_run, rows)
+    registration_count = migrate_registrations(dry_run, rows)
+
+    if draft_count == 0 and registration_count == 0:
+        logger.info('Migration complete! No more drafts or registrations need migrating.')
+        sentry.log_message('`migrate_registration_responses` command found nothing to migrate!')
+
+    script_finish_time = datetime.datetime.now()
+    logger.info(f'Script finished time: {script_finish_time}')
+    logger.info(f'Run time {script_finish_time - script_start_time}')
+
+
+class Command(BaseCommand):
+    help = """ Incrementally migrates DraftRegistration.registration_metadata
+    -> DraftRegistration.registration_responses, and Registration.registered_meta
+    -> Registration.registered_responses. registration_responses is a flattened version
+    of registration_metadata/registered_meta.
+
+    This will need to be run multiple times to migrate all records on prod.
+     """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--dry_run',
+            type=bool,
+            default=False,
+            help='Run queries but do not write files',
+        )
+        parser.add_argument(
+            '--rows',
+            type=int,
+            default=5000,
+            help='How many rows to process during this run',
+        )
+
+    # Management command handler
+    def handle(self, *args, **options):
+        dry_run = options['dry_run']
+        rows = options['rows']
+        if dry_run:
+            logger.info('DRY RUN')
+
+        migrate_registration_responses(dry_run, rows)

osf_tests/management_commands/test_migrate_preprint_affiliations.py

@@ -441,7 +441,6 @@ class CeleryConfig:
         'osf.management.commands.migrate_pagecounter_data',
         'osf.management.commands.migrate_deleted_date',
         'osf.management.commands.addon_deleted_date',
-        'osf.management.commands.migrate_registration_responses',
         'osf.management.commands.archive_registrations_on_IA'
         'osf.management.commands.sync_doi_metadata',
         'osf.management.commands.sync_collection_provider_indices',
@@ -693,9 +692,6 @@ class CeleryConfig:
             #   'task': 'management.commands.migrate_pagecounter_data',
             #   'schedule': crontab(minute=0, hour=7),  # Daily 2:00 a.m.
             # },
-            # 'migrate_registration_responses': {
-            #   'task': 'management.commands.migrate_registration_responses',
-            #   'schedule': crontab(minute=32, hour=7),  # Daily 2:32 a.m.
             # 'migrate_deleted_date': {
             #   'task': 'management.commands.migrate_deleted_date',
             #   'schedule': crontab(minute=0, hour=3),