|
| 1 | +import logging |
| 2 | + |
| 3 | +from django.core.management.base import BaseCommand |
| 4 | +from django.db import transaction |
| 5 | +from django.db.models import Count |
| 6 | + |
| 7 | +from osf.models.files import BaseFileNode, BaseFileVersionsThrough |
| 8 | +from osf.utils.requests import check_select_for_update |
| 9 | + |
| 10 | +logger = logging.getLogger(__name__) |
| 11 | + |
| 12 | + |
| 13 | +def find_duplicate_groups(): |
| 14 | + """ |
| 15 | + Yields {'basefilenode_id', 'fileversion__identifier', 'count'} for every |
| 16 | + (file, identifier) pair that has more than one linked FileVersion. |
| 17 | + """ |
| 18 | + return ( |
| 19 | + BaseFileVersionsThrough.objects |
| 20 | + .values('basefilenode_id', 'fileversion__identifier') |
| 21 | + .annotate(count=Count('id')) |
| 22 | + .filter(count__gt=1) |
| 23 | + .order_by('basefilenode_id') |
| 24 | + ) |
| 25 | + |
| 26 | + |
| 27 | +def fetch_group_rows(basefilenode_id, identifier): |
| 28 | + # `fileversion__id` is a secondary sort key so the choice of keeper is fully |
| 29 | + # deterministic even if two duplicates share the exact same `created` timestamp. |
| 30 | + return list( |
| 31 | + BaseFileVersionsThrough.objects |
| 32 | + .filter(basefilenode_id=basefilenode_id, fileversion__identifier=identifier) |
| 33 | + .select_related('fileversion') |
| 34 | + .order_by('fileversion__created', 'fileversion__id') |
| 35 | + ) |
| 36 | + |
| 37 | + |
| 38 | +def log_group(file_label, identifier, row_to_keep, rows_to_delete, dry_run): |
| 39 | + logger.info( |
| 40 | + f'{"[DRY-RUN] " if dry_run else ""}file={file_label} identifier={identifier} ' |
| 41 | + f'keeping {row_to_keep.fileversion._id} (location={row_to_keep.fileversion.location}) ' |
| 42 | + f'discarding={[(row.fileversion._id, row.fileversion.location) for row in rows_to_delete]}' |
| 43 | + ) |
| 44 | + |
| 45 | + |
| 46 | +def resolve_group(basefilenode_id, identifier, file_label, dry_run): |
| 47 | + """ |
| 48 | + Fetches the current rows for one duplicate group, logs the keep/discard |
| 49 | + decision, unless dry_run - deletes the discarded duplicate(s. |
| 50 | + """ |
| 51 | + through_rows = fetch_group_rows(basefilenode_id, identifier) |
| 52 | + if len(through_rows) < 2: |
| 53 | + return False |
| 54 | + |
| 55 | + row_to_keep, rows_to_delete = through_rows[0], through_rows[1:] |
| 56 | + log_group(file_label, identifier, row_to_keep, rows_to_delete, dry_run=dry_run) |
| 57 | + |
| 58 | + if not dry_run: |
| 59 | + for row in rows_to_delete: |
| 60 | + row.delete() |
| 61 | + |
| 62 | + return True |
| 63 | + |
| 64 | + |
| 65 | +def dedupe_file_versions(dry_run=True): |
| 66 | + """ |
| 67 | + Finds FileVersions that share the same `identifier` for the same file because of |
| 68 | + race condition in OsfStorageFile.create_version() and delete duplicate |
| 69 | + """ |
| 70 | + if dry_run: |
| 71 | + logger.info('[DRY-RUN] Data will not be modified.') |
| 72 | + |
| 73 | + groups = list(find_duplicate_groups()) |
| 74 | + file_labels = dict( |
| 75 | + BaseFileNode.objects |
| 76 | + .filter(id__in={group['basefilenode_id'] for group in groups}) |
| 77 | + .values_list('id', '_id') |
| 78 | + ) |
| 79 | + |
| 80 | + fixed = 0 |
| 81 | + |
| 82 | + for group in groups: |
| 83 | + basefilenode_id = group['basefilenode_id'] |
| 84 | + identifier = group['fileversion__identifier'] |
| 85 | + file_label = file_labels.get(basefilenode_id, basefilenode_id) |
| 86 | + |
| 87 | + if dry_run: |
| 88 | + resolved = resolve_group(basefilenode_id, identifier, file_label, dry_run=True) |
| 89 | + else: |
| 90 | + with transaction.atomic(): |
| 91 | + if check_select_for_update(): |
| 92 | + # Lock the file row for the duration of this group's cleanup so |
| 93 | + # a concurrent create_version() call for the same file can't |
| 94 | + # interleave with the read-then-delete in resolve_group(). |
| 95 | + BaseFileNode.objects.select_for_update().get(pk=basefilenode_id) |
| 96 | + resolved = resolve_group(basefilenode_id, identifier, file_label, dry_run=False) |
| 97 | + |
| 98 | + if resolved: |
| 99 | + fixed += 1 |
| 100 | + |
| 101 | + logger.info(f'{fixed} duplicate group(s) resolved.') |
| 102 | + return fixed |
| 103 | + |
| 104 | + |
| 105 | +class Command(BaseCommand): |
| 106 | + help = """ |
| 107 | + Finds FileVersions that share the same `identifier` on the same file because of |
| 108 | + a race condition in OsfStorageFile.create_version() and delete duplicate |
| 109 | + """ |
| 110 | + |
| 111 | + def add_arguments(self, parser): |
| 112 | + parser.add_argument( |
| 113 | + '--apply', |
| 114 | + action='store_false', |
| 115 | + dest='dry_run', |
| 116 | + default=True, |
| 117 | + help='Actually unlink duplicate versions. Without this flag, only reports what would change.', |
| 118 | + ) |
| 119 | + |
| 120 | + # Management command handler |
| 121 | + def handle(self, *args, **options): |
| 122 | + dry_run = options.get('dry_run', True) |
| 123 | + dedupe_file_versions(dry_run=dry_run) |
0 commit comments