From 981a408529330ce747abf2c4019a6032333cd89a Mon Sep 17 00:00:00 2001 From: Mike Pesavento Date: Thu, 2 Jul 2026 12:55:00 -1000 Subject: [PATCH] fix: preserve created_at/updated_at on annotations imported via storage sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ImportStorage.add_task passes each annotation dict through AnnotationSerializer(...).save(). The serializer treats created_at and updated_at as read-only because of auto_now_add / auto_now on the Annotation model, so any timestamps supplied by the source JSON are silently dropped and Django overwrites both fields with now() on save. For users importing historical or migrated annotation data through a storage sync (S3, GCS, Azure, local files), this silently destroys the real annotation timestamps — the audit trail, time-based analytics, and any downstream reporting keyed on annotation time. Add an opt-in Django setting sourced from the environment variable LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS (default false, current behavior unchanged). When true: - ImportStorage.add_task uses a private _ImportAnnotationSerializer subclass that makes created_at / updated_at writable so the source values are not dropped. - Missing timestamps in the source JSON fall back to now() so suppress_autotime never writes a NULL value. - serializer.save() runs inside a suppress_autotime(Annotation, ...) context manager so auto_now_add / auto_now cannot overwrite the supplied values at the model layer. - The serializer flow is preserved end to end — AnnotationDuplicateError handling, FSM state initialisation, and feature-flag validation gates still fire. Also extract the pre-existing suppress_autotime helper out of core/old_ls_migration.py (a legacy module new code should not couple to) and into core/utils/db.py alongside the other database utilities. No behaviour change to the helper itself. Acceptance criteria: - When a user sets LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS=true and triggers a storage sync on a task JSON containing an annotation with an explicit created_at / updated_at, then the resulting Annotation row's created_at / updated_at equal the source values. - When a user sets LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS=true and the source JSON omits created_at / updated_at, then both fields on the resulting Annotation row are populated with the sync time (not NULL). - When LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS is unset or false and the source JSON contains created_at / updated_at, then both fields on the resulting Annotation row are populated with the sync time (existing behaviour, unchanged). Tests in label_studio/io_storages/tests/test_preserve_import_timestamps.py cover all three paths. Closes #9797 --- docs/source/guide/start.md | 1 + label_studio/core/old_ls_migration.py | 20 +-- label_studio/core/settings/base.py | 4 + label_studio/core/utils/db.py | 25 ++++ label_studio/io_storages/base_models.py | 36 +++++- .../tests/test_preserve_import_timestamps.py | 116 ++++++++++++++++++ 6 files changed, 181 insertions(+), 21 deletions(-) create mode 100644 label_studio/io_storages/tests/test_preserve_import_timestamps.py diff --git a/docs/source/guide/start.md b/docs/source/guide/start.md index 98a1c844bcbe..0088cd6f2c79 100644 --- a/docs/source/guide/start.md +++ b/docs/source/guide/start.md @@ -60,6 +60,7 @@ The following command line arguments are optional and must be specified with `la | `--enable-legacy-api-token` | `LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN` | `False` | Enable legacy API token authentication. Useful for running with a pre-existing token via `--user-token`. | | N/A | `LABEL_STUDIO_LOCAL_FILES_SERVING_ENABLED` | `False` | Allow Label Studio to access local file directories to import storage. See [Run Label Studio on Docker and use local storage](start.html#Run_Label_Studio_on_Docker_and_use_local_storage). | | N/A | `LABEL_STUDIO_LOCAL_FILES_DOCUMENT_ROOT` | `/` | Specify the root directory for Label Studio to use when accessing local file directories. See [Run Label Studio on Docker and use local storage](start.html#Run_Label_Studio_on_Docker_and_use_local_storage). | +| N/A | `LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS` | `False` | When `True`, preserve `created_at` and `updated_at` values supplied in the source annotation JSON when tasks are added by a storage sync (`ImportStorage.add_task`). When `False` (the default), those fields are set to the sync time via `auto_now_add` / `auto_now` on the `Annotation` model. Missing values in the source JSON fall back to the sync time regardless of this setting. | ### Set environment variables How you set environment variables depends on the operating system and the environment in which you deploy Label Studio. diff --git a/label_studio/core/old_ls_migration.py b/label_studio/core/old_ls_migration.py index 81b08ff03fa2..1f027d8b52e7 100644 --- a/label_studio/core/old_ls_migration.py +++ b/label_studio/core/old_ls_migration.py @@ -1,10 +1,10 @@ -import contextlib import datetime import io import json import os import pathlib +from core.utils.db import suppress_autotime from core.utils.io import get_data_dir from core.utils.params import get_env from data_import.models import FileUpload @@ -18,24 +18,6 @@ from tasks.models import Annotation, Prediction, Task -@contextlib.contextmanager -def suppress_autotime(model, fields): - """allow to keep original created_at value for auto_now_add=True field""" - _original_values = {} - for field in model._meta.local_fields: - if field.name in fields: - _original_values[field.name] = {'auto_now': field.auto_now, 'auto_now_add': field.auto_now_add} - field.auto_now = False - field.auto_now_add = False - try: - yield - finally: - for field in model._meta.local_fields: - if field.name in fields: - field.auto_now = _original_values[field.name]['auto_now'] - field.auto_now_add = _original_values[field.name]['auto_now_add'] - - def _migrate_tasks(project_path, project): """Migrate tasks from json file to database objects""" tasks_path = project_path / 'tasks.json' diff --git a/label_studio/core/settings/base.py b/label_studio/core/settings/base.py index 05416436a80c..13f79a58abbc 100644 --- a/label_studio/core/settings/base.py +++ b/label_studio/core/settings/base.py @@ -586,6 +586,10 @@ IMPORT_BATCH_SIZE = int(get_env('IMPORT_BATCH_SIZE', 500)) # Batch size for processing prediction imports to avoid memory issues with large datasets PREDICTION_IMPORT_BATCH_SIZE = int(get_env('PREDICTION_IMPORT_BATCH_SIZE', 500)) +# When True, ImportStorage.add_task preserves created_at / updated_at supplied by the +# source annotation JSON instead of overwriting them with now(). Missing timestamps +# still fall back to now(). Default False preserves current behavior. +PRESERVE_IMPORT_TIMESTAMPS = get_bool_env('LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS', False) PROJECT_TITLE_MIN_LEN = 3 PROJECT_TITLE_MAX_LEN = 50 LOGIN_REDIRECT_URL = '/' diff --git a/label_studio/core/utils/db.py b/label_studio/core/utils/db.py index 8944b36388fc..a4dbbe1c7637 100644 --- a/label_studio/core/utils/db.py +++ b/label_studio/core/utils/db.py @@ -1,3 +1,4 @@ +import contextlib import itertools import logging import time @@ -201,3 +202,27 @@ def signal_clear_column_presence_cache(**_kwargs): so that the next migration can introspect the new column using has_column_cached().""" logger.debug('Clearing column presence cache in post_migrate signal') _column_presence_cache.clear() + + +@contextlib.contextmanager +def suppress_autotime(model, fields): + """Temporarily disable auto_now / auto_now_add on the named model fields. + + Use as a context manager around a .save() call when you need to persist + caller-supplied timestamps on fields that are otherwise stamped + automatically by Django (e.g., importing rows with historical + created_at / updated_at values). + """ + _original_values = {} + for field in model._meta.local_fields: + if field.name in fields: + _original_values[field.name] = {'auto_now': field.auto_now, 'auto_now_add': field.auto_now_add} + field.auto_now = False + field.auto_now_add = False + try: + yield + finally: + for field in model._meta.local_fields: + if field.name in fields: + field.auto_now = _original_values[field.name]['auto_now'] + field.auto_now_add = _original_values[field.name]['auto_now_add'] diff --git a/label_studio/io_storages/base_models.py b/label_studio/io_storages/base_models.py index 0d4d1fda66d7..37620eee940e 100644 --- a/label_studio/io_storages/base_models.py +++ b/label_studio/io_storages/base_models.py @@ -20,6 +20,7 @@ from core.feature_flags import flag_set from core.redis import redis_connected, start_job_async_or_sync from core.utils.common import load_func +from core.utils.db import suppress_autotime from core.utils.iterators import iterate_queryset from data_export.serializers import ExportDataSerializer from django.conf import settings @@ -31,6 +32,7 @@ from django.utils.translation import gettext_lazy as _ from fsm.functions import backfill_fsm_states_for_tasks from io_storages.utils import StorageObject, get_all_uris_via_regex, get_uri_via_regex, parse_bucket_uri +from rest_framework import serializers from rest_framework.exceptions import ValidationError from rq.job import Job from tasks.models import Annotation, Task @@ -43,6 +45,20 @@ logger = logging.getLogger(__name__) +class _ImportAnnotationSerializer(AnnotationSerializer): + """Variant of AnnotationSerializer used by ImportStorage.add_task when + settings.PRESERVE_IMPORT_TIMESTAMPS is True. Overrides created_at and + updated_at so caller-supplied timestamps in the source JSON are not + silently dropped as read-only fields. The serializer.save() call must + still be wrapped in suppress_autotime(Annotation, ['created_at', + 'updated_at']) so that Django's auto_now / auto_now_add flags don't + overwrite the values at the model layer. + """ + + created_at = serializers.DateTimeField(required=False) + updated_at = serializers.DateTimeField(required=False) + + class StorageInfo(models.Model): """ StorageInfo helps to understand storage status and progress @@ -544,14 +560,30 @@ def add_task(cls, project, maximum_annotations, max_inner_id, storage, link_obje # add annotations logger.debug(f'Create {len(annotations)} annotations for task={task}') + preserve_timestamps: bool = settings.PRESERVE_IMPORT_TIMESTAMPS + now_ts: datetime | None = timezone.now() if preserve_timestamps else None for annotation in annotations: annotation['task'] = task.id annotation['project'] = project.id - annotation_ser = AnnotationSerializer(data=annotations, many=True) + if preserve_timestamps: + # Fill in 'now' for any missing timestamp so that + # suppress_autotime never writes a NULL value. + # Caller-supplied values (when present) win. + annotation.setdefault('created_at', now_ts) + annotation.setdefault('updated_at', now_ts) + + if preserve_timestamps: + annotation_ser = _ImportAnnotationSerializer(data=annotations, many=True) + else: + annotation_ser = AnnotationSerializer(data=annotations, many=True) # Always validate annotations, but control error handling based on FF if annotation_ser.is_valid(): - annotation_ser.save() + if preserve_timestamps: + with suppress_autotime(Annotation, ['created_at', 'updated_at']): + annotation_ser.save() + else: + annotation_ser.save() else: # Log validation errors but don't save invalid annotations logger.error(f'Invalid annotations for task {task.id}: {annotation_ser.errors}') diff --git a/label_studio/io_storages/tests/test_preserve_import_timestamps.py b/label_studio/io_storages/tests/test_preserve_import_timestamps.py new file mode 100644 index 000000000000..c589b2c58df2 --- /dev/null +++ b/label_studio/io_storages/tests/test_preserve_import_timestamps.py @@ -0,0 +1,116 @@ +"""Coverage for LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS opt-in behavior in +ImportStorage.add_task. See core/settings/base.py for the setting and +io_storages/base_models.py for the code path. +""" + +import json +from datetime import datetime +from datetime import timezone as dt_timezone + +import boto3 +import pytest +from django.test import override_settings +from django.utils import timezone +from io_storages.models import S3ImportStorage +from moto import mock_s3 +from projects.models import Project +from projects.tests.factories import ProjectFactory +from tasks.models import Annotation +from tests.conftest import set_feature_flag_envvar # noqa: F401 + +LABEL_CONFIG = """ + + + + + + + +""" + + +def _annotation_result() -> list[dict]: + return [ + { + 'from_name': 'sentiment', + 'to_name': 'text', + 'type': 'choices', + 'value': {'choices': ['positive']}, + } + ] + + +TASK_WITH_TIMESTAMPED_ANNOTATION: dict = { + 'data': {'text': 'hello'}, + 'annotations': [ + { + 'result': _annotation_result(), + 'created_at': '2024-06-01T12:00:00Z', + 'updated_at': '2024-06-01T12:05:00Z', + } + ], +} + +TASK_WITHOUT_TIMESTAMPS: dict = { + 'data': {'text': 'hello'}, + 'annotations': [{'result': _annotation_result()}], +} + + +def _sync_via_s3(project: Project, task_json: dict, bucket_name: str = 'pytest-preserve-timestamps') -> None: + """Push a task JSON to a mocked S3 bucket and sync it into the project.""" + with mock_s3(): + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket=bucket_name) + s3.put_object(Bucket=bucket_name, Key='task.json', Body=json.dumps([task_json])) + storage = S3ImportStorage( + project=project, + bucket=bucket_name, + aws_access_key_id='example', + aws_secret_access_key='example', + use_blob_urls=False, + ) + storage.save() + storage.sync() + + +@pytest.mark.django_db +class TestPreserveImportTimestamps: + @pytest.fixture + def project(self): + return ProjectFactory(label_config=LABEL_CONFIG) + + @override_settings(PRESERVE_IMPORT_TIMESTAMPS=True) + def test_source_timestamps_preserved_when_enabled(self, project, set_feature_flag_envvar): + """With the setting on, created_at/updated_at from the source JSON survive the save.""" + _sync_via_s3(project, TASK_WITH_TIMESTAMPED_ANNOTATION) + + ann = Annotation.objects.get(project=project) + assert ann.created_at == datetime(2024, 6, 1, 12, 0, tzinfo=dt_timezone.utc) + assert ann.updated_at == datetime(2024, 6, 1, 12, 5, tzinfo=dt_timezone.utc) + + def test_source_timestamps_ignored_by_default(self, project, set_feature_flag_envvar): + """Default (setting off): auto_now_add / auto_now overwrite the source values + with the sync time. Confirms we haven't accidentally changed default behavior. + """ + before = timezone.now() + _sync_via_s3(project, TASK_WITH_TIMESTAMPED_ANNOTATION) + + ann = Annotation.objects.get(project=project) + # Source JSON said 2024-06-01; expected now(), not that value. + assert ann.created_at >= before + assert ann.updated_at >= before + + @override_settings(PRESERVE_IMPORT_TIMESTAMPS=True) + def test_missing_source_timestamps_fall_back_to_now(self, project, set_feature_flag_envvar): + """With the setting on but no timestamps in the source JSON, both fields fall + back to now() rather than being left NULL by suppress_autotime. + """ + before = timezone.now() + _sync_via_s3(project, TASK_WITHOUT_TIMESTAMPS) + + ann = Annotation.objects.get(project=project) + assert ann.created_at is not None + assert ann.updated_at is not None + assert ann.created_at >= before + assert ann.updated_at >= before