fix: preserve created_at/updated_at on annotations imported via storage sync

docs/source/guide/start.md

@@ -60,6 +60,7 @@ The following command line arguments are optional and must be specified with `la
 | `--enable-legacy-api-token` | `LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN` | `False` | Enable legacy API token authentication. Useful for running with a pre-existing token via `--user-token`. |
 | N/A | `LABEL_STUDIO_LOCAL_FILES_SERVING_ENABLED` | `False` | Allow Label Studio to access local file directories to import storage. See [Run Label Studio on Docker and use local storage](start.html#Run_Label_Studio_on_Docker_and_use_local_storage). |
 | N/A | `LABEL_STUDIO_LOCAL_FILES_DOCUMENT_ROOT` | `/` | Specify the root directory for Label Studio to use when accessing local file directories. See [Run Label Studio on Docker and use local storage](start.html#Run_Label_Studio_on_Docker_and_use_local_storage). |
+| N/A | `LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS` | `False` | When `True`, preserve `created_at` and `updated_at` values supplied in the source annotation JSON when tasks are added by a storage sync (`ImportStorage.add_task`). When `False` (the default), those fields are set to the sync time via `auto_now_add` / `auto_now` on the `Annotation` model. Missing values in the source JSON fall back to the sync time regardless of this setting. |
 
 ### Set environment variables
 How you set environment variables depends on the operating system and the environment in which you deploy Label Studio. 

label_studio/core/old_ls_migration.py

@@ -1,10 +1,10 @@
-import contextlib
 import datetime
 import io
 import json
 import os
 import pathlib
 
+from core.utils.db import suppress_autotime
 from core.utils.io import get_data_dir
 from core.utils.params import get_env
 from data_import.models import FileUpload
@@ -18,24 +18,6 @@
 from tasks.models import Annotation, Prediction, Task
 
 
-@contextlib.contextmanager
-def suppress_autotime(model, fields):
-    """allow to keep original created_at value for auto_now_add=True field"""
-    _original_values = {}
-    for field in model._meta.local_fields:
-        if field.name in fields:
-            _original_values[field.name] = {'auto_now': field.auto_now, 'auto_now_add': field.auto_now_add}
-            field.auto_now = False
-            field.auto_now_add = False
-    try:
-        yield
-    finally:
-        for field in model._meta.local_fields:
-            if field.name in fields:
-                field.auto_now = _original_values[field.name]['auto_now']
-                field.auto_now_add = _original_values[field.name]['auto_now_add']
-
-
 def _migrate_tasks(project_path, project):
     """Migrate tasks from json file to database objects"""
     tasks_path = project_path / 'tasks.json'

label_studio/core/settings/base.py

@@ -586,6 +586,10 @@
 IMPORT_BATCH_SIZE = int(get_env('IMPORT_BATCH_SIZE', 500))
 # Batch size for processing prediction imports to avoid memory issues with large datasets
 PREDICTION_IMPORT_BATCH_SIZE = int(get_env('PREDICTION_IMPORT_BATCH_SIZE', 500))
+# When True, ImportStorage.add_task preserves created_at / updated_at supplied by the
+# source annotation JSON instead of overwriting them with now(). Missing timestamps
+# still fall back to now(). Default False preserves current behavior.
+PRESERVE_IMPORT_TIMESTAMPS = get_bool_env('LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS', False)
 PROJECT_TITLE_MIN_LEN = 3
 PROJECT_TITLE_MAX_LEN = 50
 LOGIN_REDIRECT_URL = '/'

label_studio/core/utils/db.py

@@ -1,3 +1,4 @@
+import contextlib
 import itertools
 import logging
 import time
@@ -201,3 +202,27 @@ def signal_clear_column_presence_cache(**_kwargs):
     so that the next migration can introspect the new column using has_column_cached()."""
     logger.debug('Clearing column presence cache in post_migrate signal')
     _column_presence_cache.clear()
+
+
+@contextlib.contextmanager
+def suppress_autotime(model, fields):
+    """Temporarily disable auto_now / auto_now_add on the named model fields.
+
+    Use as a context manager around a .save() call when you need to persist
+    caller-supplied timestamps on fields that are otherwise stamped
+    automatically by Django (e.g., importing rows with historical
+    created_at / updated_at values).
+    """
+    _original_values = {}
+    for field in model._meta.local_fields:
+        if field.name in fields:
+            _original_values[field.name] = {'auto_now': field.auto_now, 'auto_now_add': field.auto_now_add}
+            field.auto_now = False
+            field.auto_now_add = False
+    try:
+        yield
+    finally:
+        for field in model._meta.local_fields:
+            if field.name in fields:
+                field.auto_now = _original_values[field.name]['auto_now']
+                field.auto_now_add = _original_values[field.name]['auto_now_add']

label_studio/io_storages/base_models.py

@@ -20,6 +20,7 @@
 from core.feature_flags import flag_set
 from core.redis import redis_connected, start_job_async_or_sync
 from core.utils.common import load_func
+from core.utils.db import suppress_autotime
 from core.utils.iterators import iterate_queryset
 from data_export.serializers import ExportDataSerializer
 from django.conf import settings
@@ -31,6 +32,7 @@
 from django.utils.translation import gettext_lazy as _
 from fsm.functions import backfill_fsm_states_for_tasks
 from io_storages.utils import StorageObject, get_all_uris_via_regex, get_uri_via_regex, parse_bucket_uri
+from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
 from rq.job import Job
 from tasks.models import Annotation, Task
@@ -43,6 +45,20 @@
 logger = logging.getLogger(__name__)
 
 
+class _ImportAnnotationSerializer(AnnotationSerializer):
+    """Variant of AnnotationSerializer used by ImportStorage.add_task when
+    settings.PRESERVE_IMPORT_TIMESTAMPS is True. Overrides created_at and
+    updated_at so caller-supplied timestamps in the source JSON are not
+    silently dropped as read-only fields. The serializer.save() call must
+    still be wrapped in suppress_autotime(Annotation, ['created_at',
+    'updated_at']) so that Django's auto_now / auto_now_add flags don't
+    overwrite the values at the model layer.
+    """
+
+    created_at = serializers.DateTimeField(required=False)
+    updated_at = serializers.DateTimeField(required=False)
+
+
 class StorageInfo(models.Model):
     """
     StorageInfo helps to understand storage status and progress
@@ -544,14 +560,30 @@ def add_task(cls, project, maximum_annotations, max_inner_id, storage, link_obje
 
             # add annotations
             logger.debug(f'Create {len(annotations)} annotations for task={task}')
+            preserve_timestamps: bool = settings.PRESERVE_IMPORT_TIMESTAMPS
+            now_ts: datetime | None = timezone.now() if preserve_timestamps else None
             for annotation in annotations:
                 annotation['task'] = task.id
                 annotation['project'] = project.id
-            annotation_ser = AnnotationSerializer(data=annotations, many=True)
+                if preserve_timestamps:
+                    # Fill in 'now' for any missing timestamp so that
+                    # suppress_autotime never writes a NULL value.
+                    # Caller-supplied values (when present) win.
+                    annotation.setdefault('created_at', now_ts)
+                    annotation.setdefault('updated_at', now_ts)
+
+            if preserve_timestamps:
+                annotation_ser = _ImportAnnotationSerializer(data=annotations, many=True)
+            else:
+                annotation_ser = AnnotationSerializer(data=annotations, many=True)
 
             # Always validate annotations, but control error handling based on FF
             if annotation_ser.is_valid():
-                annotation_ser.save()
+                if preserve_timestamps:
+                    with suppress_autotime(Annotation, ['created_at', 'updated_at']):
+                        annotation_ser.save()
+                else:
+                    annotation_ser.save()
             else:
                 # Log validation errors but don't save invalid annotations
                 logger.error(f'Invalid annotations for task {task.id}: {annotation_ser.errors}')

label_studio/io_storages/tests/test_preserve_import_timestamps.py

@@ -0,0 +1,116 @@
+"""Coverage for LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS opt-in behavior in
+ImportStorage.add_task. See core/settings/base.py for the setting and
+io_storages/base_models.py for the code path.
+"""
+
+import json
+from datetime import datetime
+from datetime import timezone as dt_timezone
+
+import boto3
+import pytest
+from django.test import override_settings
+from django.utils import timezone
+from io_storages.models import S3ImportStorage
+from moto import mock_s3
+from projects.models import Project
+from projects.tests.factories import ProjectFactory
+from tasks.models import Annotation
+from tests.conftest import set_feature_flag_envvar  # noqa: F401
+
+LABEL_CONFIG = """
+<View>
+  <Text name="text" value="$text"/>
+  <Choices name="sentiment" toName="text">
+    <Choice value="positive"/>
+    <Choice value="negative"/>
+  </Choices>
+</View>
+"""
+
+
+def _annotation_result() -> list[dict]:
+    return [
+        {
+            'from_name': 'sentiment',
+            'to_name': 'text',
+            'type': 'choices',
+            'value': {'choices': ['positive']},
+        }
+    ]
+
+
+TASK_WITH_TIMESTAMPED_ANNOTATION: dict = {
+    'data': {'text': 'hello'},
+    'annotations': [
+        {
+            'result': _annotation_result(),
+            'created_at': '2024-06-01T12:00:00Z',
+            'updated_at': '2024-06-01T12:05:00Z',
+        }
+    ],
+}
+
+TASK_WITHOUT_TIMESTAMPS: dict = {
+    'data': {'text': 'hello'},
+    'annotations': [{'result': _annotation_result()}],
+}
+
+
+def _sync_via_s3(project: Project, task_json: dict, bucket_name: str = 'pytest-preserve-timestamps') -> None:
+    """Push a task JSON to a mocked S3 bucket and sync it into the project."""
+    with mock_s3():
+        s3 = boto3.client('s3', region_name='us-east-1')
+        s3.create_bucket(Bucket=bucket_name)
+        s3.put_object(Bucket=bucket_name, Key='task.json', Body=json.dumps([task_json]))
+        storage = S3ImportStorage(
+            project=project,
+            bucket=bucket_name,
+            aws_access_key_id='example',
+            aws_secret_access_key='example',
+            use_blob_urls=False,
+        )
+        storage.save()
+        storage.sync()
+
+
+@pytest.mark.django_db
+class TestPreserveImportTimestamps:
+    @pytest.fixture
+    def project(self):
+        return ProjectFactory(label_config=LABEL_CONFIG)
+
+    @override_settings(PRESERVE_IMPORT_TIMESTAMPS=True)
+    def test_source_timestamps_preserved_when_enabled(self, project, set_feature_flag_envvar):
+        """With the setting on, created_at/updated_at from the source JSON survive the save."""
+        _sync_via_s3(project, TASK_WITH_TIMESTAMPED_ANNOTATION)
+
+        ann = Annotation.objects.get(project=project)
+        assert ann.created_at == datetime(2024, 6, 1, 12, 0, tzinfo=dt_timezone.utc)
+        assert ann.updated_at == datetime(2024, 6, 1, 12, 5, tzinfo=dt_timezone.utc)
+
+    def test_source_timestamps_ignored_by_default(self, project, set_feature_flag_envvar):
+        """Default (setting off): auto_now_add / auto_now overwrite the source values
+        with the sync time. Confirms we haven't accidentally changed default behavior.
+        """
+        before = timezone.now()
+        _sync_via_s3(project, TASK_WITH_TIMESTAMPED_ANNOTATION)
+
+        ann = Annotation.objects.get(project=project)
+        # Source JSON said 2024-06-01; expected now(), not that value.
+        assert ann.created_at >= before
+        assert ann.updated_at >= before
+
+    @override_settings(PRESERVE_IMPORT_TIMESTAMPS=True)
+    def test_missing_source_timestamps_fall_back_to_now(self, project, set_feature_flag_envvar):
+        """With the setting on but no timestamps in the source JSON, both fields fall
+        back to now() rather than being left NULL by suppress_autotime.
+        """
+        before = timezone.now()
+        _sync_via_s3(project, TASK_WITHOUT_TIMESTAMPS)
+
+        ann = Annotation.objects.get(project=project)
+        assert ann.created_at is not None
+        assert ann.updated_at is not None
+        assert ann.created_at >= before
+        assert ann.updated_at >= before