diff --git a/docs/source/guide/start.md b/docs/source/guide/start.md index 98a1c844bcbe..0088cd6f2c79 100644 --- a/docs/source/guide/start.md +++ b/docs/source/guide/start.md @@ -60,6 +60,7 @@ The following command line arguments are optional and must be specified with `la | `--enable-legacy-api-token` | `LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN` | `False` | Enable legacy API token authentication. Useful for running with a pre-existing token via `--user-token`. | | N/A | `LABEL_STUDIO_LOCAL_FILES_SERVING_ENABLED` | `False` | Allow Label Studio to access local file directories to import storage. See [Run Label Studio on Docker and use local storage](start.html#Run_Label_Studio_on_Docker_and_use_local_storage). | | N/A | `LABEL_STUDIO_LOCAL_FILES_DOCUMENT_ROOT` | `/` | Specify the root directory for Label Studio to use when accessing local file directories. See [Run Label Studio on Docker and use local storage](start.html#Run_Label_Studio_on_Docker_and_use_local_storage). | +| N/A | `LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS` | `False` | When `True`, preserve `created_at` and `updated_at` values supplied in the source annotation JSON when tasks are added by a storage sync (`ImportStorage.add_task`). When `False` (the default), those fields are set to the sync time via `auto_now_add` / `auto_now` on the `Annotation` model. Missing values in the source JSON fall back to the sync time regardless of this setting. | ### Set environment variables How you set environment variables depends on the operating system and the environment in which you deploy Label Studio. diff --git a/label_studio/core/old_ls_migration.py b/label_studio/core/old_ls_migration.py index 81b08ff03fa2..1f027d8b52e7 100644 --- a/label_studio/core/old_ls_migration.py +++ b/label_studio/core/old_ls_migration.py @@ -1,10 +1,10 @@ -import contextlib import datetime import io import json import os import pathlib +from core.utils.db import suppress_autotime from core.utils.io import get_data_dir from core.utils.params import get_env from data_import.models import FileUpload @@ -18,24 +18,6 @@ from tasks.models import Annotation, Prediction, Task -@contextlib.contextmanager -def suppress_autotime(model, fields): - """allow to keep original created_at value for auto_now_add=True field""" - _original_values = {} - for field in model._meta.local_fields: - if field.name in fields: - _original_values[field.name] = {'auto_now': field.auto_now, 'auto_now_add': field.auto_now_add} - field.auto_now = False - field.auto_now_add = False - try: - yield - finally: - for field in model._meta.local_fields: - if field.name in fields: - field.auto_now = _original_values[field.name]['auto_now'] - field.auto_now_add = _original_values[field.name]['auto_now_add'] - - def _migrate_tasks(project_path, project): """Migrate tasks from json file to database objects""" tasks_path = project_path / 'tasks.json' diff --git a/label_studio/core/settings/base.py b/label_studio/core/settings/base.py index 05416436a80c..13f79a58abbc 100644 --- a/label_studio/core/settings/base.py +++ b/label_studio/core/settings/base.py @@ -586,6 +586,10 @@ IMPORT_BATCH_SIZE = int(get_env('IMPORT_BATCH_SIZE', 500)) # Batch size for processing prediction imports to avoid memory issues with large datasets PREDICTION_IMPORT_BATCH_SIZE = int(get_env('PREDICTION_IMPORT_BATCH_SIZE', 500)) +# When True, ImportStorage.add_task preserves created_at / updated_at supplied by the +# source annotation JSON instead of overwriting them with now(). Missing timestamps +# still fall back to now(). Default False preserves current behavior. +PRESERVE_IMPORT_TIMESTAMPS = get_bool_env('LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS', False) PROJECT_TITLE_MIN_LEN = 3 PROJECT_TITLE_MAX_LEN = 50 LOGIN_REDIRECT_URL = '/' diff --git a/label_studio/core/utils/db.py b/label_studio/core/utils/db.py index 8944b36388fc..a4dbbe1c7637 100644 --- a/label_studio/core/utils/db.py +++ b/label_studio/core/utils/db.py @@ -1,3 +1,4 @@ +import contextlib import itertools import logging import time @@ -201,3 +202,27 @@ def signal_clear_column_presence_cache(**_kwargs): so that the next migration can introspect the new column using has_column_cached().""" logger.debug('Clearing column presence cache in post_migrate signal') _column_presence_cache.clear() + + +@contextlib.contextmanager +def suppress_autotime(model, fields): + """Temporarily disable auto_now / auto_now_add on the named model fields. + + Use as a context manager around a .save() call when you need to persist + caller-supplied timestamps on fields that are otherwise stamped + automatically by Django (e.g., importing rows with historical + created_at / updated_at values). + """ + _original_values = {} + for field in model._meta.local_fields: + if field.name in fields: + _original_values[field.name] = {'auto_now': field.auto_now, 'auto_now_add': field.auto_now_add} + field.auto_now = False + field.auto_now_add = False + try: + yield + finally: + for field in model._meta.local_fields: + if field.name in fields: + field.auto_now = _original_values[field.name]['auto_now'] + field.auto_now_add = _original_values[field.name]['auto_now_add'] diff --git a/label_studio/io_storages/base_models.py b/label_studio/io_storages/base_models.py index 0d4d1fda66d7..37620eee940e 100644 --- a/label_studio/io_storages/base_models.py +++ b/label_studio/io_storages/base_models.py @@ -20,6 +20,7 @@ from core.feature_flags import flag_set from core.redis import redis_connected, start_job_async_or_sync from core.utils.common import load_func +from core.utils.db import suppress_autotime from core.utils.iterators import iterate_queryset from data_export.serializers import ExportDataSerializer from django.conf import settings @@ -31,6 +32,7 @@ from django.utils.translation import gettext_lazy as _ from fsm.functions import backfill_fsm_states_for_tasks from io_storages.utils import StorageObject, get_all_uris_via_regex, get_uri_via_regex, parse_bucket_uri +from rest_framework import serializers from rest_framework.exceptions import ValidationError from rq.job import Job from tasks.models import Annotation, Task @@ -43,6 +45,20 @@ logger = logging.getLogger(__name__) +class _ImportAnnotationSerializer(AnnotationSerializer): + """Variant of AnnotationSerializer used by ImportStorage.add_task when + settings.PRESERVE_IMPORT_TIMESTAMPS is True. Overrides created_at and + updated_at so caller-supplied timestamps in the source JSON are not + silently dropped as read-only fields. The serializer.save() call must + still be wrapped in suppress_autotime(Annotation, ['created_at', + 'updated_at']) so that Django's auto_now / auto_now_add flags don't + overwrite the values at the model layer. + """ + + created_at = serializers.DateTimeField(required=False) + updated_at = serializers.DateTimeField(required=False) + + class StorageInfo(models.Model): """ StorageInfo helps to understand storage status and progress @@ -544,14 +560,30 @@ def add_task(cls, project, maximum_annotations, max_inner_id, storage, link_obje # add annotations logger.debug(f'Create {len(annotations)} annotations for task={task}') + preserve_timestamps: bool = settings.PRESERVE_IMPORT_TIMESTAMPS + now_ts: datetime | None = timezone.now() if preserve_timestamps else None for annotation in annotations: annotation['task'] = task.id annotation['project'] = project.id - annotation_ser = AnnotationSerializer(data=annotations, many=True) + if preserve_timestamps: + # Fill in 'now' for any missing timestamp so that + # suppress_autotime never writes a NULL value. + # Caller-supplied values (when present) win. + annotation.setdefault('created_at', now_ts) + annotation.setdefault('updated_at', now_ts) + + if preserve_timestamps: + annotation_ser = _ImportAnnotationSerializer(data=annotations, many=True) + else: + annotation_ser = AnnotationSerializer(data=annotations, many=True) # Always validate annotations, but control error handling based on FF if annotation_ser.is_valid(): - annotation_ser.save() + if preserve_timestamps: + with suppress_autotime(Annotation, ['created_at', 'updated_at']): + annotation_ser.save() + else: + annotation_ser.save() else: # Log validation errors but don't save invalid annotations logger.error(f'Invalid annotations for task {task.id}: {annotation_ser.errors}') diff --git a/label_studio/io_storages/tests/test_preserve_import_timestamps.py b/label_studio/io_storages/tests/test_preserve_import_timestamps.py new file mode 100644 index 000000000000..c589b2c58df2 --- /dev/null +++ b/label_studio/io_storages/tests/test_preserve_import_timestamps.py @@ -0,0 +1,116 @@ +"""Coverage for LABEL_STUDIO_PRESERVE_IMPORT_TIMESTAMPS opt-in behavior in +ImportStorage.add_task. See core/settings/base.py for the setting and +io_storages/base_models.py for the code path. +""" + +import json +from datetime import datetime +from datetime import timezone as dt_timezone + +import boto3 +import pytest +from django.test import override_settings +from django.utils import timezone +from io_storages.models import S3ImportStorage +from moto import mock_s3 +from projects.models import Project +from projects.tests.factories import ProjectFactory +from tasks.models import Annotation +from tests.conftest import set_feature_flag_envvar # noqa: F401 + +LABEL_CONFIG = """ + + + + + + + +""" + + +def _annotation_result() -> list[dict]: + return [ + { + 'from_name': 'sentiment', + 'to_name': 'text', + 'type': 'choices', + 'value': {'choices': ['positive']}, + } + ] + + +TASK_WITH_TIMESTAMPED_ANNOTATION: dict = { + 'data': {'text': 'hello'}, + 'annotations': [ + { + 'result': _annotation_result(), + 'created_at': '2024-06-01T12:00:00Z', + 'updated_at': '2024-06-01T12:05:00Z', + } + ], +} + +TASK_WITHOUT_TIMESTAMPS: dict = { + 'data': {'text': 'hello'}, + 'annotations': [{'result': _annotation_result()}], +} + + +def _sync_via_s3(project: Project, task_json: dict, bucket_name: str = 'pytest-preserve-timestamps') -> None: + """Push a task JSON to a mocked S3 bucket and sync it into the project.""" + with mock_s3(): + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket=bucket_name) + s3.put_object(Bucket=bucket_name, Key='task.json', Body=json.dumps([task_json])) + storage = S3ImportStorage( + project=project, + bucket=bucket_name, + aws_access_key_id='example', + aws_secret_access_key='example', + use_blob_urls=False, + ) + storage.save() + storage.sync() + + +@pytest.mark.django_db +class TestPreserveImportTimestamps: + @pytest.fixture + def project(self): + return ProjectFactory(label_config=LABEL_CONFIG) + + @override_settings(PRESERVE_IMPORT_TIMESTAMPS=True) + def test_source_timestamps_preserved_when_enabled(self, project, set_feature_flag_envvar): + """With the setting on, created_at/updated_at from the source JSON survive the save.""" + _sync_via_s3(project, TASK_WITH_TIMESTAMPED_ANNOTATION) + + ann = Annotation.objects.get(project=project) + assert ann.created_at == datetime(2024, 6, 1, 12, 0, tzinfo=dt_timezone.utc) + assert ann.updated_at == datetime(2024, 6, 1, 12, 5, tzinfo=dt_timezone.utc) + + def test_source_timestamps_ignored_by_default(self, project, set_feature_flag_envvar): + """Default (setting off): auto_now_add / auto_now overwrite the source values + with the sync time. Confirms we haven't accidentally changed default behavior. + """ + before = timezone.now() + _sync_via_s3(project, TASK_WITH_TIMESTAMPED_ANNOTATION) + + ann = Annotation.objects.get(project=project) + # Source JSON said 2024-06-01; expected now(), not that value. + assert ann.created_at >= before + assert ann.updated_at >= before + + @override_settings(PRESERVE_IMPORT_TIMESTAMPS=True) + def test_missing_source_timestamps_fall_back_to_now(self, project, set_feature_flag_envvar): + """With the setting on but no timestamps in the source JSON, both fields fall + back to now() rather than being left NULL by suppress_autotime. + """ + before = timezone.now() + _sync_via_s3(project, TASK_WITHOUT_TIMESTAMPS) + + ann = Annotation.objects.get(project=project) + assert ann.created_at is not None + assert ann.updated_at is not None + assert ann.created_at >= before + assert ann.updated_at >= before