Add Translation.value and .properties, with data model contents (mozilla#3802)

Author: eemeli

pontoon/base/migrations/0110_add_translation_value_and_properties_schema.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.22 on 2025-10-06 20:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [("base", "0109_fix_xcode_translations")]
+
+    operations = [
+        migrations.AddField(
+            model_name="translation",
+            name="value",
+            field=models.JSONField(default=list),
+        ),
+        migrations.AddField(
+            model_name="translation",
+            name="properties",
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]

pontoon/base/migrations/0111_add_translation_value_and_properties_data.py

@@ -0,0 +1,102 @@
+from math import ceil
+
+from moz.l10n.formats import Format
+from moz.l10n.formats.fluent import fluent_parse_entry
+from moz.l10n.message import message_to_json, parse_message
+from moz.l10n.model import CatchallKey, PatternMessage, SelectMessage
+
+from django.db import migrations, models
+
+
+batch_size = 10000
+
+
+def set_value_and_properties(apps, schema_editor):
+    Resource = apps.get_model("base", "Resource")
+    Translation = apps.get_model("base", "Translation")
+
+    batch_total = ceil(Translation.objects.count() / batch_size)
+    batch_count = 0
+
+    def print_progress():
+        nonlocal batch_count
+        if batch_count % 10 == 0:
+            print(f".({(batch_count / batch_total):.1%})", end="", flush=True)
+        else:
+            print(".", end="", flush=True)
+        batch_count += 1
+
+    pv_trans = []
+    v_trans = []
+    format_q = models.Subquery(
+        Resource.objects.filter(id=models.OuterRef("entity__resource_id")).values(
+            "format"
+        )
+    )
+    for trans in Translation.objects.annotate(format=format_q).iterator():
+        string = trans.string
+        try:
+            match trans.format:
+                case "fluent":
+                    fe = fluent_parse_entry(string, with_linepos=False)
+                    msg = fe.value
+                    trans.properties = {
+                        name: message_to_json(msg)
+                        for name, msg in fe.properties.items()
+                    } or None
+                case "lang" | "properties" | "":
+                    msg = PatternMessage([string])
+                case "android" | "gettext" | "webext" | "xcode" | "xliff":
+                    msg = parse_message(Format.mf2, string)
+                case _:
+                    msg = parse_message(Format[trans.format], string)
+
+            # MF2 syntax does not retain the catchall name/label
+            if isinstance(msg, SelectMessage) and trans.format != "fluent":
+                for keys in msg.variants:
+                    for key in keys:
+                        if isinstance(key, CatchallKey):
+                            key.value = "other"
+
+            trans.value = message_to_json(msg)
+            if trans.properties:
+                pv_trans.append(trans)
+            else:
+                v_trans.append(trans)
+        except Exception:
+            if (
+                trans.approved
+                and not trans.entity.obsolete
+                and not trans.entity.resource.project.disabled
+            ):
+                print(
+                    f"\nUsing fallback value for approved and active {trans.format} translation {trans.pk} "
+                    f"for entity {trans.entity.pk}, locale {trans.locale.code}:\n{trans.string}",
+                    flush=True,
+                )
+            trans.value = [trans.string]
+            v_trans.append(trans)
+        if len(pv_trans) == batch_size:
+            Translation.objects.bulk_update(pv_trans, ["value", "properties"])
+            pv_trans.clear()
+            print_progress()
+        if len(v_trans) == batch_size:
+            Translation.objects.bulk_update(v_trans, ["value"])
+            v_trans.clear()
+            print_progress()
+    if pv_trans:
+        Translation.objects.bulk_update(pv_trans, ["value", "properties"])
+        print_progress()
+    if v_trans:
+        Translation.objects.bulk_update(v_trans, ["value"])
+        print_progress()
+
+
+class Migration(migrations.Migration):
+    dependencies = [("base", "0110_add_translation_value_and_properties_schema")]
+
+    operations = [
+        migrations.RunPython(
+            set_value_and_properties, reverse_code=migrations.RunPython.noop
+        ),
+    ]

pontoon/base/migrations/0112_require_translation_value.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.29 on 2026-03-31 10:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("base", "0111_add_translation_value_and_properties_data"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="translation",
+            name="value",
+            field=models.JSONField(),
+        ),
+    ]

pontoon/base/models/entity.py

@@ -46,7 +46,7 @@ def combine_entity_filters(entities, filter_choices, filters, *args):
     return reduce(ior, filters)
 
 
-class EntityQuerySet(models.QuerySet):
+class EntityQuerySet(models.QuerySet["Entity"]):
     def _get_query(self, locale: Locale, project: Project | None, query: Q) -> Q:
         from pontoon.base.models.translation import Translation
 
@@ -253,7 +253,9 @@ def prefetch_entities_data(self, locale: Locale, preferred_source_locale: str):
 
 
 class Entity(DirtyFieldsMixin, models.Model):
-    resource = models.ForeignKey(Resource, models.CASCADE, related_name="entities")
+    resource: models.ForeignKey["Resource"] = models.ForeignKey(
+        Resource, models.CASCADE, related_name="entities"
+    )
     section = models.ForeignKey(
         Section, models.SET_NULL, related_name="entities", null=True, blank=True
     )

pontoon/base/models/resource.py

@@ -2,9 +2,13 @@
 from django.db import models
 from django.utils import timezone
 
+from pontoon.base.models.project import Project
+
 
 class Resource(models.Model):
-    project = models.ForeignKey("Project", models.CASCADE, related_name="resources")
+    project: models.ForeignKey["Project"] = models.ForeignKey(
+        "Project", models.CASCADE, related_name="resources"
+    )
     path = models.TextField()  # Path to localization file
     meta = ArrayField(ArrayField(models.TextField(), size=2), default=list)
     comment = models.TextField(blank=True)

pontoon/base/models/translation.py

@@ -19,7 +19,7 @@
 from pontoon.checks.utils import save_failed_checks
 
 
-class TranslationQuerySet(models.QuerySet):
+class TranslationQuerySet(models.QuerySet["Translation"]):
     def aggregate_stats(self) -> dict[str, int]:
         """
         Aggregate translation stats for this queryset.
@@ -154,6 +154,8 @@ class Translation(DirtyFieldsMixin, models.Model):
     locale = models.ForeignKey(Locale, models.CASCADE)
     user = models.ForeignKey(User, models.SET_NULL, null=True, blank=True)
     string = models.TextField()
+    value = models.JSONField()
+    properties = models.JSONField(null=True, blank=True)
     date = models.DateTimeField(default=timezone.now)
 
     # Active translations are displayed in the string list and as the first

pontoon/base/tests/__init__.py

@@ -135,6 +135,7 @@ class TranslationFactory(DjangoModelFactory):
     entity = SubFactory(EntityFactory)
     locale = SubFactory(LocaleFactory)
     string = Sequence(lambda n: f"translation {n}")
+    value = Sequence(lambda n: [f"translation {n}"])
     user = SubFactory(UserFactory)
 
     class Meta:

pontoon/batch/utils.py

@@ -8,9 +8,11 @@
 
 from django.utils import timezone
 
-from pontoon.base.models import Entity, Resource
+from pontoon.base.models import Entity, Resource, User
+from pontoon.base.models.translation import TranslationQuerySet
 from pontoon.checks import DB_FORMATS
 from pontoon.checks.libraries import run_checks
+from pontoon.translations.utils import parse_db_string_to_json
 
 
 parser = FluentParser()
@@ -64,7 +66,9 @@ def visit_TextElement(self, node):
     return serializer.serialize_entry(new_ast)
 
 
-def find_and_replace(translations, find, replace, user):
+def find_and_replace(
+    translations: TranslationQuerySet, find: str, replace: str, user: User
+):
     """Replace text in a set of translation.
 
     :arg QuerySet translations: a list of Translation objects in which to search
@@ -97,7 +101,8 @@ def find_and_replace(translations, find, replace, user):
         # Cache the old value to identify changed translations
         new_translation = deepcopy(translation)
 
-        if translation.entity.resource.format == Resource.Format.FLUENT:
+        res_format = translation.entity.resource.format
+        if res_format == Resource.Format.FLUENT:
             new_translation.string = ftl_find_and_replace(
                 translation.string, find, replace
             )
@@ -119,15 +124,21 @@ def find_and_replace(translations, find, replace, user):
         new_translation.pretranslated = False
         new_translation.fuzzy = False
 
-        if new_translation.entity.resource.format in DB_FORMATS:
+        errors = False
+        try:
+            new_translation.value, new_translation.properties = parse_db_string_to_json(
+                res_format, new_translation.string
+            )
+        except ValueError:
+            errors = True
+
+        if not errors and res_format in DB_FORMATS:
             errors = run_checks(
                 new_translation.entity,
                 new_translation.locale.code,
                 new_translation.string,
                 use_tt_checks=False,
             )
-        else:
-            errors = {}
 
         if errors:
             translations_with_errors.append(translation.pk)

pontoon/pretranslation/tasks.py

@@ -21,6 +21,7 @@
 from pontoon.base.tasks import PontoonTask
 from pontoon.checks.libraries import run_checks
 from pontoon.checks.utils import bulk_run_checks
+from pontoon.translations.utils import parse_db_string_to_json
 
 from . import AUTHORS
 from .pretranslate import get_pretranslation
@@ -135,11 +136,16 @@ def pretranslate(project: Project, paths: set[str] | None):
                     log.info(f"Pretranslation error: {e}")
                     continue
 
+            string, author_key = pretranslation
+            value, properties = parse_db_string_to_json(entity.resource.format, string)
+
             t = Translation(
                 entity=entity,
                 locale=locale,
-                string=pretranslation[0],
-                user=pt_authors[pretranslation[1]],
+                string=string,
+                value=value,
+                properties=properties,
+                user=pt_authors[author_key],
                 approved=False,
                 pretranslated=True,
                 active=True,