Add ability to compose multi-value Machinery translations

pontoon/api/views.py

@@ -35,6 +35,7 @@
     Term,
     TermTranslation,
 )
+from pontoon.translations.utils import parse_source_string_to_json
 
 from .serializers import (
     TRANSLATION_STATS_FIELDS,
@@ -523,11 +524,21 @@ def post(self, request):
 
         locale = generics.get_object_or_404(Locale, code=locale)
 
+        fmt = resource_format or None
         project = SimpleNamespace(slug="temp-project")
-        resource = SimpleNamespace(project=project, format=resource_format or None)
-        entity = SimpleNamespace(resource=resource, string=text)
+        resource = SimpleNamespace(project=project, format=fmt)
 
         try:
+            # Parsing happens here (not before) so invalid syntax for the chosen
+            # format is reported as a 400 rather than escaping as a 500.
+            key, value, properties = parse_source_string_to_json(fmt, text)
+            entity = SimpleNamespace(
+                resource=resource,
+                string=text,
+                key=key,
+                value=value,
+                properties=properties,
+            )
             pretranslation = get_pretranslation(entity=entity, locale=locale)
         except Exception as e:
             return Response(

pontoon/base/tests/models/test_entity.py

@@ -36,6 +36,7 @@ def entity_test_models(translation_a, locale_b):
     )
     entity_a.string = "Entity zero"
     entity_a.key = [entity_a.string]
+    entity_a.value = [entity_a.string]
     entity_a.order = 0
     entity_a.save()
     entity_b = EntityFactory(
@@ -294,7 +295,7 @@ def test_entity_project_locale_no_paths(
         },
         "pk": entity_a.pk,
         "original": entity_a.string,
-        "value": [],
+        "value": ["Entity zero"],
         "date_created": entity_a.date_created,
     }
     assert e1["path"] == trX.entity.resource.path

pontoon/machinery/tests/test_composed.py

@@ -0,0 +1,319 @@
+import json
+
+from textwrap import dedent
+from unittest.mock import patch
+
+import pytest
+
+from django.urls import reverse
+
+from pontoon.test.factories import (
+    EntityFactory,
+    LocaleFactory,
+    ResourceFactory,
+    TranslationMemoryFactory,
+)
+
+
+@pytest.fixture
+def fluent_resource(project_a):
+    return ResourceFactory(project=project_a, path="resource.ftl", format="fluent")
+
+
+@pytest.mark.django_db
+def test_composed_bad_request(client, locale_a):
+    """Missing or invalid params should return 400."""
+    url = reverse("pontoon.machinery_composed")
+
+    response = client.get(url)
+    assert response.status_code == 400
+
+    response = client.get(url, {"entity": "not-a-number", "locale": locale_a.code})
+    assert response.status_code == 400
+
+    response = client.get(url, {"entity": "999999999", "locale": locale_a.code})
+    assert response.status_code == 400
+
+
+@pytest.mark.django_db
+def test_composed_single_pattern_message(client, entity_a, locale_a):
+    """Single-pattern messages have nothing to compose and skip cleanly.
+
+    A DTD entity is a single `PatternMessage` with no properties, so composing it
+    would just repeat the per-leaf machinery suggestion; we expect an empty `{}`.
+    """
+    dtd_resource = ResourceFactory(
+        project=entity_a.resource.project, path="r.dtd", format="dtd"
+    )
+    dtd_entity = EntityFactory(resource=dtd_resource, string="Hello")
+
+    url = reverse("pontoon.machinery_composed")
+    response = client.get(
+        url,
+        {
+            "entity": str(dtd_entity.pk),
+            "locale": locale_a.code,
+            "service": "translation-memory",
+        },
+    )
+    assert response.status_code == 200
+    assert json.loads(response.content) == {}
+
+
+@pytest.mark.django_db
+def test_composed_single_pattern_fluent(client, fluent_resource, locale_a):
+    """A Fluent message with a plain pattern value (no selector, no variants)
+    and no attributes skips even though its format supports composition."""
+    fluent_entity = EntityFactory(resource=fluent_resource, string="hello = Hello\n")
+
+    url = reverse("pontoon.machinery_composed")
+    response = client.get(
+        url,
+        {
+            "entity": str(fluent_entity.pk),
+            "locale": locale_a.code,
+            "service": "translation-memory",
+        },
+    )
+    assert response.status_code == 200
+    assert json.loads(response.content) == {}
+
+
+@pytest.mark.django_db
+def test_composed_unknown_service(client, fluent_resource, locale_a):
+    fluent_entity = EntityFactory(resource=fluent_resource, string="hello = Hello\n")
+    url = reverse("pontoon.machinery_composed")
+    response = client.get(
+        url,
+        {
+            "entity": str(fluent_entity.pk),
+            "locale": locale_a.code,
+            "service": "bogus",
+        },
+    )
+    assert response.status_code == 400
+
+
+@pytest.mark.django_db
+def test_composed_mt_service_requires_auth(
+    client, fluent_resource, google_translate_locale
+):
+    """MT services require authentication; TM-only is anonymous-friendly."""
+    fluent_entity = EntityFactory(resource=fluent_resource, string="hello = Hello\n")
+    url = reverse("pontoon.machinery_composed")
+    response = client.get(
+        url,
+        {
+            "entity": str(fluent_entity.pk),
+            "locale": google_translate_locale.code,
+            "service": "google-translate",
+        },
+    )
+    assert response.status_code == 403
+
+
+@pytest.mark.django_db
+def test_composed_tm_only_full_hit(client, fluent_resource, entity_a, locale_a):
+    """When every leaf has a TM hit, TM-only returns a composed Fluent string."""
+    fluent_string = dedent(
+        """
+        button = Click Me
+            .title = Tooltip text
+        """
+    )
+    fluent_entity = EntityFactory(resource=fluent_resource, string=fluent_string)
+
+    TranslationMemoryFactory.create(
+        entity=entity_a, source="Click Me", target="TM_value", locale=locale_a
+    )
+    TranslationMemoryFactory.create(
+        entity=entity_a,
+        source="Tooltip text",
+        target="TM_tooltip",
+        locale=locale_a,
+    )
+
+    url = reverse("pontoon.machinery_composed")
+    response = client.get(
+        url,
+        {
+            "entity": str(fluent_entity.pk),
+            "locale": locale_a.code,
+            "service": "translation-memory",
+        },
+    )
+    assert response.status_code == 200
+    body = json.loads(response.content)
+    assert body["original"] == fluent_string
+    assert "TM_value" in body["translation"]
+    assert "TM_tooltip" in body["translation"]
+    assert body["sources"] == ["translation-memory"]
+    # Every leaf is a 100% TM match, so the composed result is a full TM match.
+    assert body["quality"] == 100
+
+
+@pytest.mark.django_db
+def test_composed_expands_source_plural_for_target_locale(
+    client, fluent_resource, entity_a
+):
+    """A source with a single plural variant composes to multiple target patterns.
+
+    en-US declares only `*[other]`, but a locale with several CLDR plural
+    categories (here one/two/few/other) needs a pattern per category. The walk
+    expands the selector to the locale's categories, so the entity counts as
+    multi-pattern even though the source has a single variant, and a composed
+    suggestion is returned.
+    """
+    locale = LocaleFactory(code="sl-test", name="Plural", cldr_plurals="1,2,3,5")
+
+    fluent_string = dedent(
+        """\
+        popup =
+            { $count ->
+               *[other] Many popups.
+            }
+        """
+    )
+    fluent_entity = EntityFactory(resource=fluent_resource, string=fluent_string)
+
+    TranslationMemoryFactory.create(
+        entity=entity_a, source="Many popups.", target="TM_popups", locale=locale
+    )
+
+    url = reverse("pontoon.machinery_composed")
+    response = client.get(
+        url,
+        {
+            "entity": str(fluent_entity.pk),
+            "locale": locale.code,
+            "service": "translation-memory",
+        },
+    )
+    assert response.status_code == 200
+    body = json.loads(response.content)
+    # The single `*[other]` source expands to all four target plural categories.
+    translation = body["translation"]
+    assert "[one]" in translation
+    assert "[two]" in translation
+    assert "[few]" in translation
+    assert "*[other]" in translation
+    assert translation.count("TM_popups") == 4
+    assert body["sources"] == ["translation-memory"]
+    assert body["quality"] == 100
+
+
+@pytest.mark.django_db
+def test_composed_tm_only_partial_returns_empty(
+    client, fluent_resource, entity_a, locale_a
+):
+    """TM-only mode emits no result when any leaf misses TM."""
+    fluent_string = dedent(
+        """
+        button = Click Me
+            .title = Tooltip text
+        """
+    )
+    fluent_entity = EntityFactory(resource=fluent_resource, string=fluent_string)
+
+    # Only one of the two leaves has a TM match.
+    TranslationMemoryFactory.create(
+        entity=entity_a, source="Click Me", target="TM_value", locale=locale_a
+    )
+
+    url = reverse("pontoon.machinery_composed")
+    response = client.get(
+        url,
+        {
+            "entity": str(fluent_entity.pk),
+            "locale": locale_a.code,
+            "service": "translation-memory",
+        },
+    )
+    assert response.status_code == 200
+    assert json.loads(response.content) == {}
+
+
+@pytest.mark.django_db
+def test_composed_tm_excludes_current_entity(client, fluent_resource, locale_a):
+    """TM matches belonging to the composed entity itself are excluded.
+
+    Once the entity is translated its leaves become TM entries; like regular TM
+    matches, those must not be suggested back, so a TM-only composition that
+    relies solely on them produces no result.
+    """
+    fluent_string = dedent(
+        """
+        button = Click Me
+            .title = Tooltip text
+        """
+    )
+    fluent_entity = EntityFactory(resource=fluent_resource, string=fluent_string)
+
+    # Both leaves only match TM entries that belong to this same entity.
+    TranslationMemoryFactory.create(
+        entity=fluent_entity, source="Click Me", target="TM_value", locale=locale_a
+    )
+    TranslationMemoryFactory.create(
+        entity=fluent_entity,
+        source="Tooltip text",
+        target="TM_tooltip",
+        locale=locale_a,
+    )
+
+    url = reverse("pontoon.machinery_composed")
+    response = client.get(
+        url,
+        {
+            "entity": str(fluent_entity.pk),
+            "locale": locale_a.code,
+            "service": "translation-memory",
+        },
+    )
+    assert response.status_code == 200
+    assert json.loads(response.content) == {}
+
+
+@patch("pontoon.pretranslation.pretranslate.get_google_translate_data")
+@pytest.mark.django_db
+def test_composed_hybrid_tm_and_mt(
+    gt_mock,
+    member,
+    fluent_resource,
+    entity_a,
+    google_translate_locale,
+    google_translate_api_key,
+):
+    """TM hit for one leaf, MT fallback for the other — `sources` reflects the mix."""
+    gt_mock.return_value = "MT_tooltip"
+
+    fluent_string = dedent(
+        """
+        button = Click Me
+            .title = Tooltip text
+        """
+    )
+    fluent_entity = EntityFactory(resource=fluent_resource, string=fluent_string)
+
+    TranslationMemoryFactory.create(
+        entity=entity_a,
+        source="Click Me",
+        target="TM_value",
+        locale=google_translate_locale,
+    )
+
+    url = reverse("pontoon.machinery_composed")
+    response = member.client.get(
+        url,
+        {
+            "entity": str(fluent_entity.pk),
+            "locale": google_translate_locale.code,
+            "service": "google-translate",
+        },
+    )
+    assert response.status_code == 200
+    body = json.loads(response.content)
+    assert "TM_value" in body["translation"]
+    assert "MT_tooltip" in body["translation"]
+    assert set(body["sources"]) == {"translation-memory", "google-translate"}
+    # MT-assisted results have no meaningful aggregate quality score.
+    assert "quality" not in body