diff --git a/pontoon/api/views.py b/pontoon/api/views.py index 4294d440eb..2b8274256d 100644 --- a/pontoon/api/views.py +++ b/pontoon/api/views.py @@ -35,6 +35,7 @@ Term, TermTranslation, ) +from pontoon.translations.utils import parse_source_string_to_json from .serializers import ( TRANSLATION_STATS_FIELDS, @@ -523,11 +524,21 @@ def post(self, request): locale = generics.get_object_or_404(Locale, code=locale) + fmt = resource_format or None project = SimpleNamespace(slug="temp-project") - resource = SimpleNamespace(project=project, format=resource_format or None) - entity = SimpleNamespace(resource=resource, string=text) + resource = SimpleNamespace(project=project, format=fmt) try: + # Parsing happens here (not before) so invalid syntax for the chosen + # format is reported as a 400 rather than escaping as a 500. + key, value, properties = parse_source_string_to_json(fmt, text) + entity = SimpleNamespace( + resource=resource, + string=text, + key=key, + value=value, + properties=properties, + ) pretranslation = get_pretranslation(entity=entity, locale=locale) except Exception as e: return Response( diff --git a/pontoon/base/tests/models/test_entity.py b/pontoon/base/tests/models/test_entity.py index b162e39b6d..85653e5d31 100644 --- a/pontoon/base/tests/models/test_entity.py +++ b/pontoon/base/tests/models/test_entity.py @@ -36,6 +36,7 @@ def entity_test_models(translation_a, locale_b): ) entity_a.string = "Entity zero" entity_a.key = [entity_a.string] + entity_a.value = [entity_a.string] entity_a.order = 0 entity_a.save() entity_b = EntityFactory( @@ -294,7 +295,7 @@ def test_entity_project_locale_no_paths( }, "pk": entity_a.pk, "original": entity_a.string, - "value": [], + "value": ["Entity zero"], "date_created": entity_a.date_created, } assert e1["path"] == trX.entity.resource.path diff --git a/pontoon/machinery/tests/test_composed.py b/pontoon/machinery/tests/test_composed.py new file mode 100644 index 0000000000..58222f5685 --- /dev/null +++ b/pontoon/machinery/tests/test_composed.py @@ -0,0 +1,319 @@ +import json + +from textwrap import dedent +from unittest.mock import patch + +import pytest + +from django.urls import reverse + +from pontoon.test.factories import ( + EntityFactory, + LocaleFactory, + ResourceFactory, + TranslationMemoryFactory, +) + + +@pytest.fixture +def fluent_resource(project_a): + return ResourceFactory(project=project_a, path="resource.ftl", format="fluent") + + +@pytest.mark.django_db +def test_composed_bad_request(client, locale_a): + """Missing or invalid params should return 400.""" + url = reverse("pontoon.machinery_composed") + + response = client.get(url) + assert response.status_code == 400 + + response = client.get(url, {"entity": "not-a-number", "locale": locale_a.code}) + assert response.status_code == 400 + + response = client.get(url, {"entity": "999999999", "locale": locale_a.code}) + assert response.status_code == 400 + + +@pytest.mark.django_db +def test_composed_single_pattern_message(client, entity_a, locale_a): + """Single-pattern messages have nothing to compose and skip cleanly. + + A DTD entity is a single `PatternMessage` with no properties, so composing it + would just repeat the per-leaf machinery suggestion; we expect an empty `{}`. + """ + dtd_resource = ResourceFactory( + project=entity_a.resource.project, path="r.dtd", format="dtd" + ) + dtd_entity = EntityFactory(resource=dtd_resource, string="Hello") + + url = reverse("pontoon.machinery_composed") + response = client.get( + url, + { + "entity": str(dtd_entity.pk), + "locale": locale_a.code, + "service": "translation-memory", + }, + ) + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@pytest.mark.django_db +def test_composed_single_pattern_fluent(client, fluent_resource, locale_a): + """A Fluent message with a plain pattern value (no selector, no variants) + and no attributes skips even though its format supports composition.""" + fluent_entity = EntityFactory(resource=fluent_resource, string="hello = Hello\n") + + url = reverse("pontoon.machinery_composed") + response = client.get( + url, + { + "entity": str(fluent_entity.pk), + "locale": locale_a.code, + "service": "translation-memory", + }, + ) + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@pytest.mark.django_db +def test_composed_unknown_service(client, fluent_resource, locale_a): + fluent_entity = EntityFactory(resource=fluent_resource, string="hello = Hello\n") + url = reverse("pontoon.machinery_composed") + response = client.get( + url, + { + "entity": str(fluent_entity.pk), + "locale": locale_a.code, + "service": "bogus", + }, + ) + assert response.status_code == 400 + + +@pytest.mark.django_db +def test_composed_mt_service_requires_auth( + client, fluent_resource, google_translate_locale +): + """MT services require authentication; TM-only is anonymous-friendly.""" + fluent_entity = EntityFactory(resource=fluent_resource, string="hello = Hello\n") + url = reverse("pontoon.machinery_composed") + response = client.get( + url, + { + "entity": str(fluent_entity.pk), + "locale": google_translate_locale.code, + "service": "google-translate", + }, + ) + assert response.status_code == 403 + + +@pytest.mark.django_db +def test_composed_tm_only_full_hit(client, fluent_resource, entity_a, locale_a): + """When every leaf has a TM hit, TM-only returns a composed Fluent string.""" + fluent_string = dedent( + """ + button = Click Me + .title = Tooltip text + """ + ) + fluent_entity = EntityFactory(resource=fluent_resource, string=fluent_string) + + TranslationMemoryFactory.create( + entity=entity_a, source="Click Me", target="TM_value", locale=locale_a + ) + TranslationMemoryFactory.create( + entity=entity_a, + source="Tooltip text", + target="TM_tooltip", + locale=locale_a, + ) + + url = reverse("pontoon.machinery_composed") + response = client.get( + url, + { + "entity": str(fluent_entity.pk), + "locale": locale_a.code, + "service": "translation-memory", + }, + ) + assert response.status_code == 200 + body = json.loads(response.content) + assert body["original"] == fluent_string + assert "TM_value" in body["translation"] + assert "TM_tooltip" in body["translation"] + assert body["sources"] == ["translation-memory"] + # Every leaf is a 100% TM match, so the composed result is a full TM match. + assert body["quality"] == 100 + + +@pytest.mark.django_db +def test_composed_expands_source_plural_for_target_locale( + client, fluent_resource, entity_a +): + """A source with a single plural variant composes to multiple target patterns. + + en-US declares only `*[other]`, but a locale with several CLDR plural + categories (here one/two/few/other) needs a pattern per category. The walk + expands the selector to the locale's categories, so the entity counts as + multi-pattern even though the source has a single variant, and a composed + suggestion is returned. + """ + locale = LocaleFactory(code="sl-test", name="Plural", cldr_plurals="1,2,3,5") + + fluent_string = dedent( + """\ + popup = + { $count -> + *[other] Many popups. + } + """ + ) + fluent_entity = EntityFactory(resource=fluent_resource, string=fluent_string) + + TranslationMemoryFactory.create( + entity=entity_a, source="Many popups.", target="TM_popups", locale=locale + ) + + url = reverse("pontoon.machinery_composed") + response = client.get( + url, + { + "entity": str(fluent_entity.pk), + "locale": locale.code, + "service": "translation-memory", + }, + ) + assert response.status_code == 200 + body = json.loads(response.content) + # The single `*[other]` source expands to all four target plural categories. + translation = body["translation"] + assert "[one]" in translation + assert "[two]" in translation + assert "[few]" in translation + assert "*[other]" in translation + assert translation.count("TM_popups") == 4 + assert body["sources"] == ["translation-memory"] + assert body["quality"] == 100 + + +@pytest.mark.django_db +def test_composed_tm_only_partial_returns_empty( + client, fluent_resource, entity_a, locale_a +): + """TM-only mode emits no result when any leaf misses TM.""" + fluent_string = dedent( + """ + button = Click Me + .title = Tooltip text + """ + ) + fluent_entity = EntityFactory(resource=fluent_resource, string=fluent_string) + + # Only one of the two leaves has a TM match. + TranslationMemoryFactory.create( + entity=entity_a, source="Click Me", target="TM_value", locale=locale_a + ) + + url = reverse("pontoon.machinery_composed") + response = client.get( + url, + { + "entity": str(fluent_entity.pk), + "locale": locale_a.code, + "service": "translation-memory", + }, + ) + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@pytest.mark.django_db +def test_composed_tm_excludes_current_entity(client, fluent_resource, locale_a): + """TM matches belonging to the composed entity itself are excluded. + + Once the entity is translated its leaves become TM entries; like regular TM + matches, those must not be suggested back, so a TM-only composition that + relies solely on them produces no result. + """ + fluent_string = dedent( + """ + button = Click Me + .title = Tooltip text + """ + ) + fluent_entity = EntityFactory(resource=fluent_resource, string=fluent_string) + + # Both leaves only match TM entries that belong to this same entity. + TranslationMemoryFactory.create( + entity=fluent_entity, source="Click Me", target="TM_value", locale=locale_a + ) + TranslationMemoryFactory.create( + entity=fluent_entity, + source="Tooltip text", + target="TM_tooltip", + locale=locale_a, + ) + + url = reverse("pontoon.machinery_composed") + response = client.get( + url, + { + "entity": str(fluent_entity.pk), + "locale": locale_a.code, + "service": "translation-memory", + }, + ) + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@patch("pontoon.pretranslation.pretranslate.get_google_translate_data") +@pytest.mark.django_db +def test_composed_hybrid_tm_and_mt( + gt_mock, + member, + fluent_resource, + entity_a, + google_translate_locale, + google_translate_api_key, +): + """TM hit for one leaf, MT fallback for the other — `sources` reflects the mix.""" + gt_mock.return_value = "MT_tooltip" + + fluent_string = dedent( + """ + button = Click Me + .title = Tooltip text + """ + ) + fluent_entity = EntityFactory(resource=fluent_resource, string=fluent_string) + + TranslationMemoryFactory.create( + entity=entity_a, + source="Click Me", + target="TM_value", + locale=google_translate_locale, + ) + + url = reverse("pontoon.machinery_composed") + response = member.client.get( + url, + { + "entity": str(fluent_entity.pk), + "locale": google_translate_locale.code, + "service": "google-translate", + }, + ) + assert response.status_code == 200 + body = json.loads(response.content) + assert "TM_value" in body["translation"] + assert "MT_tooltip" in body["translation"] + assert set(body["sources"]) == {"translation-memory", "google-translate"} + # MT-assisted results have no meaningful aggregate quality score. + assert "quality" not in body diff --git a/pontoon/machinery/tests/test_views.py b/pontoon/machinery/tests/test_views.py index 45042a7403..198d2dd2d9 100644 --- a/pontoon/machinery/tests/test_views.py +++ b/pontoon/machinery/tests/test_views.py @@ -29,7 +29,7 @@ @pytest.mark.django_db def test_view_microsoft_translator_not_logged_in(client, ms_locale, ms_api_key): url = reverse("pontoon.microsoft_translator") - response = client.get(url, {"text": "text", "locale": ms_locale.ms_translator_code}) + response = client.get(url, {"text": "text", "locale": ms_locale.code}) assert response.status_code == 302 @@ -43,7 +43,7 @@ def test_view_microsoft_translator(member, ms_locale, ms_api_key): m.post("https://api.cognitive.microsofttranslator.com/translate", json=data) response = member.client.get( url, - {"text": "text", "locale": ms_locale.ms_translator_code}, + {"text": "text", "locale": ms_locale.code}, ) assert response.status_code == 200 @@ -68,7 +68,7 @@ def test_view_microsoft_translator_bad_locale(member, ms_locale, ms_api_key): url = reverse("pontoon.microsoft_translator") response = member.client.get(url, {"text": "text", "locale": "bad"}) - assert response.status_code == 401 + assert response.status_code == 400 @pytest.mark.django_db @@ -76,9 +76,7 @@ def test_view_microsoft_translator_missing_api_key(member, ms_locale, settings): settings.MICROSOFT_TRANSLATOR_API_KEY = "" cache.clear() url = reverse("pontoon.microsoft_translator") - response = member.client.get( - url, {"text": "text", "locale": ms_locale.ms_translator_code} - ) + response = member.client.get(url, {"text": "text", "locale": ms_locale.code}) assert response.status_code == 400 @@ -91,9 +89,7 @@ def test_view_microsoft_translator_api_http_error(member, ms_locale, ms_api_key) "https://api.cognitive.microsofttranslator.com/translate", status_code=401, ) - response = member.client.get( - url, {"text": "text", "locale": ms_locale.ms_translator_code} - ) + response = member.client.get(url, {"text": "text", "locale": ms_locale.code}) assert response.status_code == 401 @@ -106,9 +102,7 @@ def test_view_microsoft_translator_api_connection_error(member, ms_locale, ms_ap "https://api.cognitive.microsofttranslator.com/translate", exc=requests.exceptions.ConnectionError, ) - response = member.client.get( - url, {"text": "text", "locale": ms_locale.ms_translator_code} - ) + response = member.client.get(url, {"text": "text", "locale": ms_locale.code}) assert response.status_code == 500 @@ -121,9 +115,7 @@ def test_view_microsoft_translator_api_error_in_response(member, ms_locale, ms_a "https://api.cognitive.microsofttranslator.com/translate", json={"error": {"code": 400000, "message": "Bad request"}}, ) - response = member.client.get( - url, {"text": "text", "locale": ms_locale.ms_translator_code} - ) + response = member.client.get(url, {"text": "text", "locale": ms_locale.code}) assert response.status_code == 400 @@ -254,15 +246,11 @@ def test_view_microsoft_translator_cache(member, ms_locale, ms_api_key): data = [{"translations": [{"text": "target"}]}] m.post("https://api.cognitive.microsofttranslator.com/translate", json=data) - response1 = member.client.get( - url, {"text": "text", "locale": ms_locale.ms_translator_code} - ) + response1 = member.client.get(url, {"text": "text", "locale": ms_locale.code}) assert len(m.request_history) == 1 # Second identical request should be served from cache - response2 = member.client.get( - url, {"text": "text", "locale": ms_locale.ms_translator_code} - ) + response2 = member.client.get(url, {"text": "text", "locale": ms_locale.code}) assert len(m.request_history) == 1 assert json.loads(response1.content) == json.loads(response2.content) diff --git a/pontoon/machinery/urls.py b/pontoon/machinery/urls.py index 60e5f0f8a3..a296274ad5 100644 --- a/pontoon/machinery/urls.py +++ b/pontoon/machinery/urls.py @@ -10,6 +10,11 @@ views.translation_memory, name="pontoon.translation_memory", ), + path( + "machinery-composed/", + views.machinery_composed, + name="pontoon.machinery_composed", + ), path( "concordance-search/", views.concordance_search, diff --git a/pontoon/machinery/utils.py b/pontoon/machinery/utils.py index 008298195e..7bd9e55fb8 100644 --- a/pontoon/machinery/utils.py +++ b/pontoon/machinery/utils.py @@ -151,7 +151,8 @@ def get_google_automl_translation( return translation -def get_microsoft_translator_data(text, locale_code): +def get_microsoft_translator_data(text, locale, preserve_placeables=False): + locale_code = locale.ms_translator_code cache_key = get_machinery_service_cache_key( "microsoft_translator", text, locale_code ) diff --git a/pontoon/machinery/views.py b/pontoon/machinery/views.py index 36d77cff1b..0b22a70070 100644 --- a/pontoon/machinery/views.py +++ b/pontoon/machinery/views.py @@ -6,6 +6,7 @@ import requests +from moz.l10n.model import SelectMessage from sacremoses import MosesDetokenizer from django.contrib.auth.decorators import login_required @@ -23,6 +24,7 @@ get_microsoft_translator_data, get_translation_memory_data, ) +from pontoon.pretranslation.pretranslate import MTEngine, Pretranslation from pontoon.terminology.models import Term from .openai_service import OpenAIService @@ -31,6 +33,17 @@ log = logging.getLogger(__name__) +def _pattern_count(message): + """Number of independently-translatable patterns in a parsed message. + + A `PatternMessage` has one; a `SelectMessage` has one per variant. Used to + tell single-pattern messages (nothing to compose) from multi-pattern ones. + """ + if isinstance(message, SelectMessage): + return len(message.variants) + return 1 if message is not None else 0 + + def _machinery_error_response(service_name, e): log.error(f"{service_name} error: {e}") if isinstance(e, requests.exceptions.HTTPError): @@ -63,6 +76,118 @@ def translation_memory(request): return JsonResponse(data, safe=False) +def machinery_composed(request): + """ + Return a composed multi-value translation for a multi-pattern entity. + + Each translatable leaf — the entity's value and every property, with a + selector message contributing one leaf per variant — is looked up in + Translation Memory; leaves without a 100% TM match fall back to the requested + MT service. Mirrors the Pretranslation pipeline so the Machinery panel can + surface a directly-pasteable composed translation alongside the per-leaf + results. Single-pattern entities have nothing to compose and yield an empty + response. + + Query params: + entity: Entity pk + locale: Locale code + service: one of `translation-memory`, `google-translate`, + `microsoft-translator`. Defaults to `google-translate`. + `translation-memory` disables MT fallback. + """ + try: + entity_pk = int(request.GET["entity"]) + locale = Locale.objects.get(code=request.GET["locale"]) + service = request.GET.get("service", "google-translate") + entity = Entity.objects.select_related("resource").get(pk=entity_pk) + except ( + Entity.DoesNotExist, + Locale.DoesNotExist, + MultiValueDictKeyError, + ValueError, + ) as e: + return JsonResponse( + {"status": False, "message": f"Bad Request: {e}"}, status=400 + ) + + match service: + case "translation-memory": + # TM-only: no MT engine, so MT is never called. + mt_engine = None + case "google-translate": + mt_engine = MTEngine.GOOGLE_TRANSLATE + case "microsoft-translator": + mt_engine = MTEngine.MICROSOFT_TRANSLATOR + case _: + return JsonResponse( + { + "status": False, + "message": f"Bad Request: unknown service `{service}`", + }, + status=400, + ) + + # MT services call an external provider; translation-memory is anonymous-friendly. + if mt_engine is not None and not request.user.is_authenticated: + return JsonResponse( + {"status": False, "message": "Authentication required"}, status=403 + ) + + try: + pt = Pretranslation( + entity, + locale, + preserve_placeables=False, + mt_engine=mt_engine, + exclude_entity=True, + ) + value, properties = pt.walk_entity() + translation = pt.serialize(value, properties) + except ValueError: + # Raised when a leaf has no TM match and MT is unavailable. Compose + # endpoint treats this as "nothing to show" rather than an error. + return JsonResponse({}) + except Exception as e: + return _machinery_error_response(f"Composed machinery ({service})", e) + + # Only multi-pattern targets — those with multiple properties and/or selector + # variants — have something to compose. A single-pattern target composes to + # the same string the per-leaf machinery already returns, so there is nothing + # extra to show. We count the *target* patterns (not the source): walk_entity() + # expands plural selectors to the locale's CLDR categories, so a source with a + # single plural variant (e.g. en-US `*[other]`) still yields multiple patterns + # for locales like Slovenian (one/two/few/other). + pattern_count = _pattern_count(value) + sum( + _pattern_count(prop) for prop in properties.values() + ) + if pattern_count < 2 or not pt.services or translation == entity.string: + return JsonResponse({}) + + # Preserve insertion order while deduplicating. Map the internal service + # identifiers to the SourceType values the frontend uses for the badge. + badges = { + "tm": "translation-memory", + "gt": "google-translate", + "ms": "microsoft-translator", + } + sources_used = list(dict.fromkeys(badges.get(s, s) for s in pt.services)) + + response = { + "original": entity.string, + "translation": translation, + "sources": sources_used, + } + + # When every leaf came from a 100% TM match (`pattern()` only accepts exact + # source matches from TM), the composed string is a complete TM match — give + # it the same quality badge regular TM matches get. Hybrid results that fall + # back to MT for any leaf have no meaningful aggregate score. + if set(pt.services) == {"tm"}: + response["quality"] = 100 + + return JsonResponse(response) + + def concordance_search(request): """Search for translations in the internal translations memory.""" try: @@ -115,12 +240,12 @@ def microsoft_translator(request): """Get translation from Microsoft machine translation service.""" try: text = request.GET["text"] - locale_code = request.GET["locale"] + locale = Locale.objects.get(code=request.GET["locale"]) - if not locale_code: - raise ValueError("Locale code is empty") + if not locale.ms_translator_code: + raise ValueError("Locale code not supported") - except (MultiValueDictKeyError, ValueError) as e: + except (Locale.DoesNotExist, MultiValueDictKeyError, ValueError) as e: return JsonResponse( {"status": False, "message": f"Bad Request: {e}"}, status=400, @@ -128,7 +253,7 @@ def microsoft_translator(request): try: return JsonResponse( - {"translation": get_microsoft_translator_data(text, locale_code)} + {"translation": get_microsoft_translator_data(text, locale)} ) except Exception as e: return _machinery_error_response("Microsoft Translator", e) diff --git a/pontoon/pretranslation/pretranslate.py b/pontoon/pretranslation/pretranslate.py index 06ce069e0e..ea131cb8a6 100644 --- a/pontoon/pretranslation/pretranslate.py +++ b/pontoon/pretranslation/pretranslate.py @@ -1,16 +1,16 @@ from copy import deepcopy +from enum import Enum from re import compile -from typing import Literal +from typing import Callable, Literal -from fluent.syntax import FluentSerializer, ast as FTL +from fluent.syntax import ast as FTL from fluent.syntax.serializer import serialize_expression from moz.l10n.formats import Format from moz.l10n.formats.fluent import ( - fluent_astify_entry, fluent_astify_message, fluent_parse_entry, ) -from moz.l10n.message import parse_message, serialize_message +from moz.l10n.message import message_from_json from moz.l10n.model import ( CatchallKey, Entry, @@ -22,12 +22,49 @@ ) from pontoon.base.models import Entity, Locale, Resource, TranslationMemoryEntry -from pontoon.machinery.utils import get_google_translate_data +from pontoon.machinery.utils import ( + get_google_translate_data, + get_microsoft_translator_data, +) +from pontoon.sync.formats import as_string pt_placeholder = compile(r"{ *\$(\d+) *}") +MTService = Callable[..., str] + + +class MTEngine(Enum): + """A machine-translation engine used as fallback when no 100% TM match exists. + + Each member bundles the facets that previously travelled as three separate + ``Pretranslation`` arguments: the identifier recorded in + ``Pretranslation.services``, the ``Locale`` attribute that says whether the + engine supports a given locale, and the callable that performs the request. + """ + + GOOGLE_TRANSLATE = ("gt", "google_translate_code") + MICROSOFT_TRANSLATOR = ("ms", "ms_translator_code") + + def __init__(self, service_name: str, locale_code_attr: str): + self.service_name = service_name + self.locale_code_attr = locale_code_attr + + def supports(self, locale: Locale) -> bool: + return bool(getattr(locale, self.locale_code_attr)) + + @property + def service(self) -> MTService: + # Resolved lazily (not stored on the member) so tests can patch the + # module-level service callables. + match self: + case MTEngine.GOOGLE_TRANSLATE: + return get_google_translate_data + case MTEngine.MICROSOFT_TRANSLATOR: + return get_microsoft_translator_data + + def get_pretranslation( entity: Entity, locale: Locale, preserve_placeables: bool = False ) -> tuple[str, Literal["gt", "tm"]]: @@ -42,39 +79,9 @@ def get_pretranslation( - a pretranslation of the entity - a pretranslation service identifier, either "gt" or "tm" """ - pt = Pretranslation(entity, locale, preserve_placeables) - if entity.resource.format == Resource.Format.FLUENT: - entry = fluent_parse_entry(entity.string, with_linepos=False) - if entry.value: - pt.message(entry.value) - accesskeys: list[tuple[str, Message]] = [] - for key, prop in entry.properties.items(): - if key.endswith("accesskey"): - accesskeys.append((key, prop)) - else: - pt.message(prop) - for key, prop in accesskeys: - set_accesskey(entry, key, prop) - pt_res = FluentSerializer().serialize_entry( - fluent_astify_entry(entry, escape_syntax=False) - ) - else: - if entity.resource.format in { - Resource.Format.ANDROID, - Resource.Format.GETTEXT, - Resource.Format.WEBEXT, - Resource.Format.XCODE, - Resource.Format.XLIFF, - }: - format = Format.mf2 - msg = parse_message(format, entity.string) - else: - format = None - msg = PatternMessage([entity.string]) - pt.message(msg) - pt_res = serialize_message(format, msg) - + value, properties = pt.walk_entity() + pt_res = pt.serialize(value, properties) pt_service = max(set(pt.services), key=pt.services.count) if pt.services else "tm" return (pt_res, pt_service) @@ -83,10 +90,31 @@ class Pretranslation: format: Format | None locale: Locale preserve_placeables: bool - services: list[Literal["gt", "tm"]] + services: list[str] source: str - def __init__(self, entity: Entity, locale: Locale, preserve_placeables: bool): + def __init__( + self, + entity: Entity, + locale: Locale, + preserve_placeables: bool, + *, + mt_engine: MTEngine | None = MTEngine.GOOGLE_TRANSLATE, + exclude_entity: bool = False, + ): + """ + :param mt_engine: Machine-translation engine invoked when a leaf has no + 100% TM match. ``None`` disables MT (TM-only): a leaf that can't be + served from TM then raises ``ValueError``. MT is likewise skipped + when the engine doesn't support the locale. Defaults to Google + Translate. + :param exclude_entity: If True, the entity's own TM entries are excluded + from per-leaf TM lookups, matching ``get_translation_memory_data``. + A leaf that can only be served by the entity's own translation then + has no TM match, so a composed result is not reconstructed from the + current entity. Defaults to False. + """ + self.entity = entity match entity.resource.format: case Resource.Format.FLUENT: self.format = Format.fluent @@ -104,6 +132,60 @@ def __init__(self, entity: Entity, locale: Locale, preserve_placeables: bool): self.locale = locale self.preserve_placeables = preserve_placeables self.services = [] + # `service` is resolved here (not stored on the enum member) so tests can + # patch the module-level service callables. + self.mt_service = mt_engine.service if mt_engine is not None else None + self.mt_service_name = mt_engine.service_name if mt_engine is not None else "" + self.mt_supported = mt_engine is not None and mt_engine.supports(locale) + self.exclude_entity = exclude_entity + + def walk_entity(self) -> tuple[Message, dict[str, Message]]: + """ + Walk the entity, translating each leaf via TM (then MT fallback), and + return the translated `(value, properties)`. Serialization to a source + string is left to the caller (see `serialize`). + + For Fluent, each value, attribute, and selector variant is translated + independently and recomposed. Accesskey attributes are derived from the + translated label after all other leaves are filled. For MF2-handled + formats (Android, Gettext, Webext, Xcode, Xliff), variants are walked + with plural-category handling. All other formats are treated as a + single leaf. + """ + entity = self.entity + value = message_from_json(entity.value) + if value: + self.message(value) + + properties = ( + {key: message_from_json(prop) for key, prop in entity.properties.items()} + if entity.properties + else {} + ) + + accesskeys: list[tuple[str, Message]] = [] + for key, prop in properties.items(): + if key.endswith("accesskey"): + accesskeys.append((key, prop)) + else: + self.message(prop) + + # TODO: refactor `set_accesskey` to accept the value/properties directly + # so this temporary Entry is not required. It currently reads the label + # from `entry.value`/`entry.properties`, so reconstruct an Entry from the + # (translated) value and properties. + entry = Entry(id=tuple(entity.key), value=value, properties=properties) + for key, prop in accesskeys: + set_accesskey(entry, key, prop) + + return value, properties + + def serialize(self, value: Message, properties: dict[str, Message]) -> str: + """Serialize translated `(value, properties)` back to a source string.""" + entry = Entry(id=tuple(self.entity.key), value=value, properties=properties) + # `escape_syntax=False` keeps literal `{`/`}` from MT output raw, matching + # the prior pretranslation behavior (unlike sync, which escapes them). + return as_string(self.format, entry, escape_syntax=False) def message(self, msg: Message) -> None: """Modifies `msg`.""" @@ -163,11 +245,14 @@ def pattern(self, pattern: Pattern) -> Pattern: ) if not tm_source or tm_source.isspace(): return pattern - tm_q100 = list( - TranslationMemoryEntry.objects.filter( - locale=self.locale, source=tm_source - ).values_list("target", flat=True) + tm_entries = TranslationMemoryEntry.objects.filter( + locale=self.locale, source=tm_source ) + if self.exclude_entity: + # Mirror get_translation_memory_data(): never suggest the entity's + # own translation back to itself. + tm_entries = tm_entries.exclude(entity=self.entity) + tm_q100 = list(tm_entries.values_list("target", flat=True)) if tm_q100: tm_best = max(set(tm_q100), key=tm_q100.count) self.services.append("tm") @@ -195,14 +280,14 @@ def pattern(self, pattern: Pattern) -> Pattern: if not has_text: return pattern - if self.locale.google_translate_code: - # Try to fetch from Google Translate - gt_translation = get_google_translate_data( + if self.mt_supported: + # Try to fetch from the configured MT service (Google by default) + mt_translation = self.mt_service( text=gt_source, locale=self.locale, preserve_placeables=self.preserve_placeables, ) - self.services.append("gt") + self.services.append(self.mt_service_name) return [ el if idx % 2 == 0 @@ -211,7 +296,7 @@ def pattern(self, pattern: Pattern) -> Pattern: if int(el) < len(placeholders) else "{$" + el + "}" ) - for idx, el in enumerate(pt_placeholder.split(gt_translation)) + for idx, el in enumerate(pt_placeholder.split(mt_translation)) if el != "" ] diff --git a/pontoon/pretranslation/tests/test_pretranslate.py b/pontoon/pretranslation/tests/test_pretranslate.py index 329e6cb675..06e5898e35 100644 --- a/pontoon/pretranslation/tests/test_pretranslate.py +++ b/pontoon/pretranslation/tests/test_pretranslate.py @@ -30,27 +30,20 @@ def test_get_pretranslations_no_match(entity_a, locale_b): @pytest.mark.django_db -def test_get_pretranslations_empty_string(entity_a, locale_b): +def test_get_pretranslations_empty_string(resource_a, locale_b): # Entity.string is an empty string - entity_a.string = "" - response = get_pretranslation(entity_a, locale_b) + entity = EntityFactory(resource=resource_a, string="") + response = get_pretranslation(entity, locale_b) assert response == ("", "tm") @pytest.mark.django_db -def test_get_pretranslations_whitespace(entity_a, locale_b): - # Entity.string is an empty string - entity_a.string = " " - response = get_pretranslation(entity_a, locale_b) - assert response == (" ", "tm") - - entity_a.string = "\t" - response = get_pretranslation(entity_a, locale_b) - assert response == ("\t", "tm") - - entity_a.string = "\n" - response = get_pretranslation(entity_a, locale_b) - assert response == ("\n", "tm") +def test_get_pretranslations_whitespace(resource_a, locale_b): + # Entity.string is whitespace only + for ws in (" ", "\t", "\n"): + entity = EntityFactory(resource=resource_a, string=ws) + response = get_pretranslation(entity, locale_b) + assert response == (ws, "tm") @pytest.mark.django_db diff --git a/pontoon/sync/formats/__init__.py b/pontoon/sync/formats/__init__.py index a1c2f7bf60..ebe84db5db 100644 --- a/pontoon/sync/formats/__init__.py +++ b/pontoon/sync/formats/__init__.py @@ -64,10 +64,14 @@ def _unicode_unescape(m: Match[str]): return m[0].encode("utf-8").decode("unicode_escape") -def _as_string(format: Format | None, entry: Entry[Message]) -> str: +def as_string( + format: Format | None, entry: Entry[Message], *, escape_syntax: bool = True +) -> str: match format: case Format.fluent: - fluent_entry = fluent_astify_entry(entry, comment_str=lambda _: "") + fluent_entry = fluent_astify_entry( + entry, comment_str=lambda _: "", escape_syntax=escape_syntax + ) return _fluent_serializer.serialize_entry(fluent_entry) case Format.android | Format.gettext | Format.webext | Format.xliff: return serialize_message(Format.mf2, entry.value) @@ -95,7 +99,7 @@ def as_repo_translations(res: MozL10nResource[Message]) -> Iterator[RepoTranslat ) yield RepoTranslation( key=section.id + entry.id, - string=_as_string(res.format, entry), + string=as_string(res.format, entry), value=entry.value, properties=entry.properties, fuzzy=fuzzy, @@ -142,7 +146,7 @@ def as_entity( if entry.properties else None ), - string=_as_string(format, entry), + string=as_string(format, entry), comment=entry.comment, meta=[[m.key, m.value] for m in entry.meta], **kwargs, diff --git a/pontoon/test/factories.py b/pontoon/test/factories.py index c35e8a6e4a..ba9c614861 100644 --- a/pontoon/test/factories.py +++ b/pontoon/test/factories.py @@ -119,6 +119,28 @@ class EntityFactory(DjangoModelFactory): class Meta: model = Entity + skip_postgeneration_save = True + + @factory.post_generation + def parsed_value(entity, create, extracted, **kwargs): + # Populate key/value/properties from `string` (as sync does) so code + # that reads the parsed representation works in tests. Skipped when + # `value` is set explicitly; never blocks entity creation. + if entity.value: + return + try: + from pontoon.translations.utils import parse_source_string_to_json + + key, value, properties = parse_source_string_to_json( + entity.resource.format, entity.string + ) + except Exception: + return + entity.key = entity.key or key + entity.value = value + entity.properties = properties + if create: + entity.save() class ChangedEntityLocaleFactory(DjangoModelFactory): diff --git a/pontoon/translations/utils.py b/pontoon/translations/utils.py index cda47b9c2c..cf4d5938a7 100644 --- a/pontoon/translations/utils.py +++ b/pontoon/translations/utils.py @@ -11,10 +11,16 @@ JsonMessage = list[Any] | dict[str, Any] -def parse_db_string_to_json( +def parse_source_string_to_json( res_format: str, source: str, -) -> tuple[JsonMessage, dict[str, JsonMessage] | None]: +) -> tuple[list[str], JsonMessage, dict[str, JsonMessage] | None]: + """ + Parse an entity's `source` string into its `(key, value, properties)` JSON + representation — matching what sync stores on `Entity`, and the inverse of + `serialize_for_db`. Used to build entities that aren't backed by a synced + row (e.g. the pretranslate API and test factories). + """ match res_format: case Resource.Format.FLUENT: fe = fluent_parse_entry(source) @@ -22,7 +28,7 @@ def parse_db_string_to_json( properties = { name: message_to_json(prop) for name, prop in fe.properties.items() } or None - return value, properties + return list(fe.id), value, properties case ( Resource.Format.ANDROID | Resource.Format.GETTEXT @@ -37,9 +43,17 @@ def parse_db_string_to_json( for key in keys: if isinstance(key, CatchallKey): key.value = "other" - return message_to_json(msg), None + return [], message_to_json(msg), None case _: - return [source] if source else [], None + return [], ([source] if source else []), None + + +def parse_db_string_to_json( + res_format: str, + source: str, +) -> tuple[JsonMessage, dict[str, JsonMessage] | None]: + _, value, properties = parse_source_string_to_json(res_format, source) + return value, properties def serialize_for_db( diff --git a/translate/src/api/machinery.test.ts b/translate/src/api/machinery.test.ts index df518c0d3f..bfdd131ec6 100644 --- a/translate/src/api/machinery.test.ts +++ b/translate/src/api/machinery.test.ts @@ -1,4 +1,6 @@ -import { fetchGPTTransform } from './machinery'; +import type { Locale } from '~/context/Locale'; + +import { fetchComposedMachinery, fetchGPTTransform } from './machinery'; import * as base from './utils/base'; vi.mock('./utils/base', () => ({ @@ -56,3 +58,86 @@ describe('fetchGPTTransform', () => { expect(result).toEqual([]); }); }); + +describe('fetchComposedMachinery', () => { + const GET = vi.mocked(base.GET); + const locale = { code: 'fr' } as Locale; + + it('sends entity, locale, and service params', async () => { + GET.mockResolvedValueOnce({ + original: 'src', + translation: 'composed', + sources: ['translation-memory'], + }); + + await fetchComposedMachinery(42, locale, 'translation-memory'); + + const [url, params] = GET.mock.calls[0] as [string, URLSearchParams]; + expect(url).toBe('/machinery-composed/'); + expect(params.get('entity')).toBe('42'); + expect(params.get('locale')).toBe('fr'); + expect(params.get('service')).toBe('translation-memory'); + }); + + it('returns a MachineryTranslation with the response sources', async () => { + GET.mockResolvedValueOnce({ + original: 'Click Me\n .title = Tip', + translation: 'composed-result', + sources: ['translation-memory', 'google-translate'], + }); + + const result = await fetchComposedMachinery(1, locale, 'google-translate'); + + expect(result).toEqual([ + { + sources: ['translation-memory', 'google-translate'], + original: 'Click Me\n .title = Tip', + translation: 'composed-result', + composed: true, + }, + ]); + }); + + it('passes through the quality of a full TM match', async () => { + GET.mockResolvedValueOnce({ + original: 'Click Me\n .title = Tip', + translation: 'composed-result', + sources: ['translation-memory'], + quality: 100, + }); + + const result = await fetchComposedMachinery( + 1, + locale, + 'translation-memory', + ); + + expect(result[0].quality).toBe(100); + expect(result[0].composed).toBe(true); + }); + + it('returns empty array when response is empty', async () => { + GET.mockResolvedValueOnce({}); + const result = await fetchComposedMachinery( + 1, + locale, + 'translation-memory', + ); + expect(result).toEqual([]); + }); + + it('falls back to the requested service when sources is missing', async () => { + GET.mockResolvedValueOnce({ + original: 'src', + translation: 'composed', + }); + + const result = await fetchComposedMachinery( + 1, + locale, + 'microsoft-translator', + ); + + expect(result[0].sources).toEqual(['microsoft-translator']); + }); +}); diff --git a/translate/src/api/machinery.ts b/translate/src/api/machinery.ts index 6a82757bdd..319a40cafa 100644 --- a/translate/src/api/machinery.ts +++ b/translate/src/api/machinery.ts @@ -22,6 +22,10 @@ export type MachineryTranslation = { original: string; translation: string; quality?: number; + // Set for `/machinery-composed/` results, whose `original` and `translation` + // are full entry sources (Fluent attributes, MF2 variants) rather than plain + // strings — the Machinery panel renders these in a rich, multi-field view. + composed?: boolean; projects?: { name: string; slug: string; @@ -131,6 +135,51 @@ export async function fetchTranslationMemory( : []; } +/** + * Return a composed multi-value translation for a Fluent / MF2 entity. + * + * Each translatable leaf (Fluent value/attribute, MF2 variant) is looked up + * in Translation Memory, falling back to the requested MT service when no + * exact TM match exists. Use `service: 'translation-memory'` to disable the + * MT fallback and only emit a result when every leaf has a TM hit. + * + * Returns an empty array when the entity isn't a composable format or when + * no composed translation can be produced (e.g. MT unavailable for locale). + */ +export async function fetchComposedMachinery( + pk: number, + locale: Locale, + service: 'translation-memory' | 'google-translate' | 'microsoft-translator', +): Promise { + const url = '/machinery-composed/'; + const params = { + entity: String(pk), + locale: locale.code, + service, + }; + + const result = (await GET_(url, params)) as { + original?: string; + translation?: string; + sources?: string[]; + quality?: number; + }; + + if (!result || !result.translation || !result.original) { + return []; + } + + return [ + { + sources: (result.sources ?? [service]) as SourceType[], + original: result.original, + translation: result.translation, + quality: result.quality, + composed: true, + }, + ]; +} + /** * Return translation by Google Translate. */ @@ -209,7 +258,7 @@ export async function fetchMicrosoftTranslation( const url = '/microsoft-translator/'; const params = { text: original, - locale: locale.msTranslatorCode, + locale: locale.code, }; const { translation } = (await GET_(url, params)) as { diff --git a/translate/src/context/Editor.test.jsx b/translate/src/context/Editor.test.jsx index 6f512e4ae0..eb65e96b1a 100644 --- a/translate/src/context/Editor.test.jsx +++ b/translate/src/context/Editor.test.jsx @@ -575,6 +575,50 @@ describe('', () => { }); }); + it('distributes a composed helper entry across all fields', () => { + let editor, result, actions; + const Spy = () => { + editor = useContext(EditorData); + result = useContext(EditorResult); + actions = useContext(EditorActions); + return null; + }; + mountSpy(Spy, 'fluent', `key = VALUE\n .title = TITLE\n`); + + const source = ftl` + key = COMPOSED + .title = COMPOSED_TITLE + `; + act(() => + actions.setEditorFromHelpers(source, ['translation-memory'], true, true), + ); + + // The full entry source is spread across the value and attribute fields, + // not dumped into the first field. + expect(editor).toMatchObject({ + sourceView: false, + fields: [ + { + handle: { current: { value: 'COMPOSED' } }, + name: '', + }, + { + handle: { current: { value: 'COMPOSED_TITLE' } }, + name: 'title', + }, + ], + machinery: { manual: true, sources: ['translation-memory'] }, + }); + // The editor result is the rebuilt entry, with the composed value and + // attribute distributed into their respective patterns. + expect(result).toEqual({ + format: 'fluent', + id: 'key', + value: ['COMPOSED'], + attributes: new Map([['title', ['COMPOSED_TITLE']]]), + }); + }); + it('toggles Fluent source view', () => { let editor, result, actions; const Spy = () => { diff --git a/translate/src/context/Editor.tsx b/translate/src/context/Editor.tsx index 931e99020a..d6fb0fb617 100644 --- a/translate/src/context/Editor.tsx +++ b/translate/src/context/Editor.tsx @@ -14,6 +14,7 @@ import { buildMessageEntry, editMessageEntry, editSource, + getPlainMessage, type MessageEntry, parseEntry, requiresSourceView, @@ -97,11 +98,17 @@ export type EditorActions = { /** If `format: 'fluent'`, must be called with the source of a full entry */ setEditorFromHistory(value: string): void; - /** @param manual Set `true` when value set due to direct user action */ + /** + * @param manual Set `true` when value set due to direct user action + * @param entry Set `true` when `value` is the source of a full entry (e.g. a + * composed Machinery suggestion) that should be parsed and distributed + * across all fields rather than inserted into the focused field. + */ setEditorFromHelpers( value: string, sources: SourceType[], manual: boolean, + entry?: boolean, ): void; setEditorSelection(content: string): void; @@ -167,6 +174,50 @@ export function EditorProvider({ children }: { children: React.ReactElement }) { escapeHTML: htmlElementEscapes(sourceEntry), trim: !hasOuterWhitespace(sourceEntry), }; + + // Parse a full entry source and distribute its leaves across all editor + // fields, falling back to a raw source-view field when it can't be parsed. + // Shared by history restores and composed Machinery copies. + const distributeEntrySource = ( + prev: EditorData, + str: string, + ): EditorData => { + const next = { ...prev }; + if (specialFormats.has(format)) { + const entry = parseEntry(format, str); + if (entry) { + next.base = entry; + } else if (format !== 'fluent') { + return prev; + } + if (entry && !requiresSourceView(entry)) { + next.fields = prev.sourceView + ? editSource(entry) + : editMessageEntry(sourceEntry, entry); + } else { + next.fields = editSource(str); + next.sourceView = true; + } + } else { + next.fields = editMessageEntry(sourceEntry, prev.initial); + next.fields[0].handle.current.setValue(str); + } + // `next.fields` carry placeholder handles, but the on-screen editors stay + // bound (via their React key) to `prev.fields`' live handles. EditField + // only re-syncs when its `defaultValue` string changes, so re-applying a + // value that equals a stale `defaultValue` — e.g. restoring a composed + // suggestion after editing one field — would otherwise leave that field + // untouched until a second click. Push the values into the live handles + // directly, matching by field id, like `clearEditor` does. + for (const field of next.fields) { + const live = prev.fields.find((f) => f.id === field.id); + live?.handle.current.setValue(field.handle.current.value); + } + next.focusField.current = next.fields[0]; + setResult(buildMessageEntry(next.base, next.fields, buildOpts)); + return next; + }; + return { clearEditor() { setState((state) => { @@ -181,8 +232,25 @@ export function EditorProvider({ children }: { children: React.ReactElement }) { setEditorBusy: (busy) => setState((prev) => (busy === prev.busy ? prev : { ...prev, busy })), - setEditorFromHelpers: (str, sources, manual) => + setEditorFromHelpers: (str, sources, manual, entry) => setState((prev) => { + // Composed suggestions carry a full entry source. Outside source view + // we must parse it and distribute the leaves across all fields rather + // than dumping the raw syntax into the focused field. Record the plain + // message as `machinery.translation` so source attribution still + // matches the saved translation (see `useSendTranslation`). + if (entry && !prev.sourceView) { + const next = distributeEntrySource(prev, str); + const parsed = parseEntry(format, str); + return { + ...next, + machinery: { + manual, + translation: parsed ? getPlainMessage(parsed) : str, + sources, + }, + }; + } const { fields, focusField, sourceView } = prev; const field = focusField.current ?? fields[0]; field.handle.current.setValue(str); @@ -200,32 +268,7 @@ export function EditorProvider({ children }: { children: React.ReactElement }) { }), setEditorFromHistory: (str) => - setState((prev) => { - const next = { ...prev }; - if (specialFormats.has(format)) { - const entry = parseEntry(format, str); - if (entry) { - next.base = entry; - } else if (format !== 'fluent') { - return prev; - } - if (entry && !requiresSourceView(entry)) { - next.fields = prev.sourceView - ? editSource(entry) - : editMessageEntry(sourceEntry, entry); - } else { - next.fields = editSource(str); - next.sourceView = true; - } - } else { - next.fields = editMessageEntry(sourceEntry, prev.initial); - next.fields[0].handle.current.setValue(str); - } - next.focusField.current = next.fields[0]; - const result = buildMessageEntry(next.base, next.fields, buildOpts); - setResult(result); - return next; - }), + setState((prev) => distributeEntrySource(prev, str)), setEditorSelection: (content) => setState((state) => { diff --git a/translate/src/context/MachineryTranslations.tsx b/translate/src/context/MachineryTranslations.tsx index 378578cf2e..3d67fe8154 100644 --- a/translate/src/context/MachineryTranslations.tsx +++ b/translate/src/context/MachineryTranslations.tsx @@ -1,8 +1,10 @@ +import { isSelectMessage, type Message } from '@mozilla/l10n'; import React, { createContext, useContext, useEffect, useState } from 'react'; import { abortMachineryRequests, fetchCaighdeanTranslation, + fetchComposedMachinery, fetchGoogleTranslation, fetchMicrosoftTranslation, fetchTranslationMemory, @@ -10,7 +12,11 @@ import { } from '~/api/machinery'; import { USER } from '~/modules/user'; import { useAppSelector } from '~/hooks'; -import { getPlainMessage } from '~/utils/message'; +import { + findPluralSelectors, + getPlainMessage, + specialFormats, +} from '~/utils/message'; import { EntityView, useMachineryEntry } from './EntityView'; import { Locale } from './Locale'; @@ -31,10 +37,66 @@ const initTranslations: MachineryTranslations = { export const MachineryTranslations = createContext(initTranslations); -const sortByQuality = ( - { quality: a }: MachineryTranslation, - { quality: b }: MachineryTranslation, -) => (!a ? 1 : !b ? -1 : a > b ? -1 : a < b ? 1 : 0); +// Composed multi-value suggestions always sort to the top, ahead of the +// per-leaf matches; within each group we sort by descending quality. +const sortByQuality = (a: MachineryTranslation, b: MachineryTranslation) => { + if (a.composed !== b.composed) { + return a.composed ? -1 : 1; + } + const { quality: qa } = a; + const { quality: qb } = b; + return !qa ? 1 : !qb ? -1 : qa > qb ? -1 : qa < qb ? 1 : 0; +}; + +// Number of *target* patterns a message contributes: one per selector variant, +// with plural-selector dimensions expanded to the target locale's CLDR plural +// categories. This mirrors the plural expansion the backend's walk_entity() +// performs, so an en-US `*[other]`-only source still counts as multi-pattern +// for locales like Slovenian (one/two/few/other). +function patternCount( + msg: Message | null | undefined, + pluralCategories: number, +): number { + if (!msg) { + return 0; + } + if (!isSelectMessage(msg)) { + return 1; + } + const plurals = findPluralSelectors(msg); + let count = 1; + for (let i = 0; i < msg.sel.length; ++i) { + if (plurals.has(i)) { + count *= Math.max(1, pluralCategories); + } else { + // Non-plural selector (e.g. a gender): keep its distinct source keys. + const keys = new Set( + msg.alt.map((v) => { + const key = v.keys[i]; + return typeof key === 'string' ? key : '*'; + }), + ); + count *= keys.size || 1; + } + } + return count; +} + +// A composed translation is only meaningful when the entity has more than one +// translatable leaf in the target (Fluent attributes, MF2 selector variants). +// For a single-field target the composed result would just duplicate the +// per-leaf TM or MT match, so we skip the request entirely. +function hasMultipleFields( + value: Message, + properties: Record | undefined, + pluralCategories: number, +): boolean { + let count = patternCount(value, pluralCategories); + for (const prop of Object.values(properties ?? {})) { + count += patternCount(prop, pluralCategories); + } + return count > 1; +} export function MachineryProvider({ children, @@ -44,7 +106,7 @@ export function MachineryProvider({ const locale = useContext(Locale); const { isAuthenticated } = useAppSelector((state) => state[USER]); const { entity } = useContext(EntityView); - const { pk } = entity; + const { pk, format } = entity; const entry = useMachineryEntry(); const { query } = useContext(SearchData); @@ -91,12 +153,32 @@ export function MachineryProvider({ setFetching(true); const promises: Promise[] = []; + // Composed multi-value translations are emitted only for entity-driven + // navigation (not concordance search) and only for formats that can + // have multiple translatable leaves. + const wantsComposed = + !query && + specialFormats.has(format) && + hasMultipleFields( + entity.value, + entity.properties, + locale.cldrPlurals.length, + ); + if (!query) { promises.push( fetchTranslationMemory(plain, locale, pk).then(addResults), ); } + if (wantsComposed) { + promises.push( + fetchComposedMachinery(pk!, locale, 'translation-memory').then( + addResults, + ), + ); + } + // Only make requests to paid services if user is authenticated if (isAuthenticated) { const root = document.getElementById('root'); @@ -108,12 +190,26 @@ export function MachineryProvider({ if (isGoogleTranslateSupported && locale.googleTranslateCode) { promises.push(fetchGoogleTranslation(plain, locale).then(addResults)); + if (wantsComposed) { + promises.push( + fetchComposedMachinery(pk!, locale, 'google-translate').then( + addResults, + ), + ); + } } if (isMicrosoftTranslatorSupported && locale.msTranslatorCode) { promises.push( fetchMicrosoftTranslation(plain, locale).then(addResults), ); + if (wantsComposed) { + promises.push( + fetchComposedMachinery(pk!, locale, 'microsoft-translator').then( + addResults, + ), + ); + } } } diff --git a/translate/src/modules/machinery/components/MachineryTranslation.css b/translate/src/modules/machinery/components/MachineryTranslation.css index b1c1aba54d..49179313f4 100644 --- a/translate/src/modules/machinery/components/MachineryTranslation.css +++ b/translate/src/modules/machinery/components/MachineryTranslation.css @@ -44,6 +44,15 @@ padding-right: 3px; } +.machinery + .translation + > header + .sources:not(.projects) + > li:not(:first-child)::before { + content: '•'; + padding-right: 3px; +} + .machinery .translation > header div { cursor: default; } @@ -147,3 +156,26 @@ .machinery .translation p.suggestion .fa-spin { color: var(--translation-secondary-color); } + +/* Composed multi-value suggestions: rendered as labeled fields, mirroring the + original string panel (.fluent-rich-string base styles are shared). */ +.machinery .translation table.fluent-rich-string { + padding: 0; + margin: 0; + color: var(--translation-color); +} + +.machinery .translation table.fluent-rich-string.original { + color: var(--translation-secondary-color); + margin-bottom: 4px; +} + +.machinery .translation table.fluent-rich-string td { + vertical-align: top; +} + +.machinery .translation table.fluent-rich-string td > span { + display: block; + line-height: 22px; + white-space: pre-wrap; +} diff --git a/translate/src/modules/machinery/components/MachineryTranslation.test.js b/translate/src/modules/machinery/components/MachineryTranslation.test.js index 58e62b8494..93ef2bcf9c 100644 --- a/translate/src/modules/machinery/components/MachineryTranslation.test.js +++ b/translate/src/modules/machinery/components/MachineryTranslation.test.js @@ -1,3 +1,6 @@ +import React from 'react'; + +import { EntityView } from '~/context/EntityView'; import { createDefaultUser, createReduxStore, @@ -65,4 +68,37 @@ describe('', () => { expect(container.querySelector('.quality')).toBeInTheDocument(); expect(container.querySelector('.quality')).toHaveTextContent('100%'); }); + + it('renders a composed multi-field translation as a rich table', () => { + const translation = { + sources: ['translation-memory'], + composed: true, + quality: 100, + original: 'button = Click Me\n .title = Tooltip\n', + translation: 'button = Cliquez\n .title = Infobulle\n', + }; + const store = createReduxStore(); + const Wrapped = (props) => + React.createElement( + EntityView.Provider, + { value: { entity: { format: 'fluent' } } }, + React.createElement(MachineryTranslationComponent, props), + ); + const { container } = mountComponentWithStore(Wrapped, store, { + translation, + }); + createDefaultUser(store); + + // Each leaf (value + attribute) is shown as a labeled row, on both the + // original and the suggestion side. + const original = container.querySelector('.fluent-rich-string.original'); + const suggestion = container.querySelector( + '.fluent-rich-string.suggestion', + ); + expect(original).toBeInTheDocument(); + expect(suggestion).toBeInTheDocument(); + expect(original.querySelectorAll('tr')).toHaveLength(2); + expect(suggestion.textContent).toContain('Cliquez'); + expect(suggestion.textContent).toContain('Infobulle'); + }); }); diff --git a/translate/src/modules/machinery/components/MachineryTranslation.tsx b/translate/src/modules/machinery/components/MachineryTranslation.tsx index 20aed69f57..dc5fe3d9e5 100644 --- a/translate/src/modules/machinery/components/MachineryTranslation.tsx +++ b/translate/src/modules/machinery/components/MachineryTranslation.tsx @@ -4,11 +4,17 @@ import React, { useCallback, useContext, useEffect, useRef } from 'react'; import type { MachineryTranslation, SourceType } from '~/api/machinery'; import { logUXAction } from '~/api/uxaction'; -import { EditorActions } from '~/context/Editor'; +import { EditorActions, EditorField } from '~/context/Editor'; +import { EntityView } from '~/context/EntityView'; import { HelperSelection } from '~/context/HelperSelection'; import { Locale } from '~/context/Locale'; import { GenericTranslation } from '~/modules/translation'; import { useReadonlyEditor } from '~/hooks/useReadonlyEditor'; +import { + editMessageEntry, + parseEntry, + requiresSourceView, +} from '~/utils/message'; import { ConcordanceSearch } from './ConcordanceSearch'; import { MachineryTranslationSource } from './MachineryTranslationSource'; @@ -51,7 +57,10 @@ export function MachineryTranslationComponent({ const sources: SourceType[] = llmTranslation ? ['gpt-transform'] : translation.sources; - setEditorFromHelpers(content, sources, true); + // A composed suggestion is a full entry source (the LLM transform output + // is a plain string, so it isn't), to be spread across all fields. + const isEntry = !llmTranslation && !!translation.composed; + setEditorFromHelpers(content, sources, true, isEntry); if (llmTranslation) { logUXAction('LLM Translation Copied', 'LLM Feature Adoption', { action: 'Copy LLM Translation', @@ -110,10 +119,22 @@ function MachineryTranslationSuggestion({ translation: MachineryTranslation; }) { const { code, direction, script } = useContext(Locale); + const { entity } = useContext(EntityView); const getLLMTranslationState = useLLMTranslation(); const { llmTranslation, loading } = getLLMTranslationState(translation); + // Composed suggestions carry full entry sources (Fluent attributes, MF2 + // variants). Render them as labeled fields — the same representation as the + // original string panel — instead of a single raw block of syntax. + const originalFields = translation.composed + ? richFields(entity.format, translation.original) + : null; + const suggestionFields = translation.composed + ? richFields(entity.format, translation.translation) + : null; + const isRich = originalFields !== null && suggestionFields !== null; + return ( <>
@@ -123,30 +144,104 @@ function MachineryTranslationSuggestion({
-

- -

-

- {loading ? ( - - ) : ( - + + - )} -

+ + ) : ( + <> +

+ +

+

+ {loading ? ( + + ) : ( + + )} +

+ + )} ); } + +/** + * Parse a composed entry source into its editable fields, or return `null` when + * it can't be shown as a rich multi-field view (parse error, source-view-only + * entry, or a single field — in which case the plain rendering is used). + */ +function richFields(format: string, content: string): EditorField[] | null { + const entry = parseEntry(format, content); + if (!entry || requiresSourceView(entry)) { + return null; + } + const fields = editMessageEntry(entry); + return fields.length > 1 ? fields : null; +} + +/** Render a parsed message as a labeled table, mirroring the original string panel. */ +function RichMessage({ + className, + fields, + dir, + lang, + script, +}: { + className: string; + fields: EditorField[]; + dir?: string; + lang?: string; + script?: string; +}): React.ReactElement<'table'> { + return ( + + + {fields.map(({ handle, id, labels }) => ( + + + + + ))} + +
+ + + + + +
+ ); +} diff --git a/translate/src/modules/machinery/components/source/GoogleTranslation.tsx b/translate/src/modules/machinery/components/source/GoogleTranslation.tsx index f3b35018da..02c3b39a36 100644 --- a/translate/src/modules/machinery/components/source/GoogleTranslation.tsx +++ b/translate/src/modules/machinery/components/source/GoogleTranslation.tsx @@ -67,7 +67,11 @@ export function GoogleTranslation({ ); - return isOpenAIChatGPTSupported ? ( + // AI refinement isn't supported for composed (multi-field/plural) + // suggestions: the backend refines a single string, so it can't preserve the + // entry structure, and the rich rendering can't show the refined result. Show + // the plain source label instead. See follow-up to add composed support. + return isOpenAIChatGPTSupported && !translation.composed ? (