Update sync behaviour

Author: eemeli

pontoon/sync/core/__init__.py

@@ -11,6 +11,7 @@
     Entity,
     Locale,
     Project,
+    ProjectLocale,
     TranslatedResource,
     Translation,
     User,
@@ -50,10 +51,30 @@ def sync_project(
         log.error(f"{log_prefix} {e}")
         raise e
 
-    locale_map: dict[str, Locale] = {
-        lc.code: lc for lc in project.locales.order_by("code")
-    }
+    locale_map: dict[str, Locale]
+    if project.set_locales_from_repo:
+        if not paths.locales:
+            errmsg = "No locales found in repo"
+            log.error(f"{log_prefix} {errmsg}")
+            raise Exception(errmsg)
+        locales = Locale.objects.filter(code__in=paths.locales).order_by("code")
+        locale_map = {lc.code: lc for lc in locales}
+
+        # Update project locales
+        ProjectLocale.objects.filter(project=project).exclude(
+            locale__in=locales
+        ).delete()
+        for locale in locales:
+            # The implicit pre_save and post_save signals sent here are required
+            # to maintain django-guardian permissions.
+            ProjectLocale.objects.get_or_create(project=project, locale=locale)
+        ProjectLocale.objects.filter(project=project, readonly=True).update(
+            readonly=False
+        )
+    else:
+        locale_map = {lc.code: lc for lc in project.locales.order_by("code")}
     paths.locales = list(locale_map.keys())
+
     added_entities_count, changed_paths, removed_paths = sync_resources_from_repo(
         project, locale_map, checkouts.source, paths, now
     )

pontoon/sync/core/entities.py

@@ -375,7 +375,7 @@ def update_translated_resources(
         _, locales = paths.target(resource.path)
         for lc in locales:
             locale = locale_map.get(lc, None)
-            if is_translated_resource(paths, resource, locale):
+            if is_translated_resource(project, paths, resource, locale):
                 assert locale is not None
                 key = (resource.pk, locale.pk)
                 if key in prev_tr_keys:
@@ -403,16 +403,21 @@ def update_translated_resources(
 
 
 def is_translated_resource(
+    project: Project,
     paths: L10nConfigPaths | L10nDiscoverPaths,
     resource: Resource,
     locale: Locale | None,
 ) -> bool:
     if locale is None:
         return False
 
-    if resource.format == Resource.Format.GETTEXT:
-        # For gettext, only create TranslatedResource
-        # if the resource exists for the locale.
+    if (
+        isinstance(paths, L10nDiscoverPaths)
+        and project.set_locales_from_repo
+        and project.set_translated_resources_from_repo
+    ):
+        # With `set_translated_resources_from_repo`,
+        # only create TranslatedResource if the resource exists for the locale.
         target, _ = paths.target(resource.path)
         if target is None:
             return False

pontoon/sync/core/translations_from_repo.py

@@ -53,7 +53,15 @@ def sync_translations_from_repo(
     """(removed_resource_count, updated_translation_count)"""
     co = checkouts.target
     source_paths: set[str] = set(paths.ref_paths) if checkouts.source == co else set()
-    del_count = delete_removed_gettext_resources(project, co, paths, source_paths)
+    del_count = (
+        delete_removed_resources(project, co, paths, source_paths)
+        if (
+            isinstance(paths, L10nDiscoverPaths)
+            and project.set_locales_from_repo
+            and project.set_translated_resources_from_repo
+        )
+        else 0
+    )
 
     changed_target_paths = [
         path
@@ -83,10 +91,10 @@ def write_db_updates(
     add_translation_memory_entries(project, new_translations + updated_translations)
 
 
-def delete_removed_gettext_resources(
+def delete_removed_resources(
     project: Project,
     target: Checkout,
-    paths: L10nConfigPaths | L10nDiscoverPaths,
+    paths: L10nDiscoverPaths,
     source_paths: set[str],
 ) -> int:
     rm_t = Q()
@@ -95,7 +103,7 @@ def delete_removed_gettext_resources(
     removed_target_paths = (
         path
         for path in (join(target.path, co_path) for co_path in target.removed)
-        if path not in source_paths and splitext(path)[1] in {".po", ".pot"}
+        if path not in source_paths
     )
     for target_path in removed_target_paths:
         ref = paths.find_reference(target_path)
@@ -104,7 +112,7 @@ def delete_removed_gettext_resources(
             locale_code = get_path_locale(path_vars)
             if locale_code is not None:
                 db_path = relpath(ref_path, paths.ref_root)
-                if not project.configuration_file and db_path.endswith(".pot"):
+                if db_path.endswith(".pot"):
                     db_path = db_path[:-1]
                 rm_t |= Q(entity__resource__path=db_path, locale__code=locale_code)
                 rm_tr |= Q(resource__path=db_path, locale__code=locale_code)

pontoon/sync/tests/test_e2e.py

@@ -17,6 +17,7 @@
     ChangedEntityLocale,
     Entity,
     Locale,
+    ProjectLocale,
     Repository,
     TranslatedResource,
     Translation,
@@ -242,7 +243,11 @@ def test_add_resources():
         assert {
             (tr.resource.path, tr.locale.code)
             for tr in TranslatedResource.objects.filter(resource__project=project)
-        } == {("file.ftl", "de-Test"), ("file.xliff", "de-Test")}
+        } == {
+            ("file.ftl", "de-Test"),
+            ("file.po", "de-Test"),
+            ("file.xliff", "de-Test"),
+        }
 
         # Add an XLIFF translation
         TranslationFactory.create(
@@ -848,3 +853,82 @@ def test_add_project_locale():
             tr.locale.code: tr.total_strings
             for tr in TranslatedResource.objects.filter(resource__project=project)
         } == {"fr-Test": 1, "de-Test": 1}
+
+
+@pytest.mark.django_db
+def test_locales_from_repo():
+    mock_vcs = MockVersionControl(changed=[])
+    with mock_setup(mock_vcs) as (repo, _):
+        LocaleFactory.create(code="fr-Test", name="Test French")
+
+        # Database setup
+        project = ProjectFactory.create(
+            name="test-repo-locales",
+            locales=[],
+            repositories=[repo],
+            set_locales_from_repo=True,
+            system_project=False,
+        )
+
+        # Filesystem setup
+        makedirs(repo.checkout_path)
+        build_file_tree(
+            repo.checkout_path,
+            {
+                "en-US": {"messages.json": '{ "key": { "message": "Entity" } }'},
+                "de-Test": {"messages.json": "{}"},
+                "fr-Test": {"messages.json": "{}"},
+            },
+        )
+
+        sync_project_task(project.pk)
+
+        assert {
+            pl.locale.code for pl in ProjectLocale.objects.filter(project=project)
+        } == {"de-Test", "fr-Test"}
+        assert {
+            tr.locale.code: tr.total_strings
+            for tr in TranslatedResource.objects.filter(resource__project=project)
+        } == {"fr-Test": 1, "de-Test": 1}
+
+
+@pytest.mark.django_db
+def test_locales_from_config():
+    mock_vcs = MockVersionControl(changed=[])
+    with mock_setup(mock_vcs) as (repo, _):
+        LocaleFactory.create(code="fr-Test", name="Test French")
+
+        # Database setup
+        project = ProjectFactory.create(
+            name="test-config-locales",
+            configuration_file="l10n.toml",
+            locales=[],
+            repositories=[repo],
+            set_locales_from_repo=True,
+            system_project=False,
+        )
+
+        # Filesystem setup
+        makedirs(repo.checkout_path)
+        build_file_tree(
+            repo.checkout_path,
+            {
+                "foo": {"en": {"messages.json": '{ "key": { "message": "Entity" } }'}},
+                "l10n.toml": dedent("""\
+                    locales = ["de-Test", "fr-Test"]
+                    [[paths]]
+                        reference = "foo/en/**"
+                        l10n = "foo/{locale}/**"
+                    """),
+            },
+        )
+
+        sync_project_task(project.pk)
+
+        assert {
+            pl.locale.code for pl in ProjectLocale.objects.filter(project=project)
+        } == {"de-Test", "fr-Test"}
+        assert {
+            tr.locale.code: tr.total_strings
+            for tr in TranslatedResource.objects.filter(resource__project=project)
+        } == {"fr-Test": 1, "de-Test": 1}

pontoon/sync/tests/test_translations_from_repo.py

@@ -358,6 +358,8 @@ def test_remove_po_target_resource():
             name="test-rm-po",
             locales=[locale],
             repositories=[repo],
+            set_locales_from_repo=True,
+            set_translated_resources_from_repo=True,
             visibility="public",
         )
         res = {}