Merge pull request #259 from sbonaime/publish

Remove deprecated options and flags  from doc and translated files

Author: smathermather

.tx/config

@@ -73,14 +73,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_debug]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/debug.po
-source_file            = source/locale/pot/arguments/debug.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_dem-decimation]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/dem-decimation.po
 source_file            = source/locale/pot/arguments/dem-decimation.pot
@@ -113,14 +105,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_depthmap-resolution]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/depthmap-resolution.po
-source_file            = source/locale/pot/arguments/depthmap-resolution.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_dsm]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/dsm.po
 source_file            = source/locale/pot/arguments/dsm.pot
@@ -217,14 +201,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_matcher-distance]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/matcher-distance.po
-source_file            = source/locale/pot/arguments/matcher-distance.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_matcher-neighbors]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/matcher-neighbors.po
 source_file            = source/locale/pot/arguments/matcher-neighbors.pot
@@ -377,14 +353,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_pc-geometric]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/pc-geometric.po
-source_file            = source/locale/pot/arguments/pc-geometric.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_pc-las]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/pc-las.po
 source_file            = source/locale/pot/arguments/pc-las.pot
@@ -417,14 +385,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_pc-tile]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/pc-tile.po
-source_file            = source/locale/pot/arguments/pc-tile.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_primary-band]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/primary-band.po
 source_file            = source/locale/pot/arguments/primary-band.pot
@@ -473,14 +433,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_resize-to]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/resize-to.po
-source_file            = source/locale/pot/arguments/resize-to.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_skip-3dmodel]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/skip-3dmodel.po
 source_file            = source/locale/pot/arguments/skip-3dmodel.pot
@@ -569,14 +521,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_texturing-data-term]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/texturing-data-term.po
-source_file            = source/locale/pot/arguments/texturing-data-term.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_texturing-keep-unseen-faces]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/texturing-keep-unseen-faces.po
 source_file            = source/locale/pot/arguments/texturing-keep-unseen-faces.pot
@@ -585,14 +529,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_texturing-outlier-removal-type]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/texturing-outlier-removal-type.po
-source_file            = source/locale/pot/arguments/texturing-outlier-removal-type.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_texturing-skip-global-seam-leveling]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/texturing-skip-global-seam-leveling.po
 source_file            = source/locale/pot/arguments/texturing-skip-global-seam-leveling.pot
@@ -601,22 +537,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_texturing-skip-local-seam-leveling]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/texturing-skip-local-seam-leveling.po
-source_file            = source/locale/pot/arguments/texturing-skip-local-seam-leveling.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
-[o:americanredcross:p:opendronemap_docs:r:arguments_texturing-tone-mapping]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/texturing-tone-mapping.po
-source_file            = source/locale/pot/arguments/texturing-tone-mapping.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_tiles]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/tiles.po
 source_file            = source/locale/pot/arguments/tiles.pot
@@ -625,14 +545,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_time]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/time.po
-source_file            = source/locale/pot/arguments/time.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_use-3dmesh]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/use-3dmesh.po
 source_file            = source/locale/pot/arguments/use-3dmesh.pot
@@ -665,14 +577,6 @@ type                   = PO
 replace_edited_strings = false
 keep_translations      = false
 
-[o:americanredcross:p:opendronemap_docs:r:arguments_verbose]
-file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/verbose.po
-source_file            = source/locale/pot/arguments/verbose.pot
-source_lang            = en
-type                   = PO
-replace_edited_strings = false
-keep_translations      = false
-
 [o:americanredcross:p:opendronemap_docs:r:arguments_version]
 file_filter            = source/locale/<lang>/LC_MESSAGES/arguments/version.po
 source_file            = source/locale/pot/arguments/version.pot

Makefile

@@ -15,7 +15,7 @@ help:
 .PHONY: help Makefile
 
 # this is for testing, it will only build the EN version of the docs
-livehtml:
+livehtml: autogenerate
 	sphinx-autobuild --open-browser --host 0.0.0.0 -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)/html"
 
 # this will update `arguments.rst` to match arguments in the ODM code
@@ -24,22 +24,38 @@ livehtml:
 # all user contributed content from files in `source/arguments_edit/`
 autogenerate:
 	python scripts/extract_odm_strings.py https://raw.githubusercontent.com/OpenDroneMap/ODM/master/opendm/config.py
+	@$(SPHINXBUILD) -b gettext "$(SOURCEDIR)" "source/locale/pot"
+	sphinx-intl update --pot-dir "source/locale/pot" --language ar,cs,es,fil,fr,id,sw,te,zh
+	@$(MAKE) cleanobsolete
 
 # update main EN pot files that serve as the source for translation process
 updatepot:
 	@$(SPHINXBUILD) -b gettext "$(SOURCEDIR)" "source/locale/pot"
 
 # update the po files for each target language from the EN pot files
 updatelangpo:
-	sphinx-intl update --pot-dir "source/locale/pot" --language cs,es,fil,fr,id,sw,te,zh
+	sphinx-intl update --pot-dir "source/locale/pot" --language ar,cs,es,fil,fr,id,sw,te,zh
+	@$(MAKE) cleanobsolete
+
+# remove obsolete entries from all .po files
+cleanobsolete:
+	@echo "Removing obsolete entries from .po files..."
+	@for lang in ar cs es fil fr id sw te zh; do \
+		for po in source/locale/$$lang/LC_MESSAGES/*.po source/locale/$$lang/LC_MESSAGES/*/*.po; do \
+			if [ -f "$$po" ]; then \
+				msgattrib --no-obsolete --no-fuzzy -o "$$po.tmp" "$$po" 2>/dev/null && mv "$$po.tmp" "$$po" || rm -f "$$po.tmp"; \
+			fi; \
+		done; \
+	done
+	@echo "Done cleaning obsolete entries."
 
 # push new and changed strings to Transifex
 pushlang:
 	tx push --source
 
 # push translated strings from Transifex
 pulllang:
-	tx pull --language "cs,es,fil,fr,id,sw,te,zh"
+	tx pull --language "ar,cs,es,fil,fr,id,sw,te,zh"
 
 build:
 	@$(SPHINXBUILD) -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)/html" -nW

README.md

@@ -88,8 +88,7 @@ From the same Terminal (or command prompt) run the following:
 
 ```
 cd docs/
-python3 -m venv .venv
-source .venv/bin/activate
+python3.12 -m venv .venv
 
 # Linux/Mac
 source .venv/bin/activate

requirements.txt

@@ -4,3 +4,4 @@ sphinx-intl==2.0.1
 sphinx-rtd-theme==1.1.1
 transifex-client==0.12.5
 sphinxcontrib-mermaid==0.9.2
+sphinx-copybutton==0.3.1

requirements_prod.txt

@@ -3,4 +3,5 @@ sphinx==7.4.7
 sphinx-intl
 sphinx-rtd-theme
 sphinxcontrib-mermaid
+sphinx-copybutton
 wheel

scripts/extract_odm_strings.py

@@ -1,6 +1,11 @@
 #!/usr/bin/python3
 
-import argparse, os, urllib.request, ast, sys
+import argparse
+import os
+import urllib.request
+import ast
+import sys
+import glob
 from io import StringIO
 from pathlib import Path
 from string import Template
@@ -16,6 +21,12 @@
 tmplfile = os.path.join(os.path.dirname(__file__), "arguments.template.rst")
 argstmplfile = os.path.join(os.path.dirname(__file__), "arguments.arg.template.rst")
 argsoutdir = os.path.join(os.path.dirname(__file__), "..", "source", "arguments")
+argspotdir = os.path.join(os.path.dirname(__file__), "..", "source", "locale", "pot", "arguments")
+localedir = os.path.join(os.path.dirname(__file__), "..", "source", "locale")
+txconfigfile = os.path.join(os.path.dirname(__file__), "..", ".tx", "config")
+
+# List of supported languages (should match Makefile)
+LANGUAGES = ['ar', 'cs', 'es', 'fil', 'fr', 'id', 'sw', 'te', 'zh']
 
 strings = []
 print("Fetching %s ..." % url)
@@ -91,6 +102,37 @@ def ODM_ERROR(s):
     keys = list(options.keys())
     keys.sort(key=lambda a: a.replace("-", ""))
 
+    # Clean up the output directory - remove all existing .rst files
+    print("Cleaning up %s ..." % argsoutdir)
+    for old_file in glob.glob(os.path.join(argsoutdir, "*.rst")):
+        try:
+            os.remove(old_file)
+            print("Removed %s" % old_file)
+        except (OSError, PermissionError) as e:
+            print("Warning: Could not remove %s: %s" % (old_file, e))
+
+    # Clean up old .pot files for deprecated arguments
+    if os.path.isdir(argspotdir):
+        print("Cleaning up %s ..." % argspotdir)
+        for old_pot in glob.glob(os.path.join(argspotdir, "*.pot")):
+            try:
+                os.remove(old_pot)
+                print("Removed %s" % old_pot)
+            except (OSError, PermissionError) as e:
+                print("Warning: Could not remove %s: %s" % (old_pot, e))
+
+    # Clean up old .po files for deprecated arguments in all languages
+    for lang in LANGUAGES:
+        argspodir = os.path.join(localedir, lang, "LC_MESSAGES", "arguments")
+        if os.path.isdir(argspodir):
+            print("Cleaning up %s ..." % argspodir)
+            for old_po in glob.glob(os.path.join(argspodir, "*.po")):
+                try:
+                    os.remove(old_po)
+                    print("Removed %s" % old_po)
+                except (OSError, PermissionError) as e:
+                    print("Warning: Could not remove %s: %s" % (old_po, e))
+
     with open(argstmplfile) as f:
         argstmpl = Template(f.read())
 
@@ -150,6 +192,63 @@ def get_opt_choices(opt):
 
     print("Wrote %s" % outfile)
 
+    # Clean up orphaned files in arguments_edit/ directory
+    arguments_edit_dir = os.path.join(argsoutdir + "_edit")
+    if os.path.isdir(arguments_edit_dir):
+        print("Cleaning up orphaned files in %s ..." % arguments_edit_dir)
+        current_opt_names = set(get_opt_name(opt) for opt in keys)
+        for edit_file in glob.glob(os.path.join(arguments_edit_dir, "*.rst")):
+            edit_opt_name = os.path.splitext(os.path.basename(edit_file))[0]
+            if edit_opt_name not in current_opt_names:
+                try:
+                    os.remove(edit_file)
+                    print("Removed orphaned %s" % edit_file)
+                except (OSError, PermissionError) as e:
+                    print("Warning: Could not remove %s: %s" % (edit_file, e))
+
+    # Update .tx/config to remove deprecated options
+    if os.path.isfile(txconfigfile):
+        print("Updating %s ..." % txconfigfile)
+        with open(txconfigfile, 'r') as f:
+            config_lines = f.readlines()
+
+        # Get list of current options
+        current_opts = set(get_opt_name(opt) for opt in keys)
+
+        # Parse and filter .tx/config
+        new_config_lines = []
+        skip_section = False
+        i = 0
+        while i < len(config_lines):
+            line = config_lines[i]
+
+            # Check if this is an arguments section header
+            if line.startswith('[o:americanredcross:p:opendronemap_docs:r:arguments_'):
+                # Extract option name
+                section_name = line.strip()[1:-1]  # Remove [ and ]
+                opt_name = section_name.split(':')[-1].replace('arguments_', '')
+
+                # Check if this option still exists
+                if opt_name not in current_opts:
+                    print("Removing deprecated section: %s" % opt_name)
+                    skip_section = True
+                    # Skip this section header and all lines until next section or EOF
+                    i += 1
+                    while i < len(config_lines) and not config_lines[i].startswith('['):
+                        i += 1
+                    continue
+                else:
+                    skip_section = False
+
+            if not skip_section:
+                new_config_lines.append(line)
+
+            i += 1
+
+        # Write updated config
+        with open(txconfigfile, 'w') as f:
+            f.writelines(new_config_lines)
+        print("Updated %s" % txconfigfile)
 
 else:
     print("No strings found")