Skip to content

Commit 69403c1

Browse files
imnasnainaecimnasnainaec
authored
Purge sem dom examples (#4302)
1 parent ccd81c6 commit 69403c1

17 files changed

Lines changed: 174 additions & 398829 deletions

‎.github/workflows/localization_check.yml‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
# Verify that the language lists in the README localization badges agree with
2+
# the languages available for the user interface, semantic domains, and user guide.
3+
# Also verify that the semantic domain xml files contain no <Example*> elements.
4+
name: localization-check
5+
6+
on:
7+
push:
8+
branches: [master]
9+
paths:
10+
- .github/workflows/localization_check.yml
11+
- README.md
12+
- src/types/writingSystem.ts
13+
- deploy/scripts/semantic_domains/xml/**
14+
- docs/user_guide/mkdocs.yml
15+
pull_request:
16+
branches: [master]
17+
paths:
18+
- .github/workflows/localization_check.yml
19+
- README.md
20+
- src/types/writingSystem.ts
21+
- deploy/scripts/semantic_domains/xml/**
22+
- docs/user_guide/mkdocs.yml
23+
24+
concurrency:
25+
cancel-in-progress: true
26+
group: ${{ github.workflow }}-${{ github.ref }}
27+
28+
permissions: # added using https://github.com/step-security/secure-workflows
29+
contents: read
30+
31+
jobs:
32+
localization-check:
33+
runs-on: ubuntu-latest
34+
steps:
35+
# See https://docs.stepsecurity.io/harden-runner/getting-started/ for instructions on
36+
# configuring harden-runner and identifying allowed endpoints.
37+
- name: Harden Runner
38+
uses: step-security/harden-runner@fe104658747b27e96e4f7e80cd0a94068e53901d # v2.16.1
39+
with:
40+
disable-sudo: true
41+
egress-policy: block
42+
allowed-endpoints: >
43+
github.com:443
44+
- name: Checkout repository
45+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
46+
with:
47+
persist-credentials: false
48+
- name: Verify localization language lists agree
49+
run: |
50+
fail=0
51+
ts=src/types/writingSystem.ts
52+
53+
# Sorted bcp47 codes of a writing-system list in writingSystem.ts.
54+
ts_langs() {
55+
sed -n "/const $1 = \[/,/\];/p" "$ts" |
56+
grep -oP 'writingSystem\[Bcp47Code\.\K\w+' |
57+
while read -r key; do
58+
grep -oP "^\s+$key = \"\K[^\"]+" "$ts" || echo "$key-not-in-enum"
59+
done | sort -u
60+
}
61+
62+
# Sorted, lowercased language codes of a localization badge url in README.md.
63+
badge_langs() {
64+
grep -oP "img\.shields\.io/badge/$1-\K[^-]+" README.md |
65+
sed 's/%20/\n/g' | tr '[:upper:]' '[:lower:]' | sort -u
66+
}
67+
68+
# compare <label-a> <langs-a> <label-b> <langs-b>
69+
compare() {
70+
if [ "$2" != "$4" ]; then
71+
echo "::error::Language lists differ between $1 and $3:"
72+
diff -u --label "$1" --label "$3" <(echo "$2") <(echo "$4") || true
73+
fail=1
74+
fi
75+
}
76+
77+
# show <label> <langs>
78+
show() {
79+
echo "$1 ($(echo "$2" | wc -w)): $(echo "$2" | tr '\n' ' ')"
80+
}
81+
82+
# Semantic domains: xml files <-> semDomWritingSystems <-> sd badge.
83+
# English is implicit: its base data is in deploy/scripts/semantic_domains/json/,
84+
# so there is no SemanticDomains-en.xml.
85+
echo "::group::Semantic domains"
86+
xml_langs=$({ echo en; ls deploy/scripts/semantic_domains/xml/ |
87+
grep -oP '^SemanticDomains-\K[^.]+'; } | sort -u)
88+
sd_ts=$(ts_langs semDomWritingSystems)
89+
sd_badge=$(badge_langs 'Semantic%20Domains')
90+
show "semantic_domains/xml/ files (with en implicit)" "$xml_langs"
91+
show "semDomWritingSystems" "$sd_ts"
92+
show "localization-sd-badge" "$sd_badge"
93+
echo "::endgroup::"
94+
compare "semantic_domains/xml/ files (with en implicit)" "$xml_langs" \
95+
"semDomWritingSystems" "$sd_ts"
96+
compare "semDomWritingSystems" "$sd_ts" "localization-sd-badge" "$sd_badge"
97+
98+
# User interface: uiWritingSystems <-> ui badge <-> public/locales/ subdirectories.
99+
echo "::group::User interface"
100+
ui_ts=$(ts_langs uiWritingSystems)
101+
ui_badge=$(badge_langs 'User%20Interface')
102+
locales=$(find public/locales -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort -u)
103+
show "uiWritingSystems" "$ui_ts"
104+
show "localization-ui-badge" "$ui_badge"
105+
show "public/locales/ subdirectories" "$locales"
106+
echo "::endgroup::"
107+
compare "uiWritingSystems" "$ui_ts" "localization-ui-badge" "$ui_badge"
108+
compare "localization-ui-badge" "$ui_badge" "public/locales/ subdirectories" "$locales"
109+
110+
# User guide: mkdocs.yml locales <-> ug badge.
111+
echo "::group::User guide"
112+
ug_mkdocs=$(grep -oP '^\s*- locale: \K\S+' docs/user_guide/mkdocs.yml | sort -u)
113+
ug_badge=$(badge_langs 'User%20Guide')
114+
show "mkdocs.yml locales" "$ug_mkdocs"
115+
show "localization-ug-badge" "$ug_badge"
116+
echo "::endgroup::"
117+
compare "mkdocs.yml locales" "$ug_mkdocs" "localization-ug-badge" "$ug_badge"
118+
119+
exit $fail
120+
- name: Verify semantic domain xml files have no examples
121+
run: |
122+
if grep -rln '<Example' deploy/scripts/semantic_domains/xml/; then
123+
echo "::error::Found <Example> elements in the semantic domain xml files listed above."
124+
exit 1
125+
fi

‎README.md‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ python -m piptools compile --upgrade requirements.in
269269
### Load Semantic Domains
270270

271271
Data Entry will not work in The Combine unless the semantic domains have been loaded into the database. Follow the
272-
instuctions in [Import Semantic Domains](#import-semantic-domains) below to import the domains from at least one of the
272+
instructions in [Import Semantic Domains](#import-semantic-domains) below to import the domains from at least one of the
273273
semantic domains XML files (which each contain domain data in English and one other language.)
274274

275275
## Available Scripts
@@ -420,7 +420,14 @@ Auto-format frontend code in the `src` folder.
420420

421421
### Import Semantic Domains
422422

423-
To import Semantic Domains from the XML files in `./deploy/scripts/semantic_domains/xml`. Run from within a Python
423+
The raw semantic domain XML files are in `./deploy/scripts/semantic_domains/xml/`. If you update them, remove the
424+
`<Example*>` blocks before committing to the repo:
425+
426+
```bash
427+
python scripts/remove_sem_dom_examples.py
428+
```
429+
430+
Use the following steps to import semantic domains from the XML files into the database. Run from within a Python
424431
virtual environment.
425432

426433
1. Generate the files for import into the Mongo database:

‎deploy/scripts/sem_dom_import.py‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,7 @@ def get_questions(node: ElementTree.Element) -> Dict[str, List[DomainQuestion]]:
106106
"""
107107
Gets a list of DomainQuestion objects from the CmDomainQ element.
108108
109-
The DomainQuestion consists of the question text, the example words and
110-
the example sentences.
109+
Extracts the question text; ignores example words and sentences.
111110
"""
112111
results: Dict[str, List[DomainQuestion]] = {}
113112
for cm_domain_q in node:
@@ -117,16 +116,7 @@ def get_questions(node: ElementTree.Element) -> Dict[str, List[DomainQuestion]]:
117116
if quest_field.tag == "Question":
118117
for auni_node in quest_field:
119118
lang, question = get_auni_text(auni_node)
120-
this_question[lang] = DomainQuestion(question, "", "")
121-
elif quest_field.tag == "ExampleWords":
122-
for auni_node in quest_field:
123-
lang, words = get_auni_text(auni_node)
124-
this_question[lang].example_words = words
125-
elif quest_field.tag == "ExampleSentences":
126-
for astr_node in quest_field:
127-
lang, sentences = get_astr_text(astr_node)
128-
if len(sentences) > 0:
129-
this_question[lang].example_sentences = "\n".join(sentences)
119+
this_question[lang] = DomainQuestion(question)
130120
for lang in this_question:
131121
if lang in results:
132122
results[lang].append(this_question[lang])

‎deploy/scripts/semantic_domains.py‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@
99
@dataclass
1010
class DomainQuestion:
1111
question: str
12-
example_words: str
13-
example_sentences: str
1412

1513

1614
class SemanticDomain:
@@ -59,13 +57,7 @@ def to_json(self) -> str:
5957
if SemanticDomainFull.flatten_questions:
6058
flat_question_list.append(item.question)
6159
else:
62-
full_question_list.append(
63-
{
64-
"question": item.question,
65-
"example_words": item.example_words,
66-
"example_sentences": item.example_sentences,
67-
}
68-
)
60+
full_question_list.append({"question": item.question})
6961
data = {
7062
"guid": "" if self.guid is None else str(self.guid),
7163
"lang": self.lang,

0 commit comments

Comments
 (0)