Skip to content

Commit 9fa05aa

Browse files
tomasciccolaTomás CiccolaEvanHahn
authored
feat: Translation Api (#515)
* initial implementation of `TranslationApi` * * add @types/json-stable-strinfify * start writing unit tests * error handling for missing translation * fix error in sql query * * remove `message` from object when hashing * fix logic error when `put` * handle unexisting key in the indexing (by creating a new Set) * make `fieldRef` and `regionCode` optional when `get` * improve `get()` by ignoring not translated languages, improve typing * * make `index` private, run it on `put` * expose cache map through symbol and getter * improve `get` signature (by making `fieldRef` and `regionCode` optional) * add more unit tests * revert `index` as private method, update tests * update magic-bytes manually * add e2e/translation-api.js and expose translation api in mapeo project * * add translationTable to index writer * add translationDoc to entries for batching indexer * move decoding of translationDoc into `.index` function in translationApi * rever changes to `.index` method (better to accept doc than block) * first e2e tests * revert regionCode fallback (since its handled in an upper layer) * use default config for translationApi e2e tests, test with a bunch of translations * add translation fixtures * add check of expected translation * improve test messages * add assertion of matching preset docId with translation docIdRef * add tests and fixture for fields * chore: use private members for TranslationApi (#579) * chore: ensure translation tests are checking something (#578) These tests iterate over various documents to check things. If, somehow, we had 0 documents, the test would pass incorrectly. This fixes that by checking that we have at least one test doc. * Apply suggestions from code review 1. replace conditional with assertion 2. remove comment in `get` signature 3. remove `?` from `translatedSchemas.add` 4. Add comment to advice only using private symbol in tests Co-authored-by: Evan Hahn <me@evanhahn.com> * Fix `assert` not being present * perf test, return full doc from create * fix tests after api change * comment perf test and re indexing of translations on translationAPI constructor * added cache tests * simplify `put` logic * doc can be a const inside try now * add return type to `get` Co-authored-by: Evan Hahn <me@evanhahn.com> * chore: add "ready" promise to translation API (#589) * chore: use `instanceof`, not message, for "not found" error (#590) --------- Co-authored-by: Tomás Ciccola <tciccola@digital-democracy.com> Co-authored-by: Evan Hahn <me@evanhahn.com>
1 parent 782cfe5 commit 9fa05aa

11 files changed

Lines changed: 779 additions & 207 deletions

File tree

package-lock.json

Lines changed: 115 additions & 205 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
"@types/bogon": "^1.0.2",
8686
"@types/debug": "^4.1.8",
8787
"@types/json-schema": "^7.0.11",
88+
"@types/json-stable-stringify": "^1.0.36",
8889
"@types/nanobench": "^3.0.0",
8990
"@types/node": "^18.16.3",
9091
"@types/sinonjs__fake-timers": "^8.1.2",
@@ -144,6 +145,7 @@
144145
"hypercore-crypto": "3.4.0",
145146
"hyperdrive": "11.5.3",
146147
"hyperswarm": "4.4.1",
148+
"json-stable-stringify": "^1.1.1",
147149
"magic-bytes.js": "^1.10.0",
148150
"map-obj": "^5.0.2",
149151
"mime": "^4.0.1",

src/datatype/index.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { getTableConfig } from 'drizzle-orm/sqlite-core'
44
import { eq, inArray, sql } from 'drizzle-orm'
55
import { randomBytes } from 'node:crypto'
66
import { noop, deNullify } from '../utils.js'
7+
import { NotFoundError } from '../errors.js'
78
import crypto from 'hypercore-crypto'
89
import { TypedEmitter } from 'tiny-typed-emitter'
910

@@ -175,7 +176,7 @@ export class DataType extends TypedEmitter {
175176
async getByDocId(docId) {
176177
await this.#dataStore.indexer.idle()
177178
const result = this.#sql.getByDocId.get({ docId })
178-
if (!result) throw new Error('Not found')
179+
if (!result) throw new NotFoundError()
179180
return deNullify(result)
180181
}
181182

src/errors.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// @ts-check
2+
3+
export class NotFoundError extends Error {
4+
constructor() {
5+
super('Not found')
6+
}
7+
}

src/mapeo-project.js

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// @ts-check
22
import path from 'path'
33
import Database from 'better-sqlite3'
4-
import { decodeBlockPrefix } from '@mapeo/schema'
4+
import { decodeBlockPrefix, decode } from '@mapeo/schema'
55
import { drizzle } from 'drizzle-orm/better-sqlite3'
66
import { migrate } from 'drizzle-orm/better-sqlite3/migrator'
77
import { discoveryKey } from 'hypercore-crypto'
@@ -24,6 +24,7 @@ import {
2424
presetTable,
2525
roleTable,
2626
iconTable,
27+
translationTable,
2728
} from './schema/project.js'
2829
import {
2930
CoreOwnership,
@@ -37,6 +38,7 @@ import {
3738
LEFT_ROLE_ID,
3839
} from './roles.js'
3940
import {
41+
assert,
4042
getDeviceId,
4143
projectKeyToId,
4244
projectKeyToPublicId,
@@ -47,6 +49,7 @@ import { SyncApi, kHandleDiscoveryKey } from './sync/sync-api.js'
4749
import { Logger } from './logger.js'
4850
import { IconApi } from './icon-api.js'
4951
import { readConfig } from './config-import.js'
52+
import TranslationApi from './translation-api.js'
5053

5154
/** @typedef {Omit<import('@mapeo/schema').ProjectSettingsValue, 'schemaName'>} EditableProjectSettings */
5255

@@ -80,6 +83,7 @@ export class MapeoProject extends TypedEmitter {
8083
#memberApi
8184
#iconApi
8285
#syncApi
86+
#translationApi
8387
#l
8488

8589
static EMPTY_PROJECT_SETTINGS = EMPTY_PROJECT_SETTINGS
@@ -157,6 +161,7 @@ export class MapeoProject extends TypedEmitter {
157161
roleTable,
158162
deviceInfoTable,
159163
iconTable,
164+
translationTable,
160165
],
161166
sqlite: this.#sqlite,
162167
getWinner,
@@ -242,6 +247,11 @@ export class MapeoProject extends TypedEmitter {
242247
table: iconTable,
243248
db,
244249
}),
250+
translation: new DataType({
251+
dataStore: this.#dataStores.config,
252+
table: translationTable,
253+
db,
254+
}),
245255
}
246256
const identityKeypair = keyManager.getIdentityKeypair()
247257
const coreKeypairs = getCoreKeypairs({
@@ -310,6 +320,11 @@ export class MapeoProject extends TypedEmitter {
310320
logger: this.#l,
311321
})
312322

323+
this.#translationApi = new TranslationApi({
324+
dataType: this.#dataTypes.translation,
325+
table: translationTable,
326+
})
327+
313328
///////// 4. Replicate local peers automatically
314329

315330
// Replicate already connected local peers
@@ -420,6 +435,15 @@ export class MapeoProject extends TypedEmitter {
420435

421436
if (schemaName === 'projectSettings') {
422437
projectSettingsEntries.push(entry)
438+
} else if (schemaName === 'translation') {
439+
const doc = decode(entry.block, {
440+
coreDiscoveryKey: entry.key,
441+
index: entry.index,
442+
})
443+
444+
assert(doc.schemaName === 'translation', 'expected a translation doc')
445+
this.#translationApi.index(doc)
446+
otherEntries.push(entry)
423447
} else {
424448
otherEntries.push(entry)
425449
}
@@ -458,6 +482,10 @@ export class MapeoProject extends TypedEmitter {
458482
return this.#syncApi
459483
}
460484

485+
get $translation() {
486+
return this.#translationApi
487+
}
488+
461489
/**
462490
* @param {Partial<EditableProjectSettings>} settings
463491
* @returns {Promise<EditableProjectSettings>}

src/translation-api.js

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
import { and, eq } from 'drizzle-orm'
2+
import { kCreateWithDocId, kSelect } from './datatype/index.js'
3+
import { hashObject } from './utils.js'
4+
import { NotFoundError } from './errors.js'
5+
6+
export const ktranslatedLanguageCodeToSchemaNames = Symbol(
7+
'translatedLanguageCodeToSchemaNames'
8+
)
9+
export default class TranslationApi {
10+
/** @type {Map<
11+
* import('@mapeo/schema').TranslationValue['languageCode'],
12+
* Set<import('@mapeo/schema/dist/types.js').SchemaName>>} */
13+
#translatedLanguageCodeToSchemaNames = new Map()
14+
#dataType
15+
#table
16+
#indexPromise
17+
18+
/**
19+
* @param {Object} opts
20+
* @param {import('./datatype/index.js').DataType<
21+
* import('./datastore/index.js').DataStore<'config'>,
22+
* typeof import('./schema/project.js').translationTable,
23+
* 'translation',
24+
* import('@mapeo/schema').Translation,
25+
* import('@mapeo/schema').TranslationValue
26+
* >} opts.dataType
27+
* @param {typeof import('./schema/project.js').translationTable} opts.table
28+
*/
29+
constructor({ dataType, table }) {
30+
this.#dataType = dataType
31+
this.#table = table
32+
this.#indexPromise = this.#dataType
33+
.getMany()
34+
.then((docs) => {
35+
docs.map((doc) => this.index(doc))
36+
})
37+
.catch((err) => {
38+
throw new Error(`error loading Translation cache: ${err}`)
39+
})
40+
}
41+
42+
/** @returns {Promise<void>} */
43+
ready() {
44+
return this.#indexPromise
45+
}
46+
47+
/**
48+
* @param {import('@mapeo/schema').TranslationValue} value
49+
*/
50+
async put(value) {
51+
/* eslint-disable no-unused-vars */
52+
const { message, ...identifiers } = value
53+
const docId = hashObject(identifiers)
54+
try {
55+
const doc = await this.#dataType.getByDocId(docId)
56+
return await this.#dataType.update(doc.versionId, value)
57+
} catch (e) {
58+
if (e instanceof NotFoundError) {
59+
return await this.#dataType[kCreateWithDocId](docId, value)
60+
} else {
61+
throw new Error(`Error on translation ${e}`)
62+
}
63+
}
64+
}
65+
66+
/**
67+
* @param {import('type-fest').SetOptional<
68+
* Omit<import('@mapeo/schema').TranslationValue,'schemaName' | 'message'>,
69+
* 'fieldRef' | 'regionCode'>} value
70+
* @returns {Promise<import('@mapeo/schema').Translation[]>}
71+
*/
72+
async get(value) {
73+
await this.ready()
74+
75+
const docTypeIsTranslatedToLanguage =
76+
this.#translatedLanguageCodeToSchemaNames
77+
.get(value.languageCode)
78+
?.has(
79+
/** @type {import('@mapeo/schema/dist/types.js').SchemaName} */ (
80+
value.schemaNameRef
81+
)
82+
)
83+
if (!docTypeIsTranslatedToLanguage) return []
84+
85+
const filters = [
86+
eq(this.#table.docIdRef, value.docIdRef),
87+
eq(this.#table.schemaNameRef, value.schemaNameRef),
88+
eq(this.#table.languageCode, value.languageCode),
89+
]
90+
if (value.fieldRef) {
91+
filters.push(eq(this.#table.fieldRef, value.fieldRef))
92+
}
93+
94+
if (value.regionCode) {
95+
filters.push(eq(this.#table.regionCode, value.regionCode))
96+
}
97+
98+
return (await this.#dataType[kSelect]())
99+
.where(and.apply(null, filters))
100+
.prepare()
101+
.all()
102+
}
103+
104+
/**
105+
* @param {import('@mapeo/schema').TranslationValue} doc
106+
*/
107+
index(doc) {
108+
let translatedSchemas = this.#translatedLanguageCodeToSchemaNames.get(
109+
doc.languageCode
110+
)
111+
if (!translatedSchemas) {
112+
translatedSchemas = new Set()
113+
this.#translatedLanguageCodeToSchemaNames.set(
114+
doc.languageCode,
115+
translatedSchemas
116+
)
117+
}
118+
translatedSchemas.add(
119+
/** @type {import('@mapeo/schema/dist/types.js').SchemaName} */ (
120+
doc.schemaNameRef
121+
)
122+
)
123+
}
124+
125+
// This should only be used by tests.
126+
get [ktranslatedLanguageCodeToSchemaNames]() {
127+
return this.#translatedLanguageCodeToSchemaNames
128+
}
129+
}

src/utils.js

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import b4a from 'b4a'
22
import { keyToPublicId } from '@mapeo/crypto'
3+
import { createHash } from 'node:crypto'
4+
import stableStringify from 'json-stable-stringify'
35

46
/**
57
* @param {String|Buffer} id
@@ -191,3 +193,15 @@ export function createMap(keys, value) {
191193
}
192194
return map
193195
}
196+
197+
/**
198+
* create a sha256 hash of an object using json-stable-stringify for deterministic results
199+
* @param {Object} obj
200+
* @returns {String} hash of the object
201+
*/
202+
export function hashObject(obj) {
203+
return createHash('sha256')
204+
.update(stableStringify(obj))
205+
.digest()
206+
.toString('hex')
207+
}

test-e2e/fixtures/translations.js

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
const commonPreset = {
2+
/** @type {'translation'} */
3+
schemaName: 'translation',
4+
schemaNameRef: 'preset',
5+
languageCode: 'es',
6+
regionCode: 'AR',
7+
fieldRef: 'name',
8+
}
9+
10+
const commonField = {
11+
/** @type {'translation'} */
12+
schemaName: 'translation',
13+
schemaNameRef: 'field',
14+
languageCode: 'es',
15+
regionCode: 'AR',
16+
fieldRef: 'label',
17+
}
18+
19+
/** @type {Object.<string,string>} */
20+
export const presetsTranslationMap = {
21+
Airstrip: 'Pista de Aterrizaje',
22+
Boundry: 'Límite',
23+
Cave: 'Cueva',
24+
Building: 'Edificio',
25+
Clay: 'Arcilla',
26+
'New Area': 'Nueva Área',
27+
Camp: 'Campamento',
28+
Community: 'Comunidad',
29+
'Gathering Site': 'Zona de recolección',
30+
Hills: 'Colinas',
31+
House: 'Casa',
32+
'Hunting Site': 'Sitio de Caza',
33+
'Fishing Site': 'Sitio de Pesca',
34+
Palm: 'Palma',
35+
Plant: 'Planta',
36+
Path: 'Camino',
37+
'New point': 'Nuevo punto',
38+
River: 'Río',
39+
'New line': 'Nueva línea',
40+
Lake: 'Lago',
41+
Stream: 'Cauce',
42+
'Special site': 'Sitio especial',
43+
Farmland: 'Tierra de cultivo',
44+
Threat: 'Amenaza',
45+
Waterfall: 'Cascada',
46+
Tree: 'Árbol',
47+
}
48+
49+
/** @type {Object.<string,string>} */
50+
export const fieldsTranslationMap = {
51+
'Animal type': 'Tipo de animal',
52+
'Building type': 'Tipo de edificio',
53+
'What is gathered here?': '¿Qué se recolecta aquí?',
54+
Note: 'Nota',
55+
Owner: 'Dueño',
56+
'Plant species': 'Especie de planta',
57+
'What kind of path?': '¿Qué clase de camino?',
58+
Name: 'Nombre',
59+
'Tree species': 'Especie de árbol',
60+
}
61+
62+
export const presetTranslations = Object.keys(presetsTranslationMap).map(
63+
(key) => {
64+
const translation = presetsTranslationMap[key]
65+
return { ...commonPreset, message: translation }
66+
}
67+
)
68+
69+
export const fieldTranslations = Object.keys(fieldsTranslationMap).map(
70+
(key) => {
71+
const translation = fieldsTranslationMap[key]
72+
return { ...commonField, message: translation }
73+
}
74+
)

0 commit comments

Comments
 (0)