Skip to content

Commit 6e8ece7

Browse files
committed
Add PopulationPriorGeocoder with a documented heuristic confidence
Resolves each location mention independently: candidates come from the gazetteer lookup and are re-ranked here under the module's shared deterministic order (population descending, CITY over ADMIN over POI on population ties, then source and record id), so the result never depends on the gazetteer's return order. Unresolved mentions are omitted; results stay aligned to the input spans in input order. Confidence is an explicitly documented heuristic prior, not a probability: 0.9 for a single candidate, otherwise 0.5 + 0.4 * (p1 - p2) / (p1 + p2), monotonic in the relative population separation between winner and runner-up, 0.5 on a dead tie, and never 1.0. Document-context-aware scoring (co-occurring mentions constraining each other) is the named follow-up as a second Geocoder implementation; resolve() already receives the whole document for that reason. The candidate order moves into a shared package-private CandidateRanking used by both BundledGazetteer and the geocoder.
1 parent ea7fe5b commit 6e8ece7

4 files changed

Lines changed: 425 additions & 26 deletions

File tree

opennlp-extensions/opennlp-geo/src/main/java/opennlp/geo/BundledGazetteer.java

Lines changed: 3 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import java.io.UncheckedIOException;
2424
import java.nio.charset.StandardCharsets;
2525
import java.util.ArrayList;
26-
import java.util.Comparator;
2726
import java.util.HashMap;
2827
import java.util.HashSet;
2928
import java.util.LinkedHashMap;
@@ -77,14 +76,6 @@ public final class BundledGazetteer implements Gazetteer {
7776

7877
private static final String RESOURCE = "naturalearth-populated-places.txt";
7978

80-
// Population descending, then feature-class prior, then source and record id: a deterministic
81-
// total order over any candidate list.
82-
private static final Comparator<GazetteerEntry> RANKING =
83-
Comparator.comparingLong(GazetteerEntry::population).reversed()
84-
.thenComparingInt(entry -> featureClassRank(entry.featureClass()))
85-
.thenComparing(GazetteerEntry::source)
86-
.thenComparing(GazetteerEntry::recordId);
87-
8879
// The character-level matching chain; stateless and thread-safe, shared by index and queries.
8980
private static final TermAnalyzer FOLD =
9081
TermAnalyzer.builder().nfc().caseFold().accentFold().build();
@@ -126,13 +117,13 @@ public final class BundledGazetteer implements Gazetteer {
126117
indexName(byName, alternateName, entry);
127118
}
128119
if (entry.countryCode() != null) {
129-
byRegion.merge(entry.countryCode(), entry,
130-
(existing, candidate) -> RANKING.compare(candidate, existing) < 0 ? candidate : existing);
120+
byRegion.merge(entry.countryCode(), entry, (existing, candidate) ->
121+
CandidateRanking.BY_PRIOR.compare(candidate, existing) < 0 ? candidate : existing);
131122
}
132123
}
133124
for (final Map.Entry<String, List<GazetteerEntry>> indexed : byName.entrySet()) {
134125
final List<GazetteerEntry> ranked = new ArrayList<>(indexed.getValue());
135-
ranked.sort(RANKING);
126+
ranked.sort(CandidateRanking.BY_PRIOR);
136127
indexed.setValue(List.copyOf(ranked));
137128
}
138129
this.idIndex = byId;
@@ -225,20 +216,6 @@ static String foldKey(CharSequence name) {
225216
return key.toString();
226217
}
227218

228-
// CITY before ADMIN before POI before anything else (including unknown); used on ties only.
229-
static int featureClassRank(String featureClass) {
230-
if ("CITY".equals(featureClass)) {
231-
return 0;
232-
}
233-
if ("ADMIN".equals(featureClass)) {
234-
return 1;
235-
}
236-
if ("POI".equals(featureClass)) {
237-
return 2;
238-
}
239-
return 3;
240-
}
241-
242219
/**
243220
* Parses gazetteer rows from a stream of the bundled table format. Package-private so the
244221
* fail-loud handling can be exercised without the bundled resource.
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package opennlp.geo;
18+
19+
import java.util.Comparator;
20+
21+
import opennlp.tools.geo.GazetteerEntry;
22+
23+
/**
24+
* The shared candidate order of this module: population descending, then a small feature-class
25+
* prior ({@code CITY} before {@code ADMIN} before {@code POI} before anything else, applied on
26+
* population ties only), then source and record id. The trailing identifier comparison makes the
27+
* order total, so any candidate list sorts the same way on every run.
28+
*/
29+
final class CandidateRanking {
30+
31+
/** Population descending, feature-class prior, then source and record id. */
32+
static final Comparator<GazetteerEntry> BY_PRIOR =
33+
Comparator.comparingLong(GazetteerEntry::population).reversed()
34+
.thenComparingInt(entry -> featureClassRank(entry.featureClass()))
35+
.thenComparing(GazetteerEntry::source)
36+
.thenComparing(GazetteerEntry::recordId);
37+
38+
private CandidateRanking() {
39+
}
40+
41+
// CITY before ADMIN before POI before anything else (including unknown).
42+
static int featureClassRank(String featureClass) {
43+
if ("CITY".equals(featureClass)) {
44+
return 0;
45+
}
46+
if ("ADMIN".equals(featureClass)) {
47+
return 1;
48+
}
49+
if ("POI".equals(featureClass)) {
50+
return 2;
51+
}
52+
return 3;
53+
}
54+
}
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package opennlp.geo;
18+
19+
import java.util.ArrayList;
20+
import java.util.List;
21+
22+
import opennlp.tools.geo.Gazetteer;
23+
import opennlp.tools.geo.GazetteerEntry;
24+
import opennlp.tools.geo.GeoResolution;
25+
import opennlp.tools.geo.Geocoder;
26+
import opennlp.tools.util.Span;
27+
28+
/**
29+
* A {@link Geocoder} that resolves each mention by a population prior: candidates come from
30+
* {@link Gazetteer#lookup(CharSequence)} and the winner is the first under the module's
31+
* deterministic candidate order (population descending, then the {@code CITY} over {@code ADMIN}
32+
* over {@code POI} feature-class prior on population ties, then source and record id). The
33+
* candidates are re-sorted here, so the result does not depend on the gazetteer's own return
34+
* order.
35+
*
36+
* <p><b>Confidence is a documented heuristic, not a probability.</b> For a single candidate it
37+
* is {@code 0.9}; for several it is {@code 0.5 + 0.4 * (p1 - p2) / (p1 + p2)}, where {@code p1}
38+
* and {@code p2} are the populations of the winner and the runner-up ({@code 0.5} when both are
39+
* unknown). The value is monotonic in the relative population separation: a dead tie scores
40+
* {@code 0.5}, a dominant winner approaches {@code 0.9}, and {@code 1.0} is never reported
41+
* because a prior with no context cannot certify a resolution. Document-context-aware scoring
42+
* (co-occurring mentions constraining each other) is the named follow-up, as a second
43+
* {@link Geocoder} implementation behind the same contract; the whole document is already part
44+
* of {@link #resolve(CharSequence, List)} for exactly that reason, and this implementation
45+
* deliberately does not read it beyond the mention spans.</p>
46+
*
47+
* <p>Mentions are resolved independently and in input order; a mention with no candidates is
48+
* omitted from the result. Instances are immutable and thread-safe when the supplied
49+
* {@link Gazetteer} is (the bundled one is).</p>
50+
*/
51+
public final class PopulationPriorGeocoder implements Geocoder {
52+
53+
private static final double SINGLE_CANDIDATE_CONFIDENCE = 0.9;
54+
private static final double TIE_CONFIDENCE = 0.5;
55+
private static final double SEPARATION_WEIGHT = 0.4;
56+
57+
private final Gazetteer gazetteer;
58+
59+
/**
60+
* Creates a geocoder over the given gazetteer.
61+
*
62+
* @param gazetteer The gazetteer to resolve against. Must not be {@code null}.
63+
* @throws IllegalArgumentException Thrown if {@code gazetteer} is {@code null}.
64+
*/
65+
public PopulationPriorGeocoder(Gazetteer gazetteer) {
66+
if (gazetteer == null) {
67+
throw new IllegalArgumentException("Gazetteer must not be null");
68+
}
69+
this.gazetteer = gazetteer;
70+
}
71+
72+
@Override
73+
public List<GeoResolution> resolve(CharSequence text, List<Span> locationMentions) {
74+
if (text == null) {
75+
throw new IllegalArgumentException("Text must not be null");
76+
}
77+
if (locationMentions == null) {
78+
throw new IllegalArgumentException("LocationMentions must not be null");
79+
}
80+
for (final Span mention : locationMentions) {
81+
if (mention == null) {
82+
throw new IllegalArgumentException(
83+
"LocationMentions must not contain a null element, got: " + locationMentions);
84+
}
85+
if (mention.getEnd() > text.length()) {
86+
throw new IllegalArgumentException("Mention " + mention
87+
+ " is outside the text, whose length is " + text.length());
88+
}
89+
}
90+
final List<GeoResolution> resolutions = new ArrayList<>(locationMentions.size());
91+
for (final Span mention : locationMentions) {
92+
final CharSequence mentionText = text.subSequence(mention.getStart(), mention.getEnd());
93+
final List<GazetteerEntry> candidates = new ArrayList<>(gazetteer.lookup(mentionText));
94+
if (candidates.isEmpty()) {
95+
continue; // unresolved mentions are omitted, never fabricated
96+
}
97+
candidates.sort(CandidateRanking.BY_PRIOR);
98+
resolutions.add(new GeoResolution(mention, candidates.get(0), confidence(candidates)));
99+
}
100+
return resolutions;
101+
}
102+
103+
// The heuristic prior documented in the class javadoc: monotonic in the relative population
104+
// separation between the winner and the runner-up, bounded away from certainty.
105+
private static double confidence(List<GazetteerEntry> rankedCandidates) {
106+
if (rankedCandidates.size() == 1) {
107+
return SINGLE_CANDIDATE_CONFIDENCE;
108+
}
109+
final long first = rankedCandidates.get(0).population();
110+
final long second = rankedCandidates.get(1).population();
111+
final long total = first + second;
112+
if (total == 0) {
113+
return TIE_CONFIDENCE;
114+
}
115+
return TIE_CONFIDENCE + SEPARATION_WEIGHT * ((double) (first - second) / total);
116+
}
117+
}

0 commit comments

Comments
 (0)