Skip to content

Commit 024c1fc

Browse files
Merge remote-tracking branch 'upstream/master'
# Conflicts: # README.md # openaddr/conform.py # openaddr/tests/conform.py
2 parents 5145ed4 + 8634e8e commit 024c1fc

16 files changed

Lines changed: 500 additions & 72 deletions

CHANGELOG

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,45 @@
1+
2026-03-27 v10.0.0
2+
- Upgrade base Docker image from GDAL 3.7.1 to 3.11.0 https://github.com/openaddresses/batch-machine/pull/99
3+
- Update SSL CA certificates to fix download failures https://github.com/openaddresses/batch-machine/pull/98
4+
- Bump requests from 2.32.4 to 2.33.0 https://github.com/openaddresses/batch-machine/pull/97
5+
6+
2026-03-15 v9.18.0
7+
- Add centerline fields and mph_to_kph function https://github.com/openaddresses/batch-machine/pull/96
8+
9+
2026-03-15 v9.17.0
10+
- Add centerlines layer support https://github.com/openaddresses/batch-machine/pull/95
11+
- Fix Docker build and update test hash for Shapely compatibility https://github.com/openaddresses/batch-machine/pull/94
12+
- Bump protobuf from 4.25.8 to 5.29.6 https://github.com/openaddresses/batch-machine/pull/89
13+
14+
2026-01-31 v9.16.0
15+
- Add support for geopackage https://github.com/openaddresses/batch-machine/pull/88
16+
17+
2026-01-07 v9.15.0
18+
- Recursively expand ZIP files https://github.com/openaddresses/batch-machine/pull/87
19+
20+
2025-12-14 v9.14.0
21+
- Include building height and levels as part of schema https://github.com/openaddresses/batch-machine/pull/85
22+
23+
2025-12-14 v9.13.0
24+
- Handle accuracy as an integer when building fields to download in Esri https://github.com/openaddresses/batch-machine/pull/84
25+
- Prevent extra spaces in join function https://github.com/openaddresses/batch-machine/pull/83
26+
27+
2025-11-24 v9.12.0
28+
- Support for accuracy column in row https://github.com/openaddresses/batch-machine/pull/82
29+
- Keep filename extension if present in case of .gz files https://github.com/openaddresses/batch-machine/pull/81
30+
- Upgrade protobuf and requests https://github.com/openaddresses/batch-machine/pull/79 and https://github.com/openaddresses/batch-machine/pull/80
31+
32+
2025-04-13 v9.11.0
33+
- Make regexp function slightly more resilient in https://github.com/openaddresses/batch-machine/pull/78
34+
- Remove semicolons at the end of lines in https://github.com/openaddresses/batch-machine/pull/77
35+
- Only perform decompress step if requested in conform in https://github.com/openaddresses/batch-machine/pull/76
36+
- Move filter to the right spot when calculating field names to request in https://github.com/openaddresses/batch-machine/pull/75
37+
38+
2025-04-08 v9.10.0
39+
- Handle field names for constant function in https://github.com/openaddresses/batch-machine/pull/74
40+
141
2025-02-07 v9.9.0
2-
- Fix bug leading to excessive memory consumotion when gunziping streams
42+
- Fix bug leading to excessive memory consumotion when gunziping streams in https://github.com/openaddresses/batch-machine/pull/70
343

444
2024-11-23 v9.8.0
545
- Fix output generation when first_non_empty function doesn't generate output in https://github.com/openaddresses/batch-machine/pull/69

Dockerfile

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
FROM ghcr.io/osgeo/gdal:alpine-normal-3.7.1
1+
FROM ghcr.io/osgeo/gdal:alpine-normal-3.11.0
22

3-
RUN apk add nodejs yarn git python3 python3-dev py3-pip \
4-
make bash sqlite-dev zlib-dev geos geos-dev \
5-
postgresql-libs gcc g++ musl-dev postgresql-dev cairo \
3+
RUN apk add --no-cache nodejs yarn git python3-dev py3-pip \
4+
make sqlite-dev zlib-dev geos-dev \
5+
gcc g++ musl-dev postgresql-dev cairo \
66
py3-cairo file
77

88
# Download and install Tippecanoe
@@ -15,6 +15,6 @@ RUN git clone -b 2.31.0 https://github.com/felt/tippecanoe.git /tmp/tippecanoe &
1515
WORKDIR /usr/local/src/batch-machine
1616
ADD . /usr/local/src/batch-machine
1717

18-
RUN pip3 install .
18+
RUN pip3 install --break-system-packages .
1919

2020
CMD python3 test.py

README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,12 @@ mkdir my-output
5959
openaddr-process-one --skip-preview --layer addresses --layersource state example.json my-output
6060
```
6161

62+
Supported layer types are `addresses`, `parcels`, `buildings`, and `centerlines`.
63+
6264
Review https://github.com/openaddresses/openaddresses/blob/master/CONTRIBUTING.md for input json syntax.
6365

66+
Supported conform formats include `shapefile`, `geojson`, `csv`, `xml`, `gdb`, and `gpkg`.
67+
6468
## Geocodio fork notes
6569

6670
This is Geocodio's fork of `openaddresses/batch-machine`. The following changes

openaddr/VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
9.9.0
1+
10.0.0

openaddr/__init__.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
elaborate_filenames,
2424
ADDRESSES_SCHEMA,
2525
BUILDINGS_SCHEMA,
26+
CENTERLINES_SCHEMA,
2627
PARCELS_SCHEMA,
2728
)
2829

@@ -48,6 +49,8 @@ def __init__(self, source, layer, layersource):
4849
self.SCHEMA = BUILDINGS_SCHEMA
4950
elif self.layer == 'parcels':
5051
self.SCHEMA = PARCELS_SCHEMA
52+
elif self.layer == 'centerlines':
53+
self.SCHEMA = CENTERLINES_SCHEMA
5154

5255
def cache(source_config, destdir, extras):
5356
''' Python wrapper for openaddress-cache.
@@ -129,10 +132,15 @@ def conform(source_config, destdir, extras):
129132
downloaded_path = task1.download(source_urls, workdir, source_config)
130133
_L.info("Downloaded to %s", downloaded_path)
131134

132-
task2 = DecompressionTask.from_format_string(source_config.data_source.get('compression'))
133-
names = elaborate_filenames(source_config.data_source.get('conform', {}).get('file', None))
134-
decompressed_paths = task2.decompress(downloaded_path, workdir, names)
135-
_L.info("Decompressed to %d files", len(decompressed_paths))
135+
compression = source_config.data_source.get('compression')
136+
if compression:
137+
task2 = DecompressionTask.from_format_string(compression)
138+
names = elaborate_filenames(source_config.data_source.get('conform', {}).get('file', None))
139+
decompressed_paths = task2.decompress(downloaded_path, workdir, names)
140+
_L.info("Decompressed to %d files", len(decompressed_paths))
141+
else:
142+
decompressed_paths = downloaded_path
143+
_L.info("No decompression requested")
136144

137145
task4 = ConvertToGeojsonTask()
138146
try:

openaddr/cache.py

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,12 @@ def guess_url_file_extension(url):
153153
mimetypes.add_type('application/vnd.geo+json', '.json', False)
154154

155155
_, likely_ext = os.path.splitext(path)
156-
bad_extensions = '', '.cgi', '.php', '.aspx', '.asp', '.do'
156+
if likely_ext == '.gz':
157+
_, compressed_ext = os.path.splitext(path[:-3])
158+
if compressed_ext:
159+
likely_ext = compressed_ext + likely_ext
160+
161+
bad_extensions = "", ".cgi", ".php", ".aspx", ".asp", ".do"
157162

158163
if not query and likely_ext not in bad_extensions:
159164
#
@@ -315,18 +320,25 @@ def get_file_path(self, url, dir_path):
315320
@classmethod
316321
def fields_from_conform_function(cls, v):
317322
fxn = v.get('function')
318-
if fxn:
319-
if fxn in ('join', 'format'):
320-
return set(v['fields'])
321-
elif fxn == 'chain':
322-
fields = set()
323-
user_vars = set([v['variable']])
324-
for func in v['functions']:
325-
if isinstance(func, dict) and 'function' in func:
326-
fields |= cls.fields_from_conform_function(func) - user_vars
327-
return fields
328-
else:
329-
return set([v.get('field')])
323+
if not fxn:
324+
return set()
325+
326+
if fxn in ('join', 'format'):
327+
# Join and format functions are a list of fields
328+
return set(v['fields'])
329+
elif fxn == 'chain':
330+
# Chain function is a list of functions that we should recurse into for field names
331+
fields = set()
332+
user_vars = set([v['variable']])
333+
for func in v['functions']:
334+
if isinstance(func, dict) and 'function' in func:
335+
fields |= cls.fields_from_conform_function(func) - user_vars
336+
return fields
337+
elif fxn == 'constant':
338+
# Constant function doesn't use any fields
339+
return set()
340+
else:
341+
return set([v.get('field')])
330342

331343
@classmethod
332344
def field_names_to_request(cls, source_config):
@@ -348,11 +360,17 @@ def field_names_to_request(cls, source_config):
348360
elif isinstance(v, list):
349361
# It's a list of field names
350362
fields |= set(v)
351-
else:
363+
elif isinstance(v, str):
364+
# It's a direct field name mapping
352365
fields.add(v)
366+
# Note: We intentionally skip non-string scalar values (e.g., integers) because
367+
# Esri only supports string field names. Also, the 'accuracy' field can be
368+
# set to an integer constant (like 1) to indicate a fixed accuracy level,
369+
# rather than a field name to fetch from the source.
353370

354371
if fields:
355-
return list(filter(None, sorted(fields)))
372+
# Remove any blank or None values
373+
return list(sorted(filter(None, fields)))
356374
else:
357375
return None
358376

0 commit comments

Comments
 (0)