-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmetadata_manager.rb
More file actions
166 lines (133 loc) · 5.84 KB
/
Copy pathmetadata_manager.rb
File metadata and controls
166 lines (133 loc) · 5.84 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
require 'faraday'
require 'tempfile'
require_relative 'observability_setup'
module MetadataManager
extend self
def sync_metadata(db, route, route_name)
config_metadata = route[:metadata] || {}
special_fields = %w[name format tileSize minzoom maxzoom].map(&:to_sym)
metadata_to_sync = config_metadata.reject { |k, _| special_fields.include?(k) }.transform_keys(&:to_s).transform_values(&:to_s)
metadata_to_sync['name'] = (config_metadata[:name] || format_name(route_name)).to_s
if config_metadata[:format]
metadata_to_sync['format'] = config_metadata[:format].to_s
metadata_to_sync['tileSize'] = (config_metadata[:tileSize] || config_metadata[:tile_size]).to_s if config_metadata[:tileSize] || config_metadata[:tile_size]
elsif (format_and_size = detect_format_and_tile_size(route))
metadata_to_sync['format'] = format_and_size[:format].to_s if format_and_size[:format]
metadata_to_sync['tileSize'] = format_and_size[:tile_size].to_s if format_and_size[:tile_size]
end
%i[minzoom maxzoom].each do |key|
value = config_metadata[key] || route[key]
metadata_to_sync[key.to_s] = value.to_s if value
end
metadata_to_sync.each do |key, value|
db[:metadata].insert_conflict(target: :name, update: { value: Sequel[:excluded][:value] }).insert(name: key, value: value)
end
end
private
def format_name(route_name)
route_name.to_s.tr('_', ' ').split.map(&:capitalize).join(' ')
end
def detect_format_and_tile_size(route)
otl_span('metadata.detect', { source: route[:observability_source] }) do
config_metadata = route[:metadata] || {}
min_zoom = config_metadata[:minzoom] || route[:minzoom]
max_zoom = config_metadata[:maxzoom] || route[:maxzoom]
return nil if min_zoom.nil? || max_zoom.nil?
range = max_zoom - min_zoom
test_zooms = range <= 3 ? (min_zoom..max_zoom).to_a :
[min_zoom, min_zoom + range / 3, min_zoom + range * 2 / 3, max_zoom].map(&:round).uniq
hot_spots = [[37.6176, 55.7558], [-0.1276, 51.5074], [-74.0060, 40.7128], [0, 0]]
test_zooms.each do |zoom|
hot_spots.each do |lon, lat|
x = ((lon + 180) / 360 * (1 << zoom)).floor
y = ((1 - Math.log(Math.tan(lat * Math::PI / 180) + 1 / Math.cos(lat * Math::PI / 180)) / Math::PI) / 2 * (1 << zoom)).floor
next if x < 0 || y < 0 || x >= (1 << zoom) || y >= (1 << zoom)
result = try_fetch_tile(route, zoom, x, y)
return result if result
end
end
LOGGER.warn("event=metadata_detect_failed source=#{route[:observability_source]}")
nil
end
end
def try_fetch_tile(route, zoom, x, y)
test_url = route[:target].gsub('{z}', zoom.to_s).gsub('{x}', x.to_s).gsub('{y}', y.to_s)
test_url += "?#{URI.encode_www_form(route[:query_params])}" if route[:query_params]
uri = URI.parse(test_url)
response, duration_ms = Observability.measure_duration do
route[:client].get(uri.path + (uri.query ? "?#{uri.query}" : '')) do |req|
req.headers.merge!(test_headers(route))
req.options.timeout = 5
end
end
record_metadata_probe_result(
source: route[:observability_source],
z: zoom,
x: x,
y: y,
status: response.status,
reason: response.success? ? 'metadata_probe' : 'metadata_probe_failed',
duration_ms: duration_ms,
bytes: response.body&.bytesize,
content_type: response.headers['content-type'],
host: uri.host
)
return nil unless response.success? && response.body&.size.to_i > 100
return nil unless response.headers['content-type']&.include?('image/')
format = detect_format_from_content_type(response.headers['content-type'])
tile_size = detect_tile_size_from_data(response.body, format)
return nil unless format && tile_size
{ format: format, tile_size: tile_size }
rescue => e
LOGGER.warn("event=metadata_probe_error source=#{route[:observability_source]} z=#{zoom} x=#{x} y=#{y} error=#{e.message}")
nil
end
def record_metadata_probe_result(source:, z:, x:, y:, status:, reason:, duration_ms:, bytes: nil, content_type: nil, host: nil)
return if status.to_i < 500 && status.to_i != 0
LOGGER.warn(
"event=metadata_probe_upstream_error source=#{source} z=#{z} x=#{x} y=#{y} status=#{status} " \
"reason=#{reason} duration_ms=#{duration_ms} bytes=#{bytes} content_type=#{content_type} host=#{host}"
)
end
def detect_format_from_content_type(content_type)
case content_type
when /image\/png/ then 'png'
when /image\/jpe?g/ then 'jpg'
when /image\/webp/ then 'webp'
when /image\/tiff/ then 'tiff'
when /image\/bmp/ then 'bmp'
when /image\/gif/ then 'gif'
else nil
end
end
def detect_tile_size_from_data(image_data, format)
return nil unless image_data && format
Tempfile.create(['tile', ".#{format}"]) do |f|
f.binmode
f.write(image_data)
f.close
output = `file "#{f.path}" 2>/dev/null`
standard_sizes = [256, 512, 1024, 128, 64, 32]
return output.match(/height=(\d+).*width=(\d+)/) { |m| $1.to_i if $1.to_i == $2.to_i && standard_sizes.include?($1.to_i) } if format == 'tiff'
standard_sizes.each do |size|
if output.include?("#{size}x#{size}") || output.include?("#{size} x #{size}")
return size
end
end
nil
end
rescue => e
LOGGER.warn("event=metadata_tile_size_detect_error error=#{e.message}")
nil
end
def test_headers(route)
config_headers = route.dig(:headers, :request) || {}
default_headers = {
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Accept' => 'image/png,image/jpeg,image/webp,image/tiff,image/bmp,image/gif,*/*',
'Accept-Language' => 'en-US,en;q=0.9',
'Cache-Control' => 'no-cache'
}
default_headers.merge(config_headers)
end
end