Skip to content

Commit ab5b6c9

Browse files
zigzagdevzigzagdev
authored
Merge pull request #358 from zigzagdev/feat/introduce-study_region
Introduce study region
2 parents 625f3cd + 7338ac3 commit ab5b6c9

13 files changed

+1049
-133
lines changed

‎src/app/Console/Commands/AlgoliaImportWorldHeritages.php‎

Lines changed: 74 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,19 @@
22

33
namespace App\Console\Commands;
44

5-
use Illuminate\Console\Command;
6-
use App\Models\WorldHeritage;
75
use Algolia\AlgoliaSearch\Api\SearchClient;
8-
6+
use App\Models\WorldHeritage;
7+
use Illuminate\Console\Command;
98

109
class AlgoliaImportWorldHeritages extends Command
1110
{
12-
/**
13-
* The name and signature of the console command.
14-
*
15-
* @var string
16-
*/
1711
protected $signature = 'algolia:import-world-heritages
1812
{--chunk=500}
1913
{--truncate}
2014
{--dry-run}';
2115

22-
/**
23-
* The console command description.
24-
*
25-
* @var string
26-
*/
2716
protected $description = 'Upsert world heritages into Algolia index.';
2817

29-
/**
30-
* Execute the console command.
31-
*/
3218
public function handle(): int
3319
{
3420
$appId = config('algolia.algolia_app_id');
@@ -40,13 +26,13 @@ public function handle(): int
4026
return self::FAILURE;
4127
}
4228

43-
$chunk = max(1, (int)$this->option('chunk'));
44-
$dryRun = (bool)$this->option('dry-run');
45-
$truncate = (bool)$this->option('truncate');
29+
$chunk = max(1, (int) $this->option('chunk'));
30+
$dryRun = (bool) $this->option('dry-run');
31+
$truncate = (bool) $this->option('truncate');
32+
$processed = 0;
4633

4734
$client = SearchClient::create($appId, $apiKey);
4835

49-
// writing code to test this
5036
if ($truncate) {
5137
if ($dryRun) {
5238
$this->info('[dry-run] would clear index');
@@ -64,54 +50,90 @@ public function handle(): int
6450

6551
WorldHeritage::query()
6652
->with([
67-
'thumbnail',
68-
'countries' => function ($query) {
69-
$query->select(['countries.state_party_code', 'countries.name_jp']);
70-
},
53+
'countries',
7154
])
7255
->select([
7356
'world_heritage_sites.id',
74-
'official_name',
75-
'name',
57+
'world_heritage_sites.official_name',
58+
'world_heritage_sites.name',
7659
'world_heritage_sites.name_jp',
7760
'world_heritage_sites.region',
78-
'country',
79-
'category',
80-
'year_inscribed',
81-
'is_endangered',
82-
'image_url',
61+
'world_heritage_sites.study_region',
62+
'world_heritage_sites.category',
63+
'world_heritage_sites.year_inscribed',
64+
'world_heritage_sites.is_endangered',
65+
'world_heritage_sites.image_url',
8366
])
8467
->chunkById($chunk, function ($rows) use ($client, $indexName, $dryRun, &$processed) {
8568
$objects = [];
8669

8770
foreach ($rows as $row) {
88-
89-
$statePartyCodes = $row->countries->pluck('state_party_code')->toArray();
90-
$countryNamesJp = $row->countries->pluck('name_jp')->toArray();
71+
$countries = $row->countries
72+
->filter(fn ($country) => $country->state_party_code !== null)
73+
->values();
74+
75+
$statePartyCodes = $countries
76+
->pluck('state_party_code')
77+
->filter()
78+
->values()
79+
->toArray();
80+
81+
$countryNamesEn = $countries
82+
->pluck('name_en')
83+
->filter()
84+
->values()
85+
->toArray();
86+
87+
$countryNamesJp = $countries
88+
->pluck('name_jp')
89+
->filter()
90+
->values()
91+
->toArray();
92+
93+
$countryCount = $countries->count();
94+
95+
$country = null;
96+
$countryNameJp = null;
97+
98+
if ($countryCount === 1) {
99+
$country = $countryNamesEn[0] ?? null;
100+
$countryNameJp = $countryNamesJp[0] ?? null;
101+
}
91102

92103
$objects[] = [
93-
'objectID' => (string)$row->id,
94-
// for sorting in algolia
95-
'id' => (int)$row->id,
96-
'official_name' => (string)$row->official_name,
97-
'name' => (string)$row->name,
98-
'name_jp' => (string)$row->name_jp,
99-
'country' => $row->country !== null ? (string)$row->country : null,
100-
'country_name_jp' => $row->countries->first()?->name_jp,
101-
'region' => (string)$row->region,
102-
'category' => (string)$row->category,
103-
'year_inscribed' => (int)$row->year_inscribed,
104-
'is_endangered' => (bool)$row->is_endangered,
105-
'thumbnail_url' => $row->image_url !== null ? (string)$row->image_url : null,
106-
'country_names_jp' => $countryNamesJp,
107-
'state_party_codes' => $statePartyCodes,
104+
'objectID' => (string) $row->id,
105+
'id' => (int) $row->id,
106+
'official_name' => (string) $row->official_name,
107+
'name' => (string) $row->name,
108+
'name_jp' => (string) $row->name_jp,
109+
'country' => $country,
110+
'country_name_jp' => $countryNameJp,
111+
'region' => (string) $row->region,
112+
'study_region' => (string) $row->study_region,
113+
'category' => (string) $row->category,
114+
'year_inscribed' => $row->year_inscribed !== null ? (int) $row->year_inscribed : null,
115+
'is_endangered' => (bool) $row->is_endangered,
116+
'thumbnail_url' => $row->image_url !== null ? (string) $row->image_url : null,
117+
'state_party_codes' => $countryCount > 1 ? $statePartyCodes : [],
118+
'country_names_jp' => $countryCount > 1 ? $countryNamesJp : [],
108119
];
109120
}
110121

111122
if ($dryRun) {
123+
if ($processed === 0 && isset($objects[0])) {
124+
$this->line(json_encode($objects[0], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES));
125+
}
126+
112127
$processed += count($objects);
113128
return;
114129
}
130+
if ((int) $row->id === 1133) {
131+
dd([
132+
'state_party_codes' => $statePartyCodes,
133+
'country_names_jp' => $countryNamesJp,
134+
'object' => end($objects),
135+
]);
136+
}
115137

116138
$res = $client->saveObjects(
117139
indexName: $indexName,
@@ -123,10 +145,12 @@ public function handle(): int
123145
if ($taskId !== null) {
124146
$client->waitForTask($indexName, $taskId);
125147
}
148+
126149
$processed += count($objects);
127150
});
128151

129152
$this->info("Done: processed={$processed}");
153+
130154
return self::SUCCESS;
131155
}
132-
}
156+
}

‎src/app/Console/Commands/ImportCountriesFromSplitFile.php‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace App\Console\Commands;
44

55
use Illuminate\Console\Command;
6+
use Illuminate\Support\Facades\Config;
67
use Illuminate\Support\Facades\DB;
78
use Illuminate\Support\Facades\Storage;
89

@@ -58,13 +59,20 @@ public function handle(): int
5859
if ($max > 0 && $imported >= $max) {
5960
break;
6061
}
61-
if (!is_array($row)) { $skipped++; continue; }
62+
63+
if (!is_array($row)) {
64+
$skipped++;
65+
continue;
66+
}
6267

6368
$code = strtoupper(trim((string) ($row['state_party_code'] ?? '')));
6469
if ($code === '' || strlen($code) !== 3) {
6570
$skipped++;
6671
if ($strict) {
67-
$this->error("Strict: invalid state_party_code: " . json_encode($row, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES));
72+
$this->error(
73+
'Strict: invalid state_party_code: ' .
74+
json_encode($row, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
75+
);
6876
return self::FAILURE;
6977
}
7078
continue;
@@ -78,6 +86,15 @@ public function handle(): int
7886
$nameEn = $code;
7987
}
8088

89+
if ($nameJp === null) {
90+
$nameJp = $this->resolveCountryNameJapanese($code);
91+
}
92+
93+
if ($strict && $nameJp === null) {
94+
$this->error("Strict: name_jp could not be resolved for state_party_code [{$code}]");
95+
return self::FAILURE;
96+
}
97+
8198
$batch[] = [
8299
'state_party_code' => $code,
83100
'name_en' => $nameEn,
@@ -164,6 +181,16 @@ private function toNullableString(mixed $v): ?string
164181
}
165182

166183
$s = trim($v);
184+
167185
return $s === '' ? null : $s;
168186
}
187+
188+
private function resolveCountryNameJapanese(string $iso3): ?string
189+
{
190+
$countryNameJa = Config::get('country_ja.alpha3_to_country.' . strtoupper(trim($iso3)));
191+
192+
return is_string($countryNameJa) && $countryNameJa !== ''
193+
? $countryNameJa
194+
: null;
195+
}
169196
}

‎src/app/Console/Commands/ImportWorldHeritageFromJson.php‎

Lines changed: 25 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
use App\Models\WorldHeritage;
66
use Carbon\Carbon;
77
use Illuminate\Console\Command;
8+
use App\Support\StudyRegionResolver;
89

910
class ImportWorldHeritageFromJson extends Command
1011
{
@@ -133,26 +134,17 @@ private function mapFromUnescoApiRow(array $row): array
133134
$id = $row['id_no'] ?? null;
134135
$lat = $row['coordinates']['lat'] ?? null;
135136
$lon = $row['coordinates']['lon'] ?? null;
136-
$criteriaRaw = $row['criteria_txt'] ?? $row['criteria'] ?? null;
137-
$stateParty = $row['states'] ?? $row['state_party'] ?? null;
138-
if (is_array($stateParty)) {
139-
$stateParty = $stateParty[0] ?? null;
140-
}
141137

142-
$stateParty = is_string($stateParty) ? strtoupper(trim($stateParty)) : null;
143-
if ($stateParty === '') {
144-
$stateParty = null;
145-
}
146-
if ($stateParty !== null && !preg_match('/^[A-Z]{3}$/', $stateParty)) {
147-
$stateParty = null;
148-
}
138+
$countryName = $this->extractCountryName($row);
139+
$statePartyIso3 = $this->extractIso3StateParty($row);
149140

150141
return [
151142
'id' => $this->toNullableInt($id),
152143
'official_name' => $row['official_name'] ?? null,
153144
'name' => $row['name_en'] ?? $row['name'] ?? null,
154145
'region' => $row['region_en'] ?? $row['region'] ?? null,
155-
'state_party' => $stateParty,
146+
'state_party' => $statePartyIso3,
147+
'study_region' => StudyRegionResolver::resolve($countryName)->value,
156148
'category' => $row['category'] ?? $row['type'] ?? null,
157149
'criteria' => $row['criteria'] ?? null,
158150
'year_inscribed' => $this->toNullableInt($row['date_inscribed'] ?? $row['year_inscribed'] ?? null),
@@ -168,29 +160,6 @@ private function mapFromUnescoApiRow(array $row): array
168160
];
169161
}
170162

171-
private function criteriaFromTxt(mixed $raw): array
172-
{
173-
if ($raw === null) {
174-
return [];
175-
}
176-
177-
$s = trim((string) $raw);
178-
if ($s === '') {
179-
return [];
180-
}
181-
182-
preg_match_all('/\(([^)]+)\)/', $s, $m);
183-
if (isset($m[1]) && $m[1] !== []) {
184-
return array_values(array_filter(array_map(fn($v) => trim((string) $v), $m[1])));
185-
}
186-
187-
$s = trim($s, " \t\n\r\0\x0B()");
188-
if ($s === '') {
189-
return [];
190-
}
191-
return [$s];
192-
}
193-
194163
private function flushBatch(array $batch): int
195164
{
196165
$updateColumns = array_values(array_diff(array_keys($batch[0]), ['id']));
@@ -241,6 +210,26 @@ private function extractIso3StateParty(array $row): ?string
241210
return null;
242211
}
243212

213+
private function extractCountryName(array $row): ?string
214+
{
215+
$states = $row['states'] ?? $row['state_party'] ?? null;
216+
217+
if (is_string($states)) {
218+
$normalized = trim($states);
219+
return $normalized !== '' ? $normalized : null;
220+
}
221+
222+
if (is_array($states)) {
223+
$first = $states[0] ?? null;
224+
if (is_string($first)) {
225+
$normalized = trim($first);
226+
return $normalized !== '' ? $normalized : null;
227+
}
228+
}
229+
230+
return null;
231+
}
232+
244233
private function toNullableInt(mixed $v): ?int
245234
{
246235
if ($v === null || $v === '') {

‎src/app/Console/Commands/ImportWorldHeritageSiteFromSplitFile.php‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ public function handle(): int
7878
'official_name' => $this->toNullableString($row['official_name'] ?? null),
7979
'name' => $this->toNullableString($row['name'] ?? null),
8080
'name_jp' => $this->toNullableString($row['name_jp'] ?? null),
81+
'study_region' => $this->toNullableString($row['study_region'] ?? null),
8182
'country' => $this->toNullableString($row['country'] ?? null),
8283
'region' => $this->toNullableString($row['region'] ?? null),
8384
'state_party' => $this->toNullableString($row['state_party'] ?? null),
@@ -200,23 +201,29 @@ private function toNullableFloat(mixed $v): ?float
200201
private function toNullableBoolInt(mixed $v): ?int
201202
{
202203
if ($v === null || $v === '') {
203-
return null;
204+
return 0;
204205
}
206+
205207
if (is_bool($v)) {
206208
return $v ? 1 : 0;
207209
}
210+
208211
if (is_int($v) || is_float($v)) {
209212
return ((int) $v) === 1 ? 1 : 0;
210213
}
214+
211215
if (is_string($v)) {
212216
$s = strtolower(trim($v));
217+
213218
if (in_array($s, ['1', 'true', 't', 'yes', 'y', 'on'], true)) {
214219
return 1;
215220
}
221+
216222
if (in_array($s, ['0', 'false', 'f', 'no', 'n', 'off'], true)) {
217223
return 0;
218224
}
219225
}
220-
return null;
226+
227+
return 0;
221228
}
222229
}

0 commit comments

Comments
 (0)