diff --git a/src/app/Console/Commands/AlgoliaImportWorldHeritages.php b/src/app/Console/Commands/AlgoliaImportWorldHeritages.php index b1ac4d4..4ab5b09 100644 --- a/src/app/Console/Commands/AlgoliaImportWorldHeritages.php +++ b/src/app/Console/Commands/AlgoliaImportWorldHeritages.php @@ -145,14 +145,6 @@ public function handle(): int return; } - if ((int) $row->id === 1133) { - dd([ - 'state_party_codes' => $statePartyCodes, - 'country_names_jp' => $countryNamesJp, - 'object' => end($objects), - ]); - } - $res = $client->saveObjects( indexName: $indexName, objects: $objects diff --git a/src/app/Console/Commands/BackfillStateParties.php b/src/app/Console/Commands/BackfillStateParties.php deleted file mode 100644 index 1f15bcf..0000000 --- a/src/app/Console/Commands/BackfillStateParties.php +++ /dev/null @@ -1,76 +0,0 @@ -whereNotNull('state_party'); - - if ($ids = $this->option('site-id')) { - $query->whereIn('id', $ids); - } - - $processed = 0; - - $query->orderBy('id')->chunkById(500, function ($sites) use (&$processed) { - foreach ($sites as $site) { - $codes = collect(preg_split('/[;,\s]+/', strtoupper((string) $site->state_party))) - ->filter(fn($c) => strlen($c) === 2) - ->values(); - - if ($codes->isEmpty()) { - continue; - } - - $valid = Country::whereIn('state_party_code', $codes)->pluck('state_party_code')->all(); - $missing = array_diff($codes->all(), $valid); - if ($missing !== []) { - Log::warning("Unknown codes for site {$site->id}: ".implode(',', $missing)); - } - - $payload = []; - foreach ($valid as $i => $code) { - $payload[$code] = [ - 'is_primary' => $i === 0, - 'inscription_year' => $site->year_inscribed, - ]; - } - - if ($this->option('dry-run')) { - $this->line("[dry-run] site {$site->id} -> ".json_encode($payload)); - } else { - $site->countries()->syncWithoutDetaching($payload); - } - - $processed++; - } - }); - - $this->info("Processed {$processed} site(s)."); - return self::SUCCESS; - } -} diff --git a/src/app/Console/Commands/BackfillThumbnailImageId.php b/src/app/Console/Commands/BackfillThumbnailImageId.php deleted file mode 100644 index b8f937e..0000000 --- a/src/app/Console/Commands/BackfillThumbnailImageId.php +++ /dev/null @@ -1,43 +0,0 @@ -option('dry-run'); - - WorldHeritage::chunk(100, function ($sites) use ($dryRun) { - foreach ($sites as $site) { - $image = $site->images()->orderBy('sort_order')->first(); - - if (! $image) { - Log::warning('WorldHeritage has no images', ['id' => $site->id]); - $this->warn("Site {$site->id} has no images"); - continue; - } - - if ($site->thumbnail_image_id === $image->id) { - continue; - } - - $this->info("Site {$site->id}: thumbnail_image_id -> {$image->id}"); - - if (! $dryRun) { - $site->thumbnail_image_id = $image->id; - $site->save(); - } - } - }); - - return Command::SUCCESS; - } -} diff --git a/src/app/Console/Commands/BuildWorldHeritageLocalDb.php b/src/app/Console/Commands/BuildWorldHeritageLocalDb.php deleted file mode 100644 index a07e143..0000000 --- a/src/app/Console/Commands/BuildWorldHeritageLocalDb.php +++ /dev/null @@ -1,105 +0,0 @@ - import)'; - - /** - * Execute the console command. - */ - public function handle(): int - { - $in = (string) $this->option('in'); - $out = (string) $this->option('out'); - $pretty = (bool) $this->option('pretty'); - $clean = (bool) $this->option('clean'); - - $siteJudgementsOut = rtrim($out, '/') . '/site-country-judgements.json'; - $exceptionsOut = rtrim($out, '/') . '/exceptions-missing-iso-codes.json'; - - $this->info('Running: optimize:clear'); - $this->mustRun('optimize:clear'); - - if (!(bool) $this->option('skip-migrate')) { - $this->info('Running: migrate:fresh'); - $this->mustRun('migrate:fresh'); - } - - $this->info('Running: world-heritage:split-json'); - $splitArgs = [ - '--in' => $in, - '--out' => $out, - '--site-judgements-out' => $siteJudgementsOut, - '--exceptions-out' => $exceptionsOut, - ]; - if ($pretty) { - $splitArgs['--pretty'] = true; - } - if ($clean) { - $splitArgs['--clean'] = true; - } - $this->mustRun('world-heritage:split-json', $splitArgs); - - $this->info('Running: import-countries-split'); - $this->mustRun('world-heritage:import-countries-split', [ - '--in' => rtrim($out, '/') . '/countries.json', - ]); - - $this->info('Running: import-sites-split'); - $this->mustRun('world-heritage:import-sites-split', [ - '--in' => rtrim($out, '/') . '/world_heritage_sites.json', - ]); - - $this->info('Running: import-site-state-parties-split'); - $this->mustRun('world-heritage:import-site-state-parties-split', [ - '--in' => rtrim($out, '/') . '/site_state_parties.json', - ]); - - $this->info('Running: import-images-json'); - $this->mustRun('world-heritage:import-images-json', [ - '--path' => rtrim($out, '/') . '/world_heritage_site_images.json', - ]); - - $this->info('Running: import-site-country-exceptions'); - $this->mustRun('world-heritage:import-site-country-exceptions', [ - '--in' => rtrim($out, '/') . '/exceptions-missing-iso-codes.json', - ]); - - $this->info('Done'); - return self::SUCCESS; - } - - private function mustRun(string $command, array $args = []): void - { - $code = Artisan::call($command, $args); - $this->output->write(Artisan::output()); - - if ($code !== 0) { - $this->error("Command failed: {$command}"); - exit($code); - } - } -} diff --git a/src/app/Console/Commands/DumpUnescoWorldHeritageJson.php b/src/app/Console/Commands/DumpUnescoWorldHeritageJson.php index 687eb9d..4f0d13b 100644 --- a/src/app/Console/Commands/DumpUnescoWorldHeritageJson.php +++ b/src/app/Console/Commands/DumpUnescoWorldHeritageJson.php @@ -487,49 +487,4 @@ private function normalizeRow(array $row): array 'components_count' => $toInt($row['components_count'] ?? null), ]; } - - private function buildCriteriaFromDumpRow(array $row): array - { - $raw = $row['criteria_txt'] ?? null; - - if (!is_string($raw)) { - return []; - } - $raw = trim($raw); - if ($raw === '') { - return []; - } - - preg_match_all('/\(\s*([ivxlcdm]+)\s*\)/i', $raw, $m1); - $vals = $m1[1] ?? []; - - if (!is_array($vals) || $vals === []) { - preg_match_all('/\b([ivxlcdm]{1,6})\b/i', $raw, $m2); - $vals = $m2[1] ?? []; - } - - if (!is_array($vals) || $vals === []) { - return []; - } - - $out = []; - $seen = []; - - foreach ($vals as $v) { - $v = strtolower(trim((string) $v)); - if ($v === '') { - continue; - } - if (!preg_match('/^[ivxlcdm]+$/', $v)) { - continue; - } - - if (!isset($seen[$v])) { - $seen[$v] = true; - $out[] = $v; - } - } - - return $out; - } } diff --git a/src/app/Console/Commands/ImportCountriesFromSplitFile.php b/src/app/Console/Commands/ImportCountriesFromSplitFile.php index 6fd29fe..6b94d0d 100644 --- a/src/app/Console/Commands/ImportCountriesFromSplitFile.php +++ b/src/app/Console/Commands/ImportCountriesFromSplitFile.php @@ -5,10 +5,12 @@ use Illuminate\Console\Command; use Illuminate\Support\Facades\Config; use Illuminate\Support\Facades\DB; -use Illuminate\Support\Facades\Storage; +use App\Console\Concerns\LoadsJsonRows; class ImportCountriesFromSplitFile extends Command { + use LoadsJsonRows; + /** * The name and signature of the console command. * @@ -69,27 +71,16 @@ public function handle(): int if ($code === '' || strlen($code) !== 3) { $skipped++; if ($strict) { - $this->error( - 'Strict: invalid state_party_code: ' . - json_encode($row, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) - ); + $this->error('Strict: invalid state_party_code: ' . json_encode($row, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)); return self::FAILURE; } continue; } - $nameEn = $this->toNullableString($row['name_en'] ?? null); - $nameJp = $this->toNullableString($row['name_jp'] ?? null); + $nameEn = $this->toNullableString($row['name_en'] ?? null) ?? $code; + $nameJp = $this->toNullableString($row['name_jp'] ?? null) ?? $this->resolveCountryNameJapanese($code); $region = $this->toNullableString($row['region'] ?? null); - if ($nameEn === null) { - $nameEn = $code; - } - - if ($nameJp === null) { - $nameJp = $this->resolveCountryNameJapanese($code); - } - if ($strict && $nameJp === null) { $this->error("Strict: name_jp could not be resolved for state_party_code [{$code}]"); return self::FAILURE; @@ -122,75 +113,22 @@ private function flush(array $rows, bool $dryRun): int return count($rows); } - DB::table('countries')->upsert( - $rows, - ['state_party_code'], - ['name_en', 'name_jp', 'region'] - ); - + DB::table('countries')->upsert($rows, ['state_party_code'], ['name_en', 'name_jp', 'region']); return count($rows); } - private function loadRows(string $path): ?array - { - $raw = @file_get_contents($path); - if ($raw === false) { - return null; - } - - $json = json_decode($raw, true); - if (!is_array($json)) { - return null; - } - - if (array_key_exists('results', $json)) { - return is_array($json['results']) ? $json['results'] : null; - } - - return array_is_list($json) ? $json : null; - } - - private function resolvePath(string $path): string - { - $path = trim($path); - if ($path === '') { - return $path; - } - - if (str_starts_with($path, '/') || preg_match('/^[A-Za-z]:\\\\/', $path) === 1) { - return $path; - } - - $path = ltrim($path, '/'); - - if (str_starts_with($path, 'storage/app/')) { - $path = substr($path, strlen('storage/app/')); - } - - if (str_starts_with($path, 'private/')) { - $path = substr($path, strlen('private/')); - } - - return Storage::disk('local')->path($path); - } - private function toNullableString(mixed $v): ?string { if (!is_string($v)) { return null; } - $s = trim($v); - return $s === '' ? null : $s; } private function resolveCountryNameJapanese(string $iso3): ?string { - $countryNameJa = Config::get('country_ja.alpha3_to_country.' . strtoupper(trim($iso3))); - - return is_string($countryNameJa) && $countryNameJa !== '' - ? $countryNameJa - : null; + $name = Config::get('country_ja.alpha3_to_country.' . strtoupper(trim($iso3))); + return is_string($name) && $name !== '' ? $name : null; } } \ No newline at end of file diff --git a/src/app/Console/Commands/ImportWorldHeritageFromJson.php b/src/app/Console/Commands/ImportWorldHeritageFromJson.php deleted file mode 100644 index fdf534e..0000000 --- a/src/app/Console/Commands/ImportWorldHeritageFromJson.php +++ /dev/null @@ -1,290 +0,0 @@ -option('path'); - $max = (int) $this->option('max'); - $batchSize = max(1, (int) $this->option('batch')); - - $fullPath = $this->resolvePath($path); - - if (!file_exists($fullPath)) { - $this->error("Path not found: {$fullPath}"); - return self::FAILURE; - } - - $files = $this->collectJsonFiles($fullPath); - if ($files === []) { - $this->error("No JSON files found: {$fullPath}"); - return self::FAILURE; - } - - $imported = 0; - $skipped = 0; - $batch = []; - $now = Carbon::now(); - - foreach ($files as $filePath) { - if ($max > 0 && $imported >= $max) { - break; - } - - $results = $this->loadResultsFromJsonFile($filePath); - if ($results === null) { - $this->warn("Skipped invalid JSON: {$filePath}"); - continue; - } - - foreach ($results as $row) { - if ($max > 0 && $imported >= $max) { - break; - } - if (!is_array($row)) { $skipped++; continue; } - - $mapped = $this->mapFromUnescoApiRow($row); - - if (empty($mapped['id'])) { $skipped++; continue; } - - $mapped['updated_at'] = $now; - $mapped['created_at'] ??= $now; - - $batch[] = $mapped; - - if (count($batch) >= $batchSize) { - $imported += $this->flushBatch($batch); - $batch = []; - } - } - } - - if ($batch !== []) { - $imported += $this->flushBatch($batch); - } - - $this->info("Imported/updated {$imported} records. Skipped {$skipped} items."); - return self::SUCCESS; - } - - private function resolvePath(string $path): string - { - if ($path !== '' && ($path[0] === '/' || preg_match('/^[A-Za-z]:\\\\/', $path) === 1)) { - return $path; - } - return base_path($path); - } - - private function collectJsonFiles(string $fullPath): array - { - if (is_file($fullPath)) { - return str_ends_with($fullPath, '.json') ? [$fullPath] : []; - } - - $files = []; - $rii = new \RecursiveIteratorIterator( - new \RecursiveDirectoryIterator($fullPath, \FilesystemIterator::SKIP_DOTS) - ); - - foreach ($rii as $file) { - if ($file->isFile() && str_ends_with($file->getFilename(), '.json')) { - $files[] = $file->getPathname(); - } - } - - sort($files); - return $files; - } - - private function loadResultsFromJsonFile(string $filePath): ?array - { - $raw = @file_get_contents($filePath); - if ($raw === false) { - return null; - } - - $json = json_decode($raw, true); - if (!is_array($json)) { - return null; - } - - if (array_key_exists('results', $json)) { - return is_array($json['results']) ? $json['results'] : null; - } - - return $json; - } - - private function mapFromUnescoApiRow(array $row): array - { - $id = $row['id_no'] ?? null; - $lat = $row['coordinates']['lat'] ?? null; - $lon = $row['coordinates']['lon'] ?? null; - - $countryName = $this->extractCountryName($row); - $statePartyIso3 = $this->extractIso3StateParty($row); - - return [ - 'id' => $this->toNullableInt($id), - 'official_name' => $row['official_name'] ?? null, - 'name' => $row['name_en'] ?? $row['name'] ?? null, - 'region' => $row['region_en'] ?? $row['region'] ?? null, - 'state_party' => $statePartyIso3, - 'study_region' => StudyRegionResolver::resolveFromCountry($countryName)->value, - 'category' => $row['category'] ?? $row['type'] ?? null, - 'criteria' => $row['criteria'] ?? null, - 'year_inscribed' => $this->toNullableInt($row['date_inscribed'] ?? $row['year_inscribed'] ?? null), - 'area_hectares' => $this->toNullableFloat($row['area_hectares'] ?? null), - 'buffer_zone_hectares' => $this->toNullableFloat($row['buffer_zone_hectares'] ?? null), - 'is_endangered' => $this->toNullableBool($row['danger'] ?? $row['is_endangered'] ?? null), - 'latitude' => $this->toNullableFloat($lat), - 'longitude' => $this->toNullableFloat($lon), - 'short_description' => $row['short_description'] ?? $row['description'] ?? null, - 'image_url' => $row['image_url'] ?? null, - 'thumbnail_image_id' => null, - 'unesco_site_url' => $row['url'] ?? null, - ]; - } - - private function flushBatch(array $batch): int - { - $updateColumns = array_values(array_diff(array_keys($batch[0]), ['id'])); - - WorldHeritage::query()->upsert( - $batch, - ['id'], - $updateColumns - ); - - return count($batch); - } - - private function extractIso3StateParty(array $row): ?string - { - $candidates = []; - - foreach (['primary_state_party_code', 'state_party_code', 'iso3', 'iso_code'] as $k) { - if (!empty($row[$k]) && is_string($row[$k])) { - $candidates[] = $row[$k]; - } - } - - foreach (['state_party_codes', 'states_codes'] as $k) { - if (!empty($row[$k]) && is_array($row[$k])) { - $candidates[] = $row[$k][0] ?? null; - } - } - - $states = $row['states'] ?? $row['state_party'] ?? null; - if (is_string($states)) { - $candidates[] = $states; - } - if (is_array($states)) { - $candidates[] = $states[0] ?? null; - } - - foreach ($candidates as $c) { - if (!is_string($c)) { - continue; - } - $c = strtoupper(trim($c)); - if ($c !== '' && preg_match('/^[A-Z]{3}$/', $c)) { - return $c; - } - } - - return null; - } - - private function extractCountryName(array $row): ?string - { - $states = $row['states'] ?? $row['state_party'] ?? null; - - if (is_string($states)) { - $normalized = trim($states); - return $normalized !== '' ? $normalized : null; - } - - if (is_array($states)) { - $first = $states[0] ?? null; - if (is_string($first)) { - $normalized = trim($first); - return $normalized !== '' ? $normalized : null; - } - } - - return null; - } - - private function toNullableInt(mixed $v): ?int - { - if ($v === null || $v === '') { - return null; - } - return is_numeric($v) ? (int) $v : null; - } - - private function toNullableFloat(mixed $value): ?float - { - if ($value === null || $value === '') { - return null; - } - - if (is_string($value)) { - $value = str_replace(',', '', trim($value)); - if ($value === '') { - return null; - } - } - - return is_numeric($value) ? (float) $value : null; - } - - private function toNullableBool(mixed $value): ?bool - { - if ($value === null || $value === '') { - return null; - } - - if (is_bool($value)) { - return $value; - } - - if (is_int($value) || is_float($value)) { - return ((int) $value) === 1; - } - - if (is_string($value)) { - $v = strtolower(trim($value)); - if ($v === '') { - return null; - } - - $true = ['1', 'true', 't', 'yes', 'y', 'on']; - $false = ['0', 'false', 'f', 'no', 'n', 'off']; - - if (in_array($v, $true, true)) { - return true; - } - if (in_array($v, $false, true)) { - return false; - } - } - - return null; - } -} diff --git a/src/app/Console/Commands/ImportWorldHeritageJapaneseNameFromJson.php b/src/app/Console/Commands/ImportWorldHeritageJapaneseNameFromJson.php index 3d3a1c6..53599b4 100644 --- a/src/app/Console/Commands/ImportWorldHeritageJapaneseNameFromJson.php +++ b/src/app/Console/Commands/ImportWorldHeritageJapaneseNameFromJson.php @@ -7,6 +7,7 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Storage; +use Throwable; class ImportWorldHeritageJapaneseNameFromJson extends Command { @@ -94,38 +95,52 @@ public function handle(): int ->whereIn('id', $chunkIds) ->get(); - DB::beginTransaction(); + $upsertRows = []; - try { - foreach ($rows as $row) { - $id = (int) $row->id; + foreach ($rows as $row) { + $id = (int) $row->id; - if ($onlyEmpty) { - $current = is_string($row->name_jp) ? trim($row->name_jp) : ''; - if ($current !== '') { - $skippedAlreadySet++; - continue; - } + if ($onlyEmpty) { + $current = is_string($row->name_jp) ? trim($row->name_jp) : ''; + if ($current !== '') { + $skippedAlreadySet++; + continue; } + } - DB::table('world_heritage_sites') - ->where('id', $id) - ->update([ - 'name_jp' => $map[$id], - 'updated_at' => $now, - ]); + $upsertRows[] = [ + 'id' => $id, + 'name_jp' => $map[$id], + 'updated_at' => $now, + ]; + } - $updated++; + if ($upsertRows !== []) { + DB::beginTransaction(); + try { + $chunkIdList = array_column($upsertRows, 'id'); + $cases = implode(' ', array_map( + fn ($row) => "WHEN {$row['id']} THEN ?", + $upsertRows + )); + $bindings = array_column($upsertRows, 'name_jp'); + $bindings[] = $now; + $placeholders = implode(',', $chunkIdList); + + DB::statement( + "UPDATE world_heritage_sites SET name_jp = CASE id {$cases} END, updated_at = ? WHERE id IN ({$placeholders})", + $bindings + ); + + DB::commit(); + $updated += count($upsertRows); + } catch (Throwable $e) { + DB::rollBack(); + $this->error('Failed while updating chunk: ' . $e->getMessage()); + return self::FAILURE; } - - DB::commit(); - } catch (\Throwable $e) { - DB::rollBack(); - $this->error('Failed while updating chunk: ' . $e->getMessage()); - return self::FAILURE; } } - $this->info( "Done: updated={$updated}, missing=" . count($missing) . ", skipped_already_set={$skippedAlreadySet}, invalid/skipped={$invalid}" diff --git a/src/app/Console/Commands/ImportWorldHeritageSiteCountryExceptionsFromSplitFile.php b/src/app/Console/Commands/ImportWorldHeritageSiteCountryExceptionsFromSplitFile.php index df907ce..0be60ab 100644 --- a/src/app/Console/Commands/ImportWorldHeritageSiteCountryExceptionsFromSplitFile.php +++ b/src/app/Console/Commands/ImportWorldHeritageSiteCountryExceptionsFromSplitFile.php @@ -2,13 +2,16 @@ namespace App\Console\Commands; +use App\Console\Concerns\LoadsJsonRows; use Illuminate\Console\Command; use Illuminate\Support\Facades\DB; -use Illuminate\Support\Facades\Storage; use Carbon\Carbon; class ImportWorldHeritageSiteCountryExceptionsFromSplitFile extends Command { + + use LoadsJsonRows; + /** * The name and signature of the console command. * @@ -60,13 +63,12 @@ public function handle(): int if ($max > 0 && $imported >= $max) { break; } - if (!is_array($row)) { $skipped++; continue; } - - $idNo = $row['id_no'] - ?? $row['world_heritage_site_id'] - ?? $row['site_id'] - ?? null; + if (!is_array($row)) { + $skipped++; + continue; + } + $idNo = $row['id_no'] ?? $row['world_heritage_site_id'] ?? $row['site_id'] ?? null; if (!is_int($idNo) && !(is_string($idNo) && is_numeric($idNo))) { $skipped++; if ($strict) { @@ -87,12 +89,9 @@ public function handle(): int continue; } - if ($strict) { - $existsSite = DB::table('world_heritage_sites')->where('id', $siteId)->exists(); - if (!$existsSite) { - $this->error("Strict: FK missing. world_heritage_sites.id={$siteId} not found"); - return self::FAILURE; - } + if ($strict && !DB::table('world_heritage_sites')->where('id', $siteId)->exists()) { + $this->error("Strict: FK missing. world_heritage_sites.id={$siteId} not found"); + return self::FAILURE; } $batch[] = [ @@ -128,50 +127,6 @@ private function flush(array $rows, bool $dryRun): int ['world_heritage_site_id', 'reason'], ['raw', 'updated_at'] ); - return count($rows); } - - private function loadRows(string $path): ?array - { - $raw = @file_get_contents($path); - if ($raw === false) { - return null; - } - - $json = json_decode($raw, true); - if (!is_array($json)) { - return null; - } - - if (array_key_exists('results', $json)) { - return is_array($json['results']) ? $json['results'] : null; - } - - return array_is_list($json) ? $json : null; - } - - private function resolvePath(string $path): string - { - $path = trim($path); - if ($path === '') { - return $path; - } - - if (str_starts_with($path, '/') || preg_match('/^[A-Za-z]:\\\\/', $path) === 1) { - return $path; - } - - $path = ltrim($path, '/'); - - if (str_starts_with($path, 'storage/app/')) { - $path = substr($path, strlen('storage/app/')); - } - - if (str_starts_with($path, 'private/')) { - $path = substr($path, strlen('private/')); - } - - return Storage::disk('local')->path($path); - } } \ No newline at end of file diff --git a/src/app/Console/Commands/ImportWorldHeritageSiteFromSplitFile.php b/src/app/Console/Commands/ImportWorldHeritageSiteFromSplitFile.php index 6ad8f44..d19e9aa 100644 --- a/src/app/Console/Commands/ImportWorldHeritageSiteFromSplitFile.php +++ b/src/app/Console/Commands/ImportWorldHeritageSiteFromSplitFile.php @@ -2,13 +2,16 @@ namespace App\Console\Commands; +use App\Console\Concerns\LoadsJsonRows; use Carbon\Carbon; use Illuminate\Console\Command; use Illuminate\Support\Facades\DB; -use Illuminate\Support\Facades\Storage; class ImportWorldHeritageSiteFromSplitFile extends Command { + + use LoadsJsonRows; + /** * The name and signature of the console command. * @@ -60,7 +63,10 @@ public function handle(): int if ($max > 0 && $imported >= $max) { break; } - if (!is_array($row)) { $skipped++; continue; } + if (!is_array($row)) { + $skipped++; + continue; + } $id = $row['id'] ?? null; if (!is_int($id) && !(is_string($id) && is_numeric($id))) { @@ -71,10 +77,9 @@ public function handle(): int } continue; } - $id = (int) $id; - $mapped = [ - 'id' => $id, + $batch[] = [ + 'id' => (int) $id, 'official_name' => $this->toNullableString($row['official_name'] ?? null), 'name' => $this->toNullableString($row['name'] ?? null), 'name_jp' => $this->toNullableString($row['name_jp'] ?? null), @@ -97,8 +102,6 @@ public function handle(): int 'updated_at' => $now, ]; - $batch[] = $mapped; - if (count($batch) >= $batchSize) { $imported += $this->flush($batch, $dryRun); $batch = []; @@ -119,57 +122,14 @@ private function flush(array $rows, bool $dryRun): int return count($rows); } - $update = array_values(array_diff(array_keys($rows[0]), ['id', 'created_at'])); - DB::table('world_heritage_sites')->upsert( $rows, ['id'], - $update + array_values(array_diff(array_keys($rows[0]), ['id', 'created_at'])) ); - return count($rows); } - private function loadRows(string $path): ?array - { - $raw = @file_get_contents($path); - if ($raw === false) { - return null; - } - - $json = json_decode($raw, true); - if (!is_array($json)) { - return null; - } - - if (array_key_exists('results', $json)) { - return is_array($json['results']) ? $json['results'] : null; - } - return array_is_list($json) ? $json : null; - } - - private function resolvePath(string $path): string - { - $path = trim($path); - if ($path === '') { - return $path; - } - - if (str_starts_with($path, '/') || preg_match('/^[A-Za-z]:\\\\/', $path) === 1) { - return $path; - } - - $path = ltrim($path, '/'); - if (str_starts_with($path, 'storage/app/')) { - $path = substr($path, strlen('storage/app/')); - } - if (str_starts_with($path, 'private/')) { - $path = substr($path, strlen('private/')); - } - - return Storage::disk('local')->path($path); - } - private function toNullableString(mixed $v): ?string { if (!is_string($v)) { @@ -198,32 +158,23 @@ private function toNullableFloat(mixed $v): ?float return is_numeric($v) ? (float) $v : null; } - private function toNullableBoolInt(mixed $v): ?int + private function toNullableBoolInt(mixed $v): int { if ($v === null || $v === '') { return 0; } - if (is_bool($v)) { return $v ? 1 : 0; } - if (is_int($v) || is_float($v)) { return ((int) $v) === 1 ? 1 : 0; } - if (is_string($v)) { $s = strtolower(trim($v)); - if (in_array($s, ['1', 'true', 't', 'yes', 'y', 'on'], true)) { return 1; } - - if (in_array($s, ['0', 'false', 'f', 'no', 'n', 'off'], true)) { - return 0; - } } - return 0; } } diff --git a/src/app/Console/Commands/ImportWorldHeritageSiteImagesFromJson.php b/src/app/Console/Commands/ImportWorldHeritageSiteImagesFromJson.php index a0daf2c..903b433 100644 --- a/src/app/Console/Commands/ImportWorldHeritageSiteImagesFromJson.php +++ b/src/app/Console/Commands/ImportWorldHeritageSiteImagesFromJson.php @@ -2,9 +2,9 @@ namespace App\Console\Commands; +use App\Console\Concerns\LoadsJsonRows; use Illuminate\Console\Command; use Illuminate\Support\Facades\DB; -use Illuminate\Support\Facades\Storage; use Carbon\Carbon; use RecursiveDirectoryIterator; use RecursiveIteratorIterator; @@ -12,6 +12,9 @@ class ImportWorldHeritageSiteImagesFromJson extends Command { + + use LoadsJsonRows; + /** * The name and signature of the console command. * @@ -35,11 +38,10 @@ class ImportWorldHeritageSiteImagesFromJson extends Command public function handle(): int { $path = (string) $this->option('path'); - $max = (int) $this->option('max'); + $max = (int) $this->option('max'); $batchSize = max(1, (int) $this->option('batch')); $fullPath = $this->resolvePath($path); - if (!file_exists($fullPath)) { $this->error("Path not found: {$fullPath}"); return self::FAILURE; @@ -52,7 +54,7 @@ public function handle(): int } $imported = 0; - $skipped = 0; + $skipped = 0; $batch = []; $now = Carbon::now(); @@ -61,7 +63,7 @@ public function handle(): int break; } - $results = $this->loadResultsFromJsonFile($filePath); + $results = $this->loadRows($filePath); if ($results === null) { $this->warn("Skipped invalid JSON: {$filePath}"); continue; @@ -71,14 +73,19 @@ public function handle(): int if ($max > 0 && $imported >= $max) { break; } - if (!is_array($row)) { $skipped++; continue; } + if (!is_array($row)) { + $skipped++; + continue; + } $mapped = $this->mapRow($row); - if ($mapped === null) { $skipped++; continue; } + if ($mapped === null) { + $skipped++; + continue; + } - $mapped['updated_at'] = $now; $mapped['created_at'] ??= $now; - + $mapped['updated_at'] = $now; $batch[] = $mapped; if (count($batch) >= $batchSize) { @@ -101,12 +108,7 @@ private function mapRow(array $row): ?array $siteId = $row['world_heritage_site_id'] ?? null; $url = $row['url'] ?? null; - if (!is_numeric($siteId)) { - return null; - } - $siteId = (int) $siteId; - - if (!is_string($url)) { + if (!is_numeric($siteId) || !is_string($url)) { return null; } @@ -115,20 +117,15 @@ private function mapRow(array $row): ?array return null; } - $urlHash = hash('sha256', $url); - return [ - 'world_heritage_site_id' => $siteId, + 'world_heritage_site_id' => (int) $siteId, 'url' => $url, - 'url_hash' => $urlHash, + 'url_hash' => hash('sha256', $url), 'sort_order' => isset($row['sort_order']) ? (int) $row['sort_order'] : 0, 'is_primary' => empty($row['is_primary']) ? 0 : 1, ]; } - /** - * @param array> $batch - */ private function flushBatch(array $batch): int { DB::table('world_heritage_site_images')->upsert( @@ -136,37 +133,9 @@ private function flushBatch(array $batch): int ['world_heritage_site_id', 'url_hash'], ['url', 'sort_order', 'is_primary', 'updated_at'] ); - return count($batch); } - private function resolvePath(string $path): string - { - $path = trim($path); - if ($path === '') { - return $path; - } - - if (str_starts_with($path, '/')) { - return $path; - } - if (preg_match('/^[A-Za-z]:\\\\/', $path) === 1) { - return $path; - } - - $path = ltrim($path, '/'); - - if (str_starts_with($path, 'storage/app/')) { - $path = substr($path, strlen('storage/app/')); - } - - if (str_starts_with($path, 'private/')) { - $path = substr($path, strlen('private/')); - } - - return Storage::disk('local')->path($path); - } - private function collectJsonFiles(string $fullPath): array { if (is_file($fullPath)) { @@ -187,23 +156,4 @@ private function collectJsonFiles(string $fullPath): array sort($files); return $files; } - - private function loadResultsFromJsonFile(string $filePath): ?array - { - $raw = @file_get_contents($filePath); - if ($raw === false) { - return null; - } - - $json = json_decode($raw, true); - if (!is_array($json)) { - return null; - } - - if (array_key_exists('results', $json)) { - return is_array($json['results']) ? $json['results'] : null; - } - - return $json; - } } \ No newline at end of file diff --git a/src/app/Console/Commands/SplitCountryJson.php b/src/app/Console/Commands/SplitCountryJson.php deleted file mode 100644 index eccfb5b..0000000 --- a/src/app/Console/Commands/SplitCountryJson.php +++ /dev/null @@ -1,582 +0,0 @@ -option('in')); - $out = trim((string) $this->option('out')); - $sitesOut = trim((string) $this->option('sites-out')); - $exceptionsOut = trim((string) $this->option('exceptions-out')); - - $pretty = (bool) $this->option('pretty'); - $dryRun = (bool) $this->option('dry-run'); - $strict = (bool) $this->option('strict'); - $mergeExisting = (bool) $this->option('merge-existing'); - $clean = (bool) $this->option('clean'); - $exceptionsLimit = max(0, (int)$this->option('exceptions-limit')); - - if ($in === '') { - $this->error('Missing required option: --in'); - return self::FAILURE; - } - - $inPath = $this->resolvePath($in); - if (!file_exists($inPath)) { - $this->error("Input not found: {$inPath}"); - return self::FAILURE; - } - - $files = $this->collectJsonFiles($inPath); - if ($files === []) { - $this->error("No JSON files found in: {$inPath}"); - return self::FAILURE; - } - - $outStoragePath = ltrim($out, '/'); - $sitesOutStoragePath = ltrim($sitesOut, '/'); - $exceptionsOutStoragePath = ltrim($exceptionsOut, '/'); - - if ($clean && !$dryRun) { - foreach ([$outStoragePath, $sitesOutStoragePath, $exceptionsOutStoragePath] as $p) { - if (Storage::disk('local')->exists($p)) { - Storage::disk('local')->delete($p); - $this->warn("Deleted existing output: storage/app/{$p}"); - } - } - } - - $existingJp = []; - if ($mergeExisting && !$clean) { - $existingJp = $this->readExistingCountriesJpMap($outStoragePath); - if ($existingJp !== []) { - $this->info('Loaded existing name_jp entries: ' . count($existingJp)); - } - } - - $normalizer = app(CountryCodeNormalizer::class); - - $countryMap = []; - $siteJudgements = []; - $exceptions = []; - $exceptionsCount = 0; - $inputRows = 0; - $invalidJsonFiles = 0; - $rowsNotObject = 0; - $rowsMissingCodes = 0; - $rowsUnknownCodes = 0; - $unknownSamples = []; - - foreach ($files as $file) { - $raw = @file_get_contents($file); - if ($raw === false) { - $this->warn("Skipped unreadable file: {$file}"); - $invalidJsonFiles++; - continue; - } - - $json = json_decode($raw, true); - if (!is_array($json)) { - $this->warn("Skipped invalid JSON: {$file}"); - $invalidJsonFiles++; - continue; - } - - $rows = $this->extractRows($json); - if ($rows === null) { - $this->warn("Skipped unknown JSON shape (expected {results:[...]} or [...]): {$file}"); - $invalidJsonFiles++; - continue; - } - - foreach ($rows as $row) { - $inputRows++; - - if (!is_array($row)) { - $rowsNotObject++; - continue; - } - - $idNo = $row['id_no'] ?? null; - $nameEn = $row['name_en'] ?? null; - $statesNames = $row['states_names'] ?? null; - $regionCode = $row['region_code'] ?? null; - $codesRaw = $this->normalizeCodeList($row['states'] ?? $row['iso_codes'] ?? null); - $judgement = [ - 'id_no' => is_scalar($idNo) ? (string)$idNo : null, - 'name_en' => is_scalar($nameEn) ? (string)$nameEn : null, - 'region_code' => is_scalar($regionCode) ? (string)$regionCode : null, - 'states_names' => is_array($statesNames) ? $statesNames : null, - 'raw_codes' => $codesRaw !== [] ? $codesRaw : null, - 'iso3_codes' => null, - 'status' => null, - 'message' => null, - ]; - - if ($codesRaw === []) { - $rowsMissingCodes++; - - $judgement['status'] = 'missing'; - $judgement['message'] = 'iso_codes/states missing or empty'; - $siteJudgements[] = $judgement; - - if ($exceptionsLimit > 0 && $exceptionsCount < $exceptionsLimit) { - $exceptions[] = [ - 'file' => $file, - 'exception_type' => 'missing_country_code', - 'id_no' => $judgement['id_no'], - 'name_en' => $judgement['name_en'], - 'region_code' => $judgement['region_code'], - 'states_names' => $judgement['states_names'], - 'iso_codes' => $row['iso_codes'] ?? null, - 'states' => $row['states'] ?? null, - ]; - $exceptionsCount++; - } - - if ($strict) { - $this->error('Strict mode: missing country code row detected.'); - $this->line(json_encode($judgement, JSON_UNESCAPED_UNICODE|JSON_UNESCAPED_SLASHES)); - return self::FAILURE; - } - - continue; - } - - try { - $codes3 = $normalizer->toIso3List($codesRaw); - } catch (InvalidArgumentException $e) { - $rowsUnknownCodes++; - - $judgement['status'] = 'unknown'; - $judgement['message'] = $e->getMessage(); - $siteJudgements[] = $judgement; - - if (count($unknownSamples) < 10) { - $unknownSamples[] = [ - 'file' => $file, - 'input_codes' => $codesRaw, - 'message' => $e->getMessage(), - ]; - } - - if ($exceptionsLimit > 0 && $exceptionsCount < $exceptionsLimit) { - $exceptions[] = [ - 'file' => $file, - 'exception_type' => 'unknown_country_code', - 'id_no' => $judgement['id_no'], - 'name_en' => $judgement['name_en'], - 'region_code' => $judgement['region_code'], - 'states_names' => $judgement['states_names'], - 'raw_codes' => $codesRaw, - 'message' => $e->getMessage(), - ]; - $exceptionsCount++; - } - - if ($strict) { - $this->error('Strict mode: unknown country code detected.'); - $this->line($e->getMessage()); - return self::FAILURE; - } - - continue; - } - - if ($codes3 === []) { - $rowsMissingCodes++; - - $judgement['status'] = 'missing'; - $judgement['message'] = 'empty after normalize'; - $siteJudgements[] = $judgement; - - if ($strict) { - $this->error('Strict mode: empty iso3 after normalize.'); - $this->line(json_encode($judgement, JSON_UNESCAPED_UNICODE|JSON_UNESCAPED_SLASHES)); - return self::FAILURE; - } - - continue; - } - - $judgement['status'] = 'ok'; - $judgement['iso3_codes'] = $codes3; - $siteJudgements[] = $judgement; - $names = $this->normalizeStringList($row['states_names'] ?? null); - $region = $this->normalizeRegionCode($row['region_code'] ?? null); - - if ($names !== [] && count($names) === count($codes3)) { - foreach ($codes3 as $idx => $code) { - $en = trim((string)($names[$idx] ?? '')); - if ($en === '') { - $en = $code; - } - - $this->upsertCountryRow( - countryMap: $countryMap, - code: $code, - nameEn: $en, - existingJp: $existingJp, - region: $region - ); - } - continue; - } - - foreach ($codes3 as $code) { - $this->upsertCountryRow( - countryMap: $countryMap, - code: $code, - nameEn: null, - existingJp: $existingJp, - region: $region - ); - } - } - } - - ksort($countryMap, SORT_STRING); - $countries = array_values($countryMap); - - $this->line('----'); - $this->info('Input files: ' . count($files)); - $this->info("Input rows scanned: {$inputRows}"); - $this->info('Countries extracted (unique state_party_code): ' . count($countries)); - $this->info("Site judgements (rows): " . count($siteJudgements)); - $this->info("Invalid JSON files: {$invalidJsonFiles}"); - $this->info("Rows not object: {$rowsNotObject}"); - $this->info("Rows missing country codes: {$rowsMissingCodes}"); - $this->info("Rows unknown country codes: {$rowsUnknownCodes}"); - $this->info("Exceptions collected: " . count($exceptions)); - - if ($unknownSamples !== []) { - $this->warn('Unknown samples (up to 10):'); - foreach ($unknownSamples as $s) { - $this->line('- ' . json_encode($s, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)); - } - } - - $countriesPayload = [ - 'meta' => [ - 'schema' => 'countries.v1', - 'source' => 'unesco whc001', - 'country_code_standard' => 'alpha-3', - 'generated_at' => now()->toIso8601String(), - 'input' => $in, - 'input_files' => count($files), - 'input_rows_scanned' => $inputRows, - 'countries' => count($countries), - 'merge_existing_name_jp' => $mergeExisting && !$clean, - 'region_standard' => 'region_code(EUR/AFR/APA/ARB/LAC)', - ], - 'results' => $countries, - ]; - - $sitesPayload = [ - 'meta' => [ - 'schema' => 'country_codes.v1', - 'source' => 'world-heritage-sites.json', - 'generated_at' => now()->toIso8601String(), - 'input' => $in, - 'input_files' => count($files), - 'input_rows_scanned' => $inputRows, - 'rows' => count($siteJudgements), - 'country_code_standard' => 'alpha-3', - 'null_means' => 'could_not_determine_country', - ], - 'results' => $siteJudgements, - ]; - - $exceptionsPayload = [ - 'meta' => [ - 'schema' => 'exceptions_country_codes.v1', - 'generated_at' => now()->toIso8601String(), - 'input' => $in, - 'input_files' => count($files), - 'input_rows_scanned' => $inputRows, - 'exceptions' => count($exceptions), - 'limit' => $exceptionsLimit, - ], - 'results' => $exceptions, - ]; - - $countriesJson = $this->encodeJson($countriesPayload, $pretty); - $sitesJson = $this->encodeJson($sitesPayload, $pretty); - $exceptionsJson = $this->encodeJson($exceptionsPayload, $pretty); - - if ($countriesJson === null || $sitesJson === null || $exceptionsJson === null) { - $this->error('Failed to encode output JSON'); - return self::FAILURE; - } - - if ($dryRun) { - $this->warn("[dry] would write: storage/app/{$outStoragePath}"); - $this->warn("[dry] would write: storage/app/{$sitesOutStoragePath}"); - $this->warn("[dry] would write: storage/app/{$exceptionsOutStoragePath}"); - return self::SUCCESS; - } - - Storage::disk('local')->put($outStoragePath, $countriesJson); - Storage::disk('local')->put($sitesOutStoragePath, $sitesJson); - Storage::disk('local')->put($exceptionsOutStoragePath, $exceptionsJson); - - $this->info("Wrote: storage/app/{$outStoragePath}"); - $this->info("Wrote: storage/app/{$sitesOutStoragePath}"); - $this->info("Wrote: storage/app/{$exceptionsOutStoragePath}"); - - return self::SUCCESS; - } - - private function upsertCountryRow(array &$countryMap, string $code, ?string $nameEn, array $existingJp, ?string $region): void - { - $code = strtoupper(trim($code)); - if ($code === '') { - return; - } - - if (!isset($countryMap[$code])) { - $countryMap[$code] = [ - 'state_party_code' => $code, - 'name_en' => ($nameEn !== null && trim($nameEn) !== '') ? $nameEn : $code, - 'name_jp' => $existingJp[$code] ?? null, - 'region' => $region, - ]; - return; - } - - if ($nameEn !== null) { - $nameEn = trim($nameEn); - if ($nameEn !== '') { - $current = (string)($countryMap[$code]['name_en'] ?? $code); - if ($current === $code) { - $countryMap[$code]['name_en'] = $nameEn; - } - } - } - - if (($countryMap[$code]['region'] ?? null) === null && $region !== null) { - $countryMap[$code]['region'] = $region; - } - - if (($countryMap[$code]['name_jp'] ?? null) === null && isset($existingJp[$code])) { - $countryMap[$code]['name_jp'] = $existingJp[$code]; - } - } - - private function normalizeRegionCode(mixed $v): ?string - { - if (!is_string($v)) { - return null; - } - - $code = strtoupper(trim($v)); - if ($code === '') { - return null; - } - - $allowed = ['EUR', 'AFR', 'APA', 'ARB', 'LAC']; - return in_array($code, $allowed, true) ? $code : null; - } - - private function collectJsonFiles(string $path): array - { - if (is_file($path)) { - return str_ends_with($path, '.json') ? [$path] : []; - } - - if (!is_dir($path)) { - return []; - } - - $files = []; - $rii = new \RecursiveIteratorIterator( - new \RecursiveDirectoryIterator($path, \FilesystemIterator::SKIP_DOTS) - ); - - foreach ($rii as $file) { - if ($file->isFile() && str_ends_with($file->getFilename(), '.json')) { - $files[] = $file->getPathname(); - } - } - - sort($files); - return $files; - } - - private function resolvePath(string $path): string - { - $path = trim($path); - if ($path === '') { - return $path; - } - - if (str_starts_with($path, '/')) { - return $path; - } - if (preg_match('/^[A-Za-z]:\\\\/', $path) === 1) { - return $path; - } - - $storageCandidate = storage_path('app/' . ltrim($path, '/')); - if (file_exists($storageCandidate)) { - return $storageCandidate; - } - - return base_path($path); - } - - private function extractRows(array $json): ?array - { - if (array_key_exists('results', $json)) { - return is_array($json['results']) ? $json['results'] : null; - } - - return array_is_list($json) ? $json : null; - } - - private function readExistingCountriesJpMap(string $storageOutPath): array - { - if (!Storage::disk('local')->exists($storageOutPath)) { - return []; - } - - $raw = (string) Storage::disk('local')->get($storageOutPath); - $json = json_decode($raw, true); - if (!is_array($json)) { - return []; - } - - $rows = $this->extractRows($json); - if ($rows === null) { - return []; - } - - $map = []; - foreach ($rows as $row) { - if (!is_array($row)) { - continue; - } - $code = strtoupper(trim((string)($row['state_party_code'] ?? ''))); - if ($code === '') { - continue; - } - - $jp = $row['name_jp'] ?? null; - if (is_string($jp)) { - $jp = trim($jp); - } - if ($jp === '') { - $jp = null; - } - - if ($jp !== null) { - $map[$code] = $jp; - } - } - - ksort($map, SORT_STRING); - return $map; - } - - private function normalizeStringList(mixed $v): array - { - if (!is_array($v)) { - return []; - } - - $out = []; - foreach ($v as $x) { - if (!is_string($x)) { - continue; - } - $x = trim($x); - if ($x === '') { - continue; - } - $out[] = $x; - } - return $this->uniqueList($out); - } - - private function normalizeCodeList(mixed $v): array - { - $out = []; - - if (is_array($v)) { - foreach ($v as $x) { - if (!is_string($x)) { - continue; - } - $x = strtoupper(trim($x)); - if ($x !== '') { - $out[] = $x; - } - } - return $this->uniqueList($out); - } - - if (is_string($v)) { - $s = trim($v); - if ($s === '') { - return []; - } - - $parts = preg_split('/[,\|;\/\s]+/', $s) ?: []; - foreach ($parts as $p) { - $p = strtoupper(trim($p)); - if ($p !== '') { - $out[] = $p; - } - } - return $this->uniqueList($out); - } - - return []; - } - - private function uniqueList(array $list): array - { - $seen = []; - $out = []; - foreach ($list as $v) { - if (isset($seen[$v])) { - continue; - } - $seen[$v] = true; - $out[] = $v; - } - return $out; - } - - private function encodeJson(mixed $payload, bool $pretty): ?string - { - $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES; - if ($pretty) { - $flags |= JSON_PRETTY_PRINT; - } - - $json = json_encode($payload, $flags); - return $json === false ? null : $json; - } -} diff --git a/src/app/Console/Commands/SplitWorldHeritageImageJson.php b/src/app/Console/Commands/SplitWorldHeritageImageJson.php deleted file mode 100644 index 690c9a2..0000000 --- a/src/app/Console/Commands/SplitWorldHeritageImageJson.php +++ /dev/null @@ -1,226 +0,0 @@ -option('in')); - $out = trim((string)$this->option('out')); - $pretty = (bool)$this->option('pretty'); - $dryRun = (bool)$this->option('dry-run'); - - if ($in === '') { - $this->error('Missing required option: --in'); - return self::FAILURE; - } - - $inPath = $this->resolvePathToFile($in); - if (!is_file($inPath)) { - $this->error("Input JSON not found: {$inPath}"); - return self::FAILURE; - } - - $raw = @file_get_contents($inPath); - if ($raw === false) { - $this->error("Failed to read input file: {$inPath}"); - return self::FAILURE; - } - - $json = json_decode($raw, true); - if (!is_array($json)) { - $this->error("Invalid JSON: {$inPath}"); - return self::FAILURE; - } - - $results = $json['results'] ?? null; - if (!is_array($results)) { - $this->error('Invalid raw format: expected {"results":[...]}'); - return self::FAILURE; - } - - $images = []; - $scanned = 0; - $skippedNoId = 0; - $skippedNoImages = 0; - - foreach ($results as $row) { - $scanned++; - if (!is_array($row)) { - $skippedNoId++; - continue; - } - - $idNoRaw = trim((string)($row['id_no'] ?? ($row['id'] ?? ''))); - if ($idNoRaw === '' || !is_numeric($idNoRaw)) { - $skippedNoId++; - continue; - } - $siteId = (int)$idNoRaw; - - $urls = $this->extractImageUrlsPreferImagesUrls($row); - if ($urls === []) { - $skippedNoImages++; - continue; - } - - foreach ($urls as $idx => $url) { - $images[] = [ - 'world_heritage_site_id' => $siteId, - 'url' => $url, - 'url_hash' => hash('sha256', $url), - 'sort_order' => $idx, - 'is_primary' => $idx === 0 ? 1 : 0, - ]; - } - } - - $payload = [ - 'meta' => [ - 'schema' => 'world_heritage_site_images.import.v1', - 'source_raw' => $in, - 'generated_at' => now()->toIso8601String(), - 'rows_scanned' => $scanned, - 'images' => count($images), - 'skipped_no_id' => $skippedNoId, - 'skipped_no_images' => $skippedNoImages, - 'target_table' => 'world_heritage_site_images', - 'rule' => 'prefer images_urls; fallback main_image_url.url', - ], - 'results' => $images, - ]; - - $encoded = $this->encodeJson($payload, $pretty); - if ($encoded === null) { - $this->error('Failed to encode output JSON'); - return self::FAILURE; - } - - $outPath = $this->resolvePathToFile($out); - - if ($dryRun) { - $this->info("[dry] would write: {$outPath}"); - $this->info("scanned={$scanned} images=" . count($images) . " skipped_no_id={$skippedNoId} skipped_no_images={$skippedNoImages}"); - return self::SUCCESS; - } - - $dir = dirname($outPath); - if (!is_dir($dir) && (!@mkdir($dir, 0777, true) && !is_dir($dir))) { - $this->error("Failed to create output dir: {$dir}"); - return self::FAILURE; - } - - if (@file_put_contents($outPath, $encoded) === false) { - $this->error("Failed to write: {$outPath}"); - return self::FAILURE; - } - - $this->info("Wrote {$outPath} (" . count($images) . " records)"); - $this->info("scanned={$scanned} skipped_no_id={$skippedNoId} skipped_no_images={$skippedNoImages}"); - return self::SUCCESS; - } - - private function extractImageUrlsPreferImagesUrls(array $row): array - { - $urls = []; - $images = $row['images_urls'] ?? null; - - if (is_array($images) && $images !== []) { - foreach ($images as $p) { - if (!is_string($p)) { - continue; - } - $p = trim($p); - if ($p !== '') { - $urls[] = $p; - } - } - } elseif (is_string($images)) { - $parts = preg_split('/\s*,\s*/', trim($images)) ?: []; - foreach ($parts as $p) { - $p = trim($p); - if ($p !== '') { - $urls[] = $p; - } - } - } - - if ($urls === []) { - $main = $row['main_image_url']['url'] ?? null; - if (is_string($main)) { - $main = trim($main); - if ($main !== '') { - $urls[] = $main; - } - } - } - - $seen = []; - $out = []; - - foreach ($urls as $u) { - if (isset($seen[$u])) { - continue; - } - $seen[$u] = true; - $out[] = $u; - } - - return $out; - } - - private function resolvePathToFile(string $path): string - { - $path = trim($path); - if ($path === '') { - return $path; - } - - if (str_starts_with($path, '/')) { - return $path; - } - if (preg_match('/^[A-Za-z]:\\\\/', $path) === 1) { - return $path; - } - - if (str_starts_with($path, 'storage/app/')) { - $path = substr($path, strlen('storage/app/')); - } - return storage_path('app/' . ltrim($path, '/')); - } - - private function encodeJson(mixed $payload, bool $pretty): ?string - { - $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES; - if ($pretty) { - $flags |= JSON_PRETTY_PRINT; - } - - $json = json_encode($payload, $flags); - return $json === false ? null : $json; - } -} diff --git a/src/app/Console/Concerns/LoadsJsonRows.php b/src/app/Console/Concerns/LoadsJsonRows.php index 9871a01..4f3bb81 100644 --- a/src/app/Console/Concerns/LoadsJsonRows.php +++ b/src/app/Console/Concerns/LoadsJsonRows.php @@ -2,27 +2,11 @@ namespace App\Console\Concerns; +use Illuminate\Support\Facades\Storage; + trait LoadsJsonRows { - protected function resolvePath(string $path): string - { - $path = trim($path); - if ($path === '') { - return $path; - } - - if (str_starts_with($path, '/')) { - return $path; - } - - if (preg_match('/^[A-Za-z]:\\\\/', $path) === 1) { - return $path; - } - - return base_path($path); - } - - protected function loadRows(string $path): ?array + private function loadRows(string $path): ?array { $raw = @file_get_contents($path); if ($raw === false) { @@ -40,4 +24,28 @@ protected function loadRows(string $path): ?array return array_is_list($json) ? $json : null; } + + private function resolvePath(string $path): string + { + $path = trim($path); + if ($path === '') { + return $path; + } + + if (str_starts_with($path, '/') || preg_match('/^[A-Za-z]:\\\\/', $path) === 1) { + return $path; + } + + $path = ltrim($path, '/'); + + if (str_starts_with($path, 'storage/app/')) { + $path = substr($path, strlen('storage/app/')); + } + + if (str_starts_with($path, 'private/')) { + $path = substr($path, strlen('private/')); + } + + return Storage::disk('local')->path($path); + } } \ No newline at end of file diff --git a/src/app/Packages/Domains/Test/QueryService/WorldHeritageQueryService_getByIdTest.php b/src/app/Packages/Domains/Test/QueryService/WorldHeritageQueryService_getByIdTest.php index dd83156..3d1ae23 100644 --- a/src/app/Packages/Domains/Test/QueryService/WorldHeritageQueryService_getByIdTest.php +++ b/src/app/Packages/Domains/Test/QueryService/WorldHeritageQueryService_getByIdTest.php @@ -64,7 +64,7 @@ private function arrayData(): array 'name' => "Ancient and Primeval Beech Forests", 'heritage_name_jp' => "カルパティア山脈とヨーロッパ各地の古代及び原生ブナ林", 'country' => 'Slovakia', - 'region' => 'Europe', + 'study_region' => 'Europe', 'category' => 'Natural', 'criteria' => ['ix'], 'state_party' => null, @@ -149,7 +149,7 @@ public function test_check_data_value(): void $this->assertEquals($this->arrayData()['name'], $result->getName()); $this->assertEquals($this->arrayData()['heritage_name_jp'], $result->getHeritageNameJp()); $this->assertEquals($this->arrayData()['country'], $result->getCountry()); - $this->assertEquals($this->arrayData()['region'], $result->getRegion()); + $this->assertEquals($this->arrayData()['study_region'], $result->getRegion()); $this->assertEquals($this->arrayData()['category'], $result->getCategory()); $this->assertEquals($this->arrayData()['criteria'], $result->getCriteria()); $this->assertEquals($this->arrayData()['state_party'], $result->getStateParty()); diff --git a/src/app/Packages/Domains/WorldHeritageQueryService.php b/src/app/Packages/Domains/WorldHeritageQueryService.php index 67eb801..b2cbce9 100644 --- a/src/app/Packages/Domains/WorldHeritageQueryService.php +++ b/src/app/Packages/Domains/WorldHeritageQueryService.php @@ -170,7 +170,7 @@ public function getHeritageById(int $id): WorldHeritageDto 'heritage_name_jp' => $heritage->name_jp, 'country' => $displayCountry, 'country_name_jp' => $countryNameJp, - 'region' => $heritage->region, + 'region' => $heritage->study_region, 'category' => $heritage->category, 'year_inscribed' => $heritage->year_inscribed, 'latitude' => $heritage->latitude, @@ -199,6 +199,7 @@ public function getHeritagesByIds(array $ids, int $currentPage, int $perPage): P 'name_jp', 'country', 'region', + 'study_region', 'category', 'criteria', 'year_inscribed', @@ -222,15 +223,8 @@ public function getHeritagesByIds(array $ids, int $currentPage, int $perPage): P $thumbnailQuery->select([ 'images.id', 'images.world_heritage_id', - 'disk', 'path', - 'width', - 'height', - 'format', - 'checksum', 'sort_order', - 'alt', - 'credit', ]); }, ]) @@ -283,7 +277,7 @@ public function getHeritagesByIds(array $ids, int $currentPage, int $perPage): P 'name' => $heritage->name, 'name_jp' => $heritage->name_jp, 'country' => $heritage->country, - 'region' => $heritage->region, + 'region' => $heritage->study_region, 'category' => $heritage->category, 'criteria' => $heritage->criteria, 'state_party' => $statePartyName, diff --git a/src/database/seeders/WorldHeritageSeeder.php b/src/database/seeders/WorldHeritageSeeder.php index 3df1cfa..e68ad2e 100644 --- a/src/database/seeders/WorldHeritageSeeder.php +++ b/src/database/seeders/WorldHeritageSeeder.php @@ -21,6 +21,7 @@ public function run(): void 'name_jp' => '姫路城', 'country' => 'Japan', 'region' => 'Asia', + 'study_region' => 'Asia', 'state_party' => 'JPN', // ISO3 を持たせておく 'category' => 'Cultural', 'criteria' => json_encode(['i','iv']), @@ -44,6 +45,7 @@ public function run(): void 'name_jp' => '屋久島', 'country' => 'Japan', 'region' => 'Asia', + 'study_region' => 'Asia', 'state_party' => 'JPN', 'category' => 'Natural', 'criteria' => json_encode(['vii','ix']), @@ -67,6 +69,7 @@ public function run(): void 'name_jp' => '白神山地', 'country' => 'Japan', 'region' => 'Asia', + 'study_region' => 'Asia', 'state_party' => 'JPN', 'category' => 'Natural', 'criteria' => json_encode(['ix','x']), @@ -90,6 +93,7 @@ public function run(): void 'name_jp' => '古都京都の文化財', 'country' => 'Japan', 'region' => 'Asia', + 'study_region' => 'Asia', 'state_party' => 'JPN', 'category' => 'Cultural', 'criteria' => json_encode(['ii','iv']), @@ -113,6 +117,7 @@ public function run(): void 'name_jp' => 'カルパティア山脈とヨーロッパ各地の古代及び原生ブナ林', 'country' => 'Slovakia', 'region' => 'Europe', + 'study_region' => 'Europe', 'state_party' => null, // 越境資産なので primary は pivot から出してる想定 'category' => 'Natural', 'criteria' => json_encode(['ix']), @@ -136,6 +141,7 @@ public function run(): void 'name_jp' => '紀伊山地の霊場と参詣道', 'country' => 'Japan', 'region' => 'Asia', + 'study_region' => 'Asia', 'state_party' => 'JPN', 'category' => 'Cultural', 'criteria' => json_encode(['ii','iii','iv','vi']), @@ -159,6 +165,7 @@ public function run(): void 'name_jp' => '富士山—信仰の対象と芸術の源泉', 'country' => 'Japan', 'region' => 'Asia', + 'study_region' => 'Asia', 'state_party' => 'JPN', 'category' => 'Cultural', 'criteria' => json_encode(['iii','vi']), @@ -182,6 +189,7 @@ public function run(): void 'name_jp' => 'シルクロード：長安－天山回廊の交易路網', 'country' => 'China', 'region' => 'Asia', + 'study_region' => 'Asia', 'state_party' => null, // primary は pivot CHN 'category' => 'Cultural', 'criteria' => json_encode(['ii','iii','vi']), @@ -205,6 +213,7 @@ public function run(): void 'name_jp' => 'シルクロード：ザラフシャン-カラクム回廊', 'country' => 'Tajikistan', 'region' => 'Asia', + 'study_region' => 'Asia', 'state_party' => null, 'category' => 'Cultural', 'criteria' => json_encode(['ii','iii']),