diff --git a/data/datasets/100.json b/data/datasets/100.json index e4df3790..8dc4ec11 100644 --- a/data/datasets/100.json +++ b/data/datasets/100.json @@ -245,13 +245,11 @@ "21": "Migration", "22": "Rivers", "23": "Team Islands", - "24": "Full Random", "25": "Scandinavia", "26": "Mongolia", "27": "Yucatan", "28": "Salt Marsh", "29": "Arena", - "30": "King of the Hill", "31": "Oasis", "32": "Ghost Lake", "33": "Nomad", @@ -266,19 +264,17 @@ "42": "Prairie", "43": "Seasons", "44": "Sherwood Forest", - "45": "Sherwood Heroes", "46": "Shipwreck", "47": "Team Glaciers", - "48": "The Unknown", - "49": "Spain", - "50": "England", + "49": "Iberia", + "50": "Britain", "51": "Mideast", "52": "Texas", "53": "Italy", - "54": "Caribbean", + "54": "Central America", "55": "France", - "56": "Jutland", - "57": "Nippon", + "56": "Norse Lands", + "57": "Sea of Japan (East Sea)", "58": "Byzantium", "67": "Acropolis", "68": "Budapest", @@ -337,8 +333,8 @@ "121": "Jungle Lanes", "122": "Alpine Lakes", "123": "Bogland", - "125": "Ravines", "124": "Mountain Ridge", + "125": "Ravines", "126": "Wolf Hill", "127": "Swirling River", "128": "Twin Forests", @@ -353,16 +349,16 @@ "139": "Golden Swamp", "140": "Four Lakes", "141": "Land Nomad", - "142": "Battle On Ice", - "143": "El Dorado", - "144": "Fall of Axum", - "145": "Fall of Rome", - "146": "Majapahit Empire", + "142": "BR Battle On Ice", + "143": "BR El Dorado", + "144": "BR Fall of Axum", + "145": "BR Fall of Rome", + "146": "BR Majapahit Empire", "147": "Amazon Tunnel", "148": "Coastal Forest", "149": "African Clearing", "150": "Atacama", - "151": "Seize the Mountain", + "151": "Seize The Mountain", "152": "Crater", "153": "Crossroads", "154": "Michi", @@ -396,20 +392,25 @@ "182": "River Divide", "183": "Sandrift", "184": "Shrubland", - "185": "Passage", + "185": "The Passage", "186": "Hollow Woodlands", "187": "Karsts", "188": "Glade", "189": "Fortified Clearing", "190": "QS Arabia", "191": "QS Fortified Clearing", + "192": "QS Glade", "193": "QS Nomad", + "194": "QS Runestones", "195": "QS Arena", + "196": "QS Black Forest", + "197": "Great Wall", + "198": "Border Dispute", "199": "Graupel", "200": "Stranded", - "201": "Forest Breach", - "202": "Border Dispute", - "203": "Graupel", + "201": "Sardis", + "202": "Aquarena", + "203": "Forest Breach", "204": "Chaos Pit", "205": "Mired", "206": "Murkwood", @@ -422,15 +423,13 @@ "213": "Stonefront", "214": "Thames", "215": "Vulpine", - "216": "QS Black Forest", - "217": "Great Wall", "10875": "Arabia", "10876": "Archipelago", "10877": "Baltic", "10878": "Black Forest", "10879": "Coastal", "10880": "Continental", - "10881": "CraterLake", + "10881": "Crater Lake", "10882": "Fortress", "10883": "Gold Rush", "10884": "Highland", @@ -441,73 +440,49 @@ "10889": "Team Islands", "10891": "Scandinavia", "10892": "Mongolia", + "10893": "Salt Marsh", "10894": "Yucatan", - "10893": "SaltMarsh", "10895": "Arena", "10897": "Oasis", "10898": "Ghost Lake", "10901": "Nomad", - "10985": "Canals", - "10986": "Capricious", - "10987": "Dingos", - "10988": "Graveyards", - "10989": "Metropolis", - "10946": "Moats", - "10991": "Paradise Island", - "10992": "Pilgrims", - "10993": "Prairie", - "10994": "Seasons", - "10995": "Sherwood Forest", - "10996": "Sherwood Heroes", - "10997": "Shipwreck", - "10998": "Team Glaciers", - "10999": "The Unknown", - "13544": "Spain", - "13545": "England", - "13546": "Mideast", - "13547": "Texas", - "13548": "Italy", - "13549": "Caribbean", - "13550": "France", - "13551": "Jutland", - "13552": "Nippon", - "13553": "Byzantium", "10914": "Acropolis", "10915": "Budapest", "10916": "Cenotes", "10917": "City of Lakes", - "10918": "Goldenpit", + "10918": "Golden Pit", "10919": "Hideout", "10920": "Hill Fort", "10921": "Lombardia", "10922": "Steppe", "10923": "Valley", - "10924": "Megarandom", + "10924": "MegaRandom", "10925": "Hamburger", - "10926": "CtrRandom", - "10927": "CtrMonsoon", - "10928": "CtrPyramidDescent", - "10929": "CtrSpiral", + "10926": "CtR Random", + "10927": "CtR Monsoon", + "10928": "CtR Pyramid Descent", + "10929": "CtR Spiral", "10930": "Golden Swamp", "10931": "Four Lakes", "10932": "Land Nomad", - "10933": "Battle On The Ice", - "10934": "El Dorado", - "10935": "Fall Of Axum", - "10936": "Fall Of Rome", - "10937": "The Majapahit Empire", - "10938": "AmazonTunnel", - "10939": "CoastalForest", + "10933": "BR Battle On Ice", + "10934": "BR El Dorado", + "10935": "BR Fall of Axum", + "10936": "BR Fall of Rome", + "10937": "BR Majapahit Empire", + "10938": "Amazon Tunnel", + "10939": "Coastal Forest", "10940": "African Clearing", "10941": "Atacama", "10942": "Seize The Mountain", "10943": "Crater", "10944": "Crossroads", "10945": "Michi", + "10946": "Team Moats", "10947": "Volcanic Island", "10948": "Acclivity", "10949": "Eruption", - "10950": "FrigidLake", + "10950": "Frigid Lake", "10951": "Greenland", "10952": "Lowland", "10953": "Marketplace", @@ -533,19 +508,33 @@ "10973": "River Divide", "10974": "Sandrift", "10975": "Shrubland", - "10976": "Passage", + "10976": "The Passage", "10977": "Hollow Woodlands", "10978": "Karsts", "10979": "Glade", "10980": "Fortified Clearing", - "10981": "QP Arabia", - "10982": "QP Fortified Clearing", - "10984": "QP Nomad", + "10981": "QS Arabia", + "10982": "QS Fortified Clearing", + "10983": "QS Glade", + "10984": "QS Nomad", + "10985": "Canals", + "10986": "Capricious", + "10987": "Dingos", + "10988": "Graveyards", + "10989": "Metropolis", + "10990": "Moats", + "10991": "Paradise Island", + "10992": "Pilgrims", + "10993": "Prairie", + "10994": "Seasons", + "10995": "Sherwood Forest", + "10997": "Shipwreck", + "10998": "Team Glaciers", "11000": "Stranded", - "11005": "QP Runestones", - "11006": "QP Arena", - "11007": "QP Black Forest", - "11008": "Manchuria", + "11005": "QS Runestones", + "11006": "QS Arena", + "11007": "QS Black Forest", + "11008": "Great Wall", "11009": "Sardis", "11010": "Aquarena", "11011": "Forest Breach", @@ -563,12 +552,16 @@ "11023": "Stonefront", "11024": "Thames", "11025": "Vulpine", - "10946": "Team Moats", - "301001": "Citadel", - "301011": "Inside Out", - "301012": "Land Nomad", - "301015": "MegaRandom", - "301019": "Oasis", + "13544": "Iberia", + "13545": "Britain", + "13546": "Mideast", + "13547": "Texas", + "13548": "Italy", + "13549": "Central America", + "13550": "France", + "13551": "Norse Lands", + "13552": "Sea of Japan (East Sea)", + "13553": "Byzantium", "301100": "Kilimanjaro", "301101": "Mountain Pass", "301102": "Nile Delta", @@ -583,20 +576,20 @@ "301111": "Bohemia", "301112": "Earth", "301113": "Canyons", - "301114": "Archipelago", + "301114": "Enemy Archipelago", "301115": "Enemy Islands", "301116": "Far Out", "301117": "Front Line", "301118": "Inner Circle", "301119": "Motherland", "301120": "Open Plains", - "301121": "Ring Of Water", - "301122": "Snake Pit", + "301121": "Ring of Water", + "301122": "Snakepit", "301123": "The Eye", "301124": "Australia", "301125": "Indochina", "301126": "Indonesia", - "301127": "Malacca", + "301127": "Strait of Malacca", "301128": "Philippines", "301129": "Bog Islands", "301130": "Mangrove Jungle", @@ -610,19 +603,19 @@ "301138": "Jungle Lanes", "301139": "Alpine Lakes", "301141": "Bogland", - "301142": "Mountain Range", + "301142": "Mountain Ridge", "301143": "Ravines", "301144": "Wolf Hill", - "301150": "Swirling River", - "301151": "Twin Forests", - "301152": "Journey South", - "301153": "Snake Forest", - "301154": "Sprawling Streams", "301145": "Antarctica", "301146": "Aral Sea", "301147": "Black Sea", "301148": "Caucasus", - "301149": "Siberia" + "301149": "Siberia", + "301150": "Swirling River", + "301151": "Twin Forests", + "301152": "Journey South", + "301153": "Snake Forest", + "301154": "Sprawling Streams" }, "technologies": { "0": "", @@ -632,7 +625,7 @@ "4": "El Dorado", "5": "Furor Celtica", "6": "Drill", - "7": "Mahouts", + "7": "Citadels", "8": "Town Watch", "9": "Zealotry", "10": "Artillery", @@ -674,7 +667,7 @@ "46": "Devotion", "47": "Chemistry", "48": "Caravan", - "49": "Berserkergang", + "49": "Bogsveigar", "50": "Masonry", "51": "Architecture", "52": "Rocketry", @@ -843,7 +836,7 @@ "215": "Squires", "216": "", "217": "Two-Handed Swordsman", - "218": "Heavy Cav Archer", + "218": "Heavy Cavalry Archer", "219": "Ring Archer Armor", "220": "", "221": "Two-Man Saw", @@ -869,7 +862,7 @@ "241": "", "242": "", "243": "", - "244": "Heavy Demolition Ship", + "244": "Heavy Demo Ship", "245": "", "246": "Fast Fire Ship", "247": "", @@ -1079,7 +1072,7 @@ "451": "", "452": "", "453": "", - "454": "Counterweights", + "454": "Counter- weights", "455": "Detinets", "456": "", "457": "Perfusion", @@ -1131,7 +1124,7 @@ "503": "", "504": "Elite Boyar", "505": "", - "506": "Sultans", + "506": "Grand Trunk Road", "507": "Shatagni", "508": "", "509": "Elite Kamayuk", @@ -1255,7 +1248,7 @@ "627": "Manipur Cavalry", "628": "Chatras", "629": "Paper Money", - "630": "", + "630": "Battle Elephant", "631": "Elite Battle Elephant", "632": "", "633": "", @@ -1382,7 +1375,7 @@ "754": "Burgundian Vineyards", "755": "Flemish Revolution", "756": "First Crusade", - "757": "Scutage", + "757": "Hauberk", "758": "", "759": "", "760": "", @@ -1426,6 +1419,7 @@ "798": "", "799": "", "800": "", + "826": "Elite Urumi Swordsman", "828": "Elite Ratha", "830": "Elite Chakram Thrower", "831": "Medical Corps", @@ -1436,21 +1430,42 @@ "836": "Frontier Guards", "838": "Siege Elephant", "840": "Elite Ghulam", - "843": "Elite Shrivamsha Rider", + "843": "E. Shrivamsha Rider", "875": "Gambesons", "882": "Elite Centurion", "883": "Ballistas", "884": "Comitatenses", "885": "Legionary", - "918": "Elite Composite Bowman", + "902": "Pirotechnia", + "918": "E. Composite Bowman", "920": "Elite Monaspa", "921": "Fereters", "922": "Cilician Fleet", "923": "Svan Towers", "924": "Aznauri Cavalry", + "980": "Heavy Rocket Cart", + "982": "Elite Fire Lancer", + "991": "Elite Iron Pagoda", + "996": "Fortified Bastions", + "997": "Thunderclap Bombs", + "1002": "Elite Liao Dao", + "1006": "Lamellar Armor", + "1007": "Ordo Cavalry", + "1010": "Dragon Ship", "1012": "Transhumance", "1013": "Pastoralism", - "1014": "Domestication" + "1014": "Domestication", + "1033": "Heavy Hei Guang Cavalry", + "1036": "Elite Tiger Cavalry", + "1061": "Tuntian", + "1062": "Ming Guang Armor", + "1064": "Elite White Feather Guard", + "1069": "Bolt Magazine", + "1070": "Coiled Serpent Array", + "1074": "Elite Fire Archer", + "1080": "Red Cliffs Tactics", + "1081": "Sitting Tiger", + "1156": "Jian Swordsman" }, "terrain": { "0": { @@ -2646,7 +2661,7 @@ "634": "Sieur de Metz", "636": "Sieur Bertrand", "637": "Temple of Heaven", - "638": "Duke D'Alen\u00e7on", + "638": "Duke D'Alençon", "639": "Penguin", "640": "La Hire", "642": "Lord de Graville", diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..f68af9e3 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,24 @@ +# Scripts + +## Import game data +The `import_game_data.py` script will read the information that can be read +from the AoE2DE installation files and patch the `data/datasets/100.json` file +with that information. The script will only print the updated JSON to standard +output, it will not actually modify the file itself. + +The script has been written with the help of [uv](https://docs.astral.sh/uv) to +manage depedencies/venv. In brief, to install dependencies, and run the script: + +```bash +# Install dependencies. It assumes you are in the scripts directory. This step is optional +scripts $ uv sync --script import_game_data.py + +# Run the script +scripts $ uv run import_game_data.py + +# To add a dependency, this how you would do it +scripts $ uv add --script import_game_data.py beautifulpackage +``` + +The script should automatically detect the installation location, both on +Windows and Linux, but assumes it is installed through Steam. diff --git a/scripts/import_game_data.py b/scripts/import_game_data.py new file mode 100644 index 00000000..87e1a432 --- /dev/null +++ b/scripts/import_game_data.py @@ -0,0 +1,404 @@ +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "pydantic", +# "vdf", +# ] +# /// + +import csv +import platform +from pathlib import Path +from typing import Annotated, Literal + +import vdf + +from pydantic import BaseModel, ConfigDict, Field + + +class DEMap(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + data_name: str + const_name: str | None = None + name_string_id: int + rollover_string_id: str | int + icon_reference: str + map_icon_filename: str + mappool_icon_filename: str | None = None + script_filename: str + scenario_filename: str | None = None + storage_id: int + style: str + land: bool = False + open: bool = False + random_map: bool = False + death_match: bool = False + regicide: bool = False + king_of_the_hill: bool = False + capture_the_relic: bool = False + sudden_death: bool = False + wonder_race: bool = False + empire_wars: bool = False + mixed: bool = False + defend_the_wonder: bool = False + nomad: bool = False + closed: bool = False + water: bool = False + migration: bool = False + battle_royale: bool = False + prohibited_game_mode: str | None = None + + +class DEMaps(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + default_map_type: int + default_nomad_map_type: int + default_battle_royale_map_type: int + map_list: list[DEMap] + + +class DECivBuilding(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + age_id: Annotated[int, Field(alias="Age ID")] + building_id: Annotated[int, Field(alias="Building ID")] + building_new_column: Annotated[bool, Field(alias="Building in new column")] + building_upgraded_from_id: Annotated[int, Field(alias="Building upgraded from ID")] + draw_node_type: Annotated[str, Field(alias="Draw Node Type")] + help_string_id: Annotated[int, Field(alias="Help String ID")] + link_id: Annotated[int, Field(alias="Link ID")] + link_node_type: Annotated[str, Field(alias="Link Node Type")] + name: Annotated[str, Field(alias="Name")] + name_string_id: Annotated[int, Field(alias="Name String ID")] + node_id: Annotated[int, Field(alias="Node ID")] + node_status: Annotated[str, Field(alias="Node Status")] + node_type: Annotated[str, Field(alias="Node Type")] + picture_index: Annotated[int, Field(alias="Picture Index")] + prerequisite_ids: Annotated[list[int], Field(alias="Prerequisite IDs")] + prerequisite_types: Annotated[list[str], Field(alias="Prerequisite Types")] + trigger_tech_id: Annotated[int, Field(alias="Trigger Tech ID")] + use_type: Annotated[str, Field(alias="Use Type")] + + +class DECivUnit(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + age_id: Annotated[int, Field(alias="Age ID")] + building_id: Annotated[int, Field(alias="Building ID")] + building_new_column: Annotated[bool, Field(alias="Building in new column")] = False + building_upgraded_from_id: Annotated[ + int, Field(alias="Building upgraded from ID") + ] = -1 + draw_node_type: Annotated[str, Field(alias="Draw Node Type")] + help_string_id: Annotated[int, Field(alias="Help String ID")] + link_id: Annotated[int, Field(alias="Link ID")] + link_node_type: Annotated[str, Field(alias="Link Node Type")] + name: Annotated[str, Field(alias="Name")] + name_string_id: Annotated[int, Field(alias="Name String ID")] + node_id: Annotated[int, Field(alias="Node ID")] + node_status: Annotated[str, Field(alias="Node Status")] + node_type: Annotated[str, Field(alias="Node Type")] + picture_index: Annotated[int, Field(alias="Picture Index")] + prerequisite_ids: Annotated[list[int], Field(alias="Prerequisite IDs")] + prerequisite_types: Annotated[list[str], Field(alias="Prerequisite Types")] + trigger_tech_id: Annotated[int, Field(alias="Trigger Tech ID")] + use_type: Annotated[str, Field(alias="Use Type")] + + +class DECivTechTree(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + civ_id: str + civ_techs_buildings: list[DECivBuilding] + civ_techs_units: list[DECivUnit] + + +class DECivTechTrees(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + civs: list[DECivTechTree] + + +class UnitStringIds(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + name: int + description: int + + +class DECivilization(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + internal_name: str + tech_tree_name: str + data_name: str + hud_style: str + tech_tree_image_path: Path | None = None + emblem_image_path: Path | None = None + unique_unit_image_paths: Annotated[list[Path], Field(default_factory=list[Path])] + name_string_id: int + computer_name_string_table_offset: int = -1 + unique_tech_id_1: int = -1 + unique_tech_id_2: int = -1 + unique_unit_line: int = -1 + unique_unit_upgrade_id: int = -1 + unique_unit_id: int = -1 + elite_unique_unit_id: int = -1 + unique_unit_string_ids: Annotated[ + list[UnitStringIds], Field(default_factory=list[UnitStringIds]) + ] + era: Literal["base"] | Literal["antiquity"] + + +class DECivilizations(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + civilization_list: list[DECivilization] + + +class AoCDatasetMeta(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + version: str + name: str + + +class AoCDataset(BaseModel): + model_config = ConfigDict(strict=True, extra="forbid") + + dataset: AoCDatasetMeta + civilizations: dict[str, dict[str, str | int]] # TODO + maps: dict[str, str] + technologies: dict[str, str] + terrain: dict[str, dict[str, str | dict[str, str]]] # TODO + objects: dict[str, str] + + def with_maps(self, maps: DEMaps, key_values: dict[str, str]): + new_maps = { + **{ + str(map.storage_id): key_values.get( + str(map.name_string_id), + map.const_name if map.const_name is not None else map.data_name, + ) + for map in maps.map_list + }, + **{ + str(map.name_string_id): key_values.get( + str(map.name_string_id), + map.const_name if map.const_name is not None else map.data_name, + ) + for map in maps.map_list + }, + } + new_dataset = self.model_copy( + update={ + "maps": { + str(map_name): new_maps[str(map_name)] + for map_name in sorted([int(k) for k in new_maps.keys()]) + } + } + ) + return new_dataset + + def with_civ_techs(self, civs: list[DECivTechTree], key_values: dict[str, str]): + new_techs = { + **self.technologies, + **{ + str(unit.trigger_tech_id): key_values.get( + str(unit.name_string_id), unit.name + ) + for civ in civs + for unit in civ.civ_techs_units + if unit.trigger_tech_id != -1 + }, + **{ + str(unit.node_id): key_values.get(str(unit.name_string_id), unit.name) + for civ in civs + for unit in civ.civ_techs_units + if unit.node_type == "Research" + }, + } + new_dataset = self.model_copy( + update={ + "technologies": { + str(tech_name): new_techs[str(tech_name)] + for tech_name in sorted([int(k) for k in new_techs.keys()]) + } + } + ) + return new_dataset + + +def get_steam_path() -> Path: + if platform.system() == "Windows": + return get_steam_path_windows() + else: + return get_steam_path_linux() + + +def get_steam_path_windows() -> Path: + import winreg + + try: + key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, "SOFTWARE\\Valve\\Steam") + steam_path, _ = winreg.QueryValueEx(key, "SteamPath") + steam_path = Path(steam_path) + winreg.CloseKey(key) + except FileNotFoundError: + raise FileNotFoundError("ERROR: Steam is not installed.") + + if not steam_path.exists(): + raise FileNotFoundError( + "ERROR: The Steam path does not exist or is not accessible." + ) + + return steam_path + + +def get_steam_path_linux() -> Path: + return Path("~/.local/share/Steam/").expanduser() + + +def get_app_path(steam_path: Path, app_id: str) -> Path: + vdf_file_path = steam_path / "steamapps" / "libraryfolders.vdf" + + with vdf_file_path.open("r") as file: + data = file.read() + parsed = vdf.loads(data) + + libraryfolders = parsed.get("libraryfolders", {}) + app_path = None + + for folder in libraryfolders.values(): + if isinstance(folder, dict) and "apps" in folder and app_id in folder["apps"]: + app_path = folder.get("path") + if app_path is None: + continue + app_path = Path(app_path) + if app_path.exists(): + break + + if app_path is None: + raise FileNotFoundError( + f"ERROR: Could not find the installation path for app {app_id}." + ) + + return app_path + + +def get_game_path(steam_path: Path, app_id: str, game_name: str) -> Path: + app_path = get_app_path(steam_path, app_id) + game_path = app_path / "steamapps" / "common" / game_name + + if not game_path.exists(): + raise FileNotFoundError( + f"ERROR: The game path does not exist. Check if the game name '{game_name}' is correct and the game is properly installed." + ) + + return game_path + + +def read_english_strings(resources_dir: Path) -> dict[str, str]: + strings_file = ( + resources_dir / "en" / "strings" / "key-value" / "key-value-strings-utf8.txt" + ) + key_values = {} + with strings_file.open("rt", newline="\r\n") as file: + for line in file: + line = line.replace("\\n", " ") + if line.lstrip().startswith("//") or line.strip() == "": + continue + if "//" in line: + line = line[: line.find("//")] + delimiter = None + for char in line: + if not char.isspace(): + continue + delimiter = char + break + if delimiter is None: + print(line) + raise Exception("Could not find a delimiter") + reader = csv.reader([line], escapechar="\\", delimiter=delimiter) + for row in reader: + if len(row) > 2: + row = [p for p in row if not (p.isspace() or p == "")] + if len(row) > 2: + row = [row[0], " ".join(row[1:])] + if len(row) == 1: + row = [*row, ""] + if len(row) != 2: + print(row) + continue + key, value = row + + if key in key_values and value != key_values[key]: + non_unique = [ + "13170", + "13171", + "200040", + "IDS_EVENT_CHALLENGE_TRAIN_ANY_UNIT_HELP_TEXT", + "120201", + "120202", + "120198", + "120199", + "120200", + "5323", + "6897", + "8084", + "28084", + ] + if key in non_unique: + continue + raise Exception( + f"Key {key} is not unique: {value=} - {key_values[key]=}" + ) + key_values[key] = value.strip().replace('"', "").replace("\n", " ") + + return key_values + + +def read_civ_tech_tree(resources_dir: Path): + civ_json = resources_dir / "_common" / "dat" / "civTechTrees.json" + return DECivTechTrees.model_validate_json(civ_json.read_text(encoding="utf8")) + + +def read_civilizations(resources_dir: Path): + civ_json = resources_dir / "_common" / "dat" / "civilizations.json" + return DECivilizations.model_validate_json(civ_json.read_text(encoding="utf8")) + + +def read_maps(resources_dir: Path): + maps_json = resources_dir / "_common" / "dat" / "maps.json" + return DEMaps.model_validate_json(maps_json.read_text(encoding="utf8")) + + +def load_dataset(): + dataset_file = Path("../data/datasets/100.json") + return AoCDataset.model_validate_json(dataset_file.read_text(encoding="utf8")) + + +def main() -> None: + dataset = load_dataset() + game_dir = get_game_path(get_steam_path(), "813780", "AoE2DE") + resources_dir = game_dir / "resources" + key_values = read_english_strings(resources_dir) + maps = read_maps(resources_dir) + civs = read_civilizations(resources_dir) + tech_tree = read_civ_tech_tree(resources_dir) + + aoe2_civs = { + civ.tech_tree_name for civ in civs.civilization_list if civ.era == "base" + } + filtered_civs = [civ for civ in tech_tree.civs if civ.civ_id in aoe2_civs] + new_dataset = dataset.with_maps(maps, key_values).with_civ_techs( + filtered_civs, key_values + ) + + print(new_dataset.model_dump_json(indent=2)) + + +if __name__ == "__main__": + main()