Skip to content

Commit 2a5bed3

Browse files
happyleavesaochappyleavesaoc
committed
initial code
1 parent 1008bd4 commit 2a5bed3

14 files changed

Lines changed: 4051 additions & 1 deletion

File tree

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
*.sqlite
2+
*.egg-info
3+
__pycache__

‎README.md‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,27 @@
1-
# aoc-metadata
1+
# aoc-reference-data
2+
3+
Database model for AoC [reference data](https://en.wikipedia.org/wiki/Reference_data). Also includes a way to populate the database once created.
4+
5+
## Description
6+
7+
- Datasets: Game edition (The Conquerors, Wololo Kingdoms, etc)
8+
- Civilizations
9+
- Bonuses
10+
- Events: Community events (NAC, ECL, etc)
11+
- Tournaments: Specific brackets belonging to an event
12+
- Rounds
13+
- Series: A set of matches and outcome
14+
- Participants: Team or single individual
15+
- Platforms: Multiplayer platforms (Voobly, VooblyCN, Aoc QQ, etc)
16+
17+
## Schema
18+
19+
![Schema](/docs/schema.png?raw=true)
20+
21+
## Sources
22+
23+
Event (and related) data is pulled from the [challonge](http://challonge.com) API. A list of AoC-related challonge brackets and associated events is maintained in `data/events/events.json`. Tournaments that don't have a challonge bracket are manually defined in `data/events/series.csv`.
24+
25+
Dataset data is generated from game data files. Code not yet included here. Datasets are numbered by their Userpatch mod identifier (0 is undefined, but selected here to represent The Conquerors).
26+
27+
Platform data is manually compiled.

‎aocref/__init__.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
"""AoC Reference Data."""
2+
3+
import os
4+
import pkg_resources
5+
6+
7+
PACKAGE_NAME = 'aocref'
8+
DATA_PATH = 'data'
9+
10+
11+
def get_metadata(filename):
12+
"""Get metadata file path."""
13+
return pkg_resources.resource_filename(PACKAGE_NAME, os.path.join(DATA_PATH, filename))
14+
15+
16+
def list_metadata(path):
17+
"""List metadata at path."""
18+
return pkg_resources.resource_listdir(PACKAGE_NAME, os.path.join(DATA_PATH, path))

‎aocref/__main__.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
"""Initialize reference data."""
2+
3+
import argparse
4+
import os
5+
6+
from sqlalchemy import create_engine
7+
from sqlalchemy.orm import sessionmaker
8+
9+
from aocref.bootstrap import bootstrap
10+
from aocref.model import BASE
11+
12+
13+
def main():
14+
"""Entry point."""
15+
parser = argparse.ArgumentParser()
16+
parser.add_argument('url', default=os.environ.get('MGZ_DB'), help='database url')
17+
args = parser.parse_args()
18+
19+
engine = create_engine(args.url, echo=False)
20+
session = sessionmaker(bind=engine)()
21+
BASE.metadata.create_all(engine)
22+
bootstrap(session)
23+
24+
25+
if __name__ == '__main__':
26+
main()

‎aocref/bootstrap.py‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
"""Bootstrap reference data."""
2+
import os
3+
import json
4+
5+
import requests_cache
6+
7+
from aocref import model, get_metadata, list_metadata
8+
from aocref import challonge
9+
10+
11+
def bootstrap(session):
12+
"""Bootstrap."""
13+
for filename in list_metadata('datasets'):
14+
dataset_id = filename.split('.')[0]
15+
data = json.loads(open(get_metadata(os.path.join('datasets', filename)), 'r').read())
16+
add_dataset(session, dataset_id, data)
17+
18+
challonge_session = requests_cache.CachedSession()
19+
challonge_session.auth = (
20+
os.environ.get('CHALLONGE_USERNAME'),
21+
os.environ.get('CHALLONGE_KEY')
22+
)
23+
24+
for event in challonge.get_events(challonge_session):
25+
add_event(session, event)
26+
27+
for platform in json.loads(open(get_metadata('platforms.json'), 'r').read()):
28+
add_platform(session, platform)
29+
30+
31+
def add_platform(session, data):
32+
"""Add a platform."""
33+
platform = model.Platform(**data)
34+
session.add(platform)
35+
session.commit()
36+
37+
38+
#def add_map(session, data):
39+
# pass
40+
# zr: bool
41+
# version: ??
42+
# UP flags
43+
# custom or builtin
44+
# code
45+
# pack
46+
# aoe2map id
47+
48+
49+
def add_event(session, data):
50+
"""Add an event."""
51+
event = model.Event(id=data['id'], name=data['name'])
52+
session.add(event)
53+
54+
for tournament_data in data['tournaments']:
55+
tournament = model.Tournament(
56+
id=tournament_data['id'],
57+
name=tournament_data['name'],
58+
event=event
59+
)
60+
session.add(tournament)
61+
for round_id, round_data in tournament_data['rounds'].items():
62+
rnd = model.Round(name=round_id, tournament=tournament)
63+
session.add(rnd)
64+
for series_data in round_data:
65+
series = model.Series(id=series_data['id'], played=series_data['played'], round=rnd)
66+
session.add(series)
67+
for participant_data in series_data['participants']:
68+
participant = model.Participant(
69+
name=participant_data['name'],
70+
score=participant_data['score'],
71+
winner=participant_data['winner'],
72+
series=series
73+
)
74+
session.add(participant)
75+
session.commit()
76+
77+
78+
def add_dataset(session, dataset_id, data):
79+
"""Add a dataset."""
80+
dataset = model.Dataset(
81+
id=int(dataset_id),
82+
name=data['dataset']['name']
83+
)
84+
session.add(dataset)
85+
session.commit()
86+
87+
for civilization_id, info in data['civilizations'].items():
88+
civilization = model.Civilization(
89+
id=civilization_id,
90+
dataset=dataset,
91+
name=info['name']
92+
)
93+
session.add(civilization)
94+
for bonus in info['description']['bonuses']:
95+
session.add(model.CivilizationBonus(
96+
civilization_id=civilization_id,
97+
dataset_id=dataset_id,
98+
type='civ',
99+
description=bonus
100+
))
101+
session.add(model.CivilizationBonus(
102+
civilization_id=civilization_id,
103+
dataset_id=dataset_id,
104+
type='team',
105+
description=info['description']['team_bonus']
106+
))
107+
session.commit()

‎aocref/challonge.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
"""Challonge integration."""
2+
3+
import csv
4+
import json
5+
import logging
6+
from collections import defaultdict
7+
8+
import iso8601
9+
10+
from aocref import get_metadata
11+
12+
13+
LOGGER = logging.getLogger(__name__)
14+
EVENT_DATA = json.loads(open(get_metadata('events/events.json'), 'r').read())
15+
SERIES_DATA = list(csv.DictReader(filter(lambda row: row[0] not in ['#', '\n'],
16+
open(get_metadata('events/series.csv'), 'r'))))
17+
CHALLONGE_API_URL = 'https://api.challonge.com/v1/tournaments/{}.json'
18+
STATE_PENDING = 'pending'
19+
SCORE_SEPARATOR_1 = ','
20+
SCORE_SEPARATOR_2 = '-'
21+
22+
23+
def _map_participants(participant_data):
24+
"""Map participant IDs to names."""
25+
participants = {}
26+
for participant_wrapper in participant_data:
27+
participant = participant_wrapper['participant']
28+
29+
# Record group stage name, if applicable
30+
if len(participant['group_player_ids']) == 1:
31+
pid = participant['group_player_ids'][0]
32+
participants[pid] = participant['name']
33+
34+
pid = participant['id']
35+
participants[pid] = participant['name']
36+
return participants
37+
38+
39+
def _compute_total_score(scores_csv):
40+
"""Compute total score per particpant.
41+
42+
Some tournament admins report scores per match in a series,
43+
but not all. Therefore, we have to return the lowest common
44+
denominator - total score for the series (challonge "match").
45+
"""
46+
total_score = [0, 0]
47+
for score in scores_csv.split(SCORE_SEPARATOR_1):
48+
parts = score.split(SCORE_SEPARATOR_2)
49+
if len(parts) == 2:
50+
total_score[0] += int(parts[0])
51+
total_score[1] += int(parts[1])
52+
return total_score
53+
54+
55+
def _format_participants(match, participants, total_score):
56+
"""Format participants as list."""
57+
return [{
58+
'id': match[id_field],
59+
'name': participants.get(match[id_field]),
60+
'score': total_score[i],
61+
'winner': match['winner_id'] == match[id_field]
62+
} for i, id_field in enumerate(['player1_id', 'player2_id'])]
63+
64+
65+
def _get_manual_tournament(tournament_id):
66+
"""Get manually-entered data for a tournament."""
67+
rounds = defaultdict(list)
68+
for row in SERIES_DATA:
69+
if row['tournament_id'] == tournament_id:
70+
total_score = _compute_total_score(row['score'])
71+
rounds[row['round_id']].append({
72+
'id': '{}-{}-{}'.format(tournament_id, row['round_id'], row['series_id']),
73+
'played': iso8601.parse_date(row['played']),
74+
'participants': [{
75+
'id': None,
76+
'name': name,
77+
'score': total_score[i],
78+
'winner': max(total_score) == total_score[i]
79+
} for i, name in enumerate([row['p1'], row['p2']])]
80+
})
81+
82+
name = None
83+
for event in EVENT_DATA:
84+
for tournament in event['tournaments']:
85+
if isinstance(tournament, list) and tournament[0] == tournament_id:
86+
name = tournament[1]
87+
88+
if name:
89+
return {
90+
'id': tournament_id,
91+
'name': name,
92+
'rounds': rounds
93+
}
94+
return None
95+
96+
97+
def get_tournament(session, tournament_id):
98+
"""Get round and participant data for a tournament."""
99+
100+
manual = _get_manual_tournament(tournament_id)
101+
if manual:
102+
LOGGER.info("falling back to manual data for %s (no Challonge bracket)", tournament_id)
103+
return manual
104+
105+
data = session.get(CHALLONGE_API_URL.format(tournament_id), params={
106+
'include_matches': 1,
107+
'include_participants': 1
108+
}).json()
109+
110+
tournament = data.get('tournament')
111+
participants = _map_participants(tournament['participants'])
112+
rounds = defaultdict(list)
113+
114+
for match_wrapper in tournament['matches']:
115+
match = match_wrapper.get('match')
116+
if match['state'] != STATE_PENDING:
117+
total_score = _compute_total_score(match['scores_csv'])
118+
round_key = '{};{}'.format(match['round'], match['group_id'] or 0)
119+
rounds[round_key].append({
120+
'id': match['id'],
121+
'participants': _format_participants(match, participants, total_score),
122+
'played': iso8601.parse_date(match['updated_at'])
123+
})
124+
return {
125+
'id': tournament_id,
126+
'name': tournament['name'],
127+
'rounds': rounds
128+
}
129+
130+
131+
def get_events(session):
132+
"""Get event data."""
133+
for event in EVENT_DATA:
134+
tournaments = []
135+
for tournament_id in event['tournaments']:
136+
if isinstance(tournament_id, list):
137+
tournament_id = tournament_id[0]
138+
tournaments.append(get_tournament(session, tournament_id))
139+
yield {
140+
'id': event['id'],
141+
'name': event['name'],
142+
'tournaments': tournaments
143+
}

0 commit comments

Comments
 (0)