Skip to content

Commit a3da5d6

Browse files
indrasuhyarindrasuhyar
committed
Add geopolitical conflict & energy price transmission dashboard
1 parent 5b0f09f commit a3da5d6

13 files changed

Lines changed: 2344 additions & 1269 deletions

‎Homepage.py‎

Lines changed: 334 additions & 383 deletions
Large diffs are not rendered by default.

‎config.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[theme]
2+
primaryColor = "#1B4F8A"
3+
backgroundColor = "#DAE8F5"
4+
secondaryBackgroundColor = "#C2D9EE"
5+
textColor = "#1A1A1A"
6+
font = "serif"

‎get_token.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import requests
2+
3+
headers = {"Content-Type": "application/x-www-form-urlencoded"}
4+
data = {
5+
"username": "ims2170@columbia.edu",
6+
"password": "Ms1994ms1994",
7+
"grant_type": "password",
8+
"client_id": "acled",
9+
"scope": "authenticated",
10+
}
11+
r = requests.post("https://acleddata.com/oauth/token", headers=headers, data=data)
12+
token = r.json()["access_token"]
13+
print(token)

‎ingest_acled.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
import pandas as pd
2+
import pandas_gbq
3+
import requests
4+
5+
# ── Config ────────────────────────────────────────────────────────────────────
6+
ACLED_EMAIL = "ims2170@columbia.edu"
7+
ACLED_PASSWORD = "Ms1994ms1994"
8+
9+
PROJECT_ID = "sipa-adv-c-giggling-wombat"
10+
DESTINATION = f"{PROJECT_ID}.giggling_wombat.acled_weekly"
11+
12+
HTTP_OK = 200
13+
PAGE_SIZE = 5000
14+
DATE_RANGE = "2012-01-01|2026-12-31"
15+
ACLED_FIELDS = (
16+
"event_date|country|disorder_type|event_type"
17+
"|actor1|civilian_targeting|fatalities"
18+
)
19+
20+
OIL_COUNTRIES = [
21+
"Russia", "Iran", "Iraq", "Saudi Arabia", "Libya",
22+
"Venezuela", "United Arab Emirates", "Kuwait", "Nigeria", "Algeria",
23+
]
24+
25+
26+
# ── Step 1: Get OAuth token ───────────────────────────────────────────────────
27+
def get_access_token(username, password):
28+
headers = {"Content-Type": "application/x-www-form-urlencoded"}
29+
data = {
30+
"username": username,
31+
"password": password,
32+
"grant_type": "password",
33+
"client_id": "acled",
34+
"scope": "authenticated",
35+
}
36+
r = requests.post(
37+
"https://acleddata.com/oauth/token", headers=headers, data=data
38+
)
39+
if r.status_code == HTTP_OK:
40+
print("Token acquired!")
41+
return r.json()["access_token"]
42+
raise Exception(f"Auth failed: {r.status_code} {r.text}")
43+
44+
45+
# ── Step 2: Fetch ACLED data with pagination ──────────────────────────────────
46+
def fetch_acled(token):
47+
all_data = []
48+
49+
for country in OIL_COUNTRIES:
50+
print(f"Fetching: {country}...")
51+
page = 1
52+
country_total = 0
53+
54+
while True:
55+
params = {
56+
"country": country,
57+
"event_date": DATE_RANGE,
58+
"event_date_where": "BETWEEN",
59+
"fields": ACLED_FIELDS,
60+
"limit": PAGE_SIZE,
61+
"page": page,
62+
}
63+
r = requests.get(
64+
"https://acleddata.com/api/acled/read?_format=json",
65+
headers={
66+
"Authorization": f"Bearer {token}",
67+
"Content-Type": "application/json",
68+
},
69+
params=params,
70+
)
71+
if r.json().get("status") == HTTP_OK:
72+
data = r.json()["data"]
73+
if not data:
74+
break
75+
all_data.extend(data)
76+
country_total += len(data)
77+
print(f" → page {page}: {len(data)} rows")
78+
if len(data) < PAGE_SIZE:
79+
break
80+
page += 1
81+
else:
82+
print(f" → Error: {r.json()}")
83+
break
84+
85+
print(f" → {country} total: {country_total} rows\n")
86+
87+
return pd.DataFrame(all_data)
88+
89+
90+
# ── Step 3: Aggregate to weekly ───────────────────────────────────────────────
91+
def aggregate_weekly(df):
92+
df["event_date"] = pd.to_datetime(df["event_date"])
93+
df["fatalities"] = pd.to_numeric(df["fatalities"], errors="coerce").fillna(0)
94+
df["week"] = df["event_date"].dt.to_period("W").apply(
95+
lambda r: r.start_time
96+
)
97+
98+
weekly = df.groupby([
99+
"week",
100+
"country",
101+
"disorder_type",
102+
"event_type",
103+
"actor1",
104+
"civilian_targeting",
105+
]).agg(
106+
event_count=("event_date", "count"),
107+
total_fatalities=("fatalities", "sum"),
108+
).reset_index()
109+
110+
return weekly
111+
112+
113+
# ── Step 4: Upload to BigQuery ────────────────────────────────────────────────
114+
def upload_to_bq(df):
115+
pandas_gbq.to_gbq(
116+
df,
117+
destination_table=DESTINATION,
118+
project_id=PROJECT_ID,
119+
if_exists="replace",
120+
)
121+
print(f"Done! Table {DESTINATION} updated.")
122+
123+
124+
# ── Main ──────────────────────────────────────────────────────────────────────
125+
def ingest():
126+
print("Getting ACLED access token...")
127+
token = get_access_token(ACLED_EMAIL, ACLED_PASSWORD)
128+
129+
print("Fetching ACLED conflict data...\n")
130+
df = fetch_acled(token)
131+
print(f"Total rows fetched: {len(df)}")
132+
133+
print("\nAggregating to weekly...")
134+
weekly = aggregate_weekly(df)
135+
print(f"Weekly rows: {len(weekly)}")
136+
print(weekly.head(10))
137+
138+
print("\nUploading to BigQuery...")
139+
upload_to_bq(weekly)
140+
141+
142+
if __name__ == "__main__":
143+
ingest()

‎ingest_gasoline.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import pandas as pd
2+
import pandas_gbq
3+
import requests
4+
5+
# ── Config ────────────────────────────────────────────────────────────────────
6+
API_KEY = "qIgxlen05S7xFsozHUuJ4HXned44qT8RF3OewtSv"
7+
PROJECT_ID = "sipa-adv-c-giggling-wombat"
8+
DESTINATION = f"{PROJECT_ID}.giggling_wombat.weekly_gasoline"
9+
10+
# Weekly U.S. Regular All Formulations Retail Gasoline Prices ($/gallon)
11+
GASOLINE_URL = (
12+
"https://api.eia.gov/v2/petroleum/pri/gnd/data/"
13+
f"?api_key={API_KEY}"
14+
"&frequency=weekly"
15+
"&data[0]=value"
16+
"&facets[series][]=EMM_EPM0_PTE_NUS_DPG"
17+
"&sort[0][column]=period"
18+
"&sort[0][direction]=asc"
19+
"&offset=0&length=5000"
20+
)
21+
22+
23+
def fetch_gasoline() -> pd.DataFrame:
24+
print("Fetching gasoline price data from EIA...")
25+
r = requests.get(GASOLINE_URL, timeout=30)
26+
r.raise_for_status()
27+
28+
data = r.json().get("response", {}).get("data", [])
29+
df = pd.DataFrame(data)
30+
31+
if df.empty:
32+
raise ValueError("No gasoline data returned from EIA.")
33+
34+
df["week"] = pd.to_datetime(df["period"], errors="coerce")
35+
df["gasoline_price"] = pd.to_numeric(df["value"], errors="coerce")
36+
37+
df = df.dropna(subset=["week", "gasoline_price"])
38+
df = df[df["week"] >= pd.Timestamp("2012-01-01")]
39+
df = df[["week", "gasoline_price"]].sort_values("week").reset_index(drop=True)
40+
41+
print(f"Fetched {len(df)} rows.")
42+
print(df.tail(5))
43+
return df
44+
45+
46+
def upload_to_bq(df: pd.DataFrame):
47+
print(f"\nUploading to BigQuery: {DESTINATION}...")
48+
pandas_gbq.to_gbq(
49+
df,
50+
destination_table=DESTINATION,
51+
project_id=PROJECT_ID,
52+
if_exists="replace",
53+
)
54+
print("Done!")
55+
56+
57+
def ingest():
58+
df = fetch_gasoline()
59+
upload_to_bq(df)
60+
61+
62+
if __name__ == "__main__":
63+
ingest()

‎ingest_gdelt.py‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import pandas_gbq
2+
3+
# ── Config ────────────────────────────────────────────────────────────────────
4+
PROJECT_ID = "sipa-adv-c-giggling-wombat"
5+
DESTINATION = f"{PROJECT_ID}.giggling_wombat.gdelt_weekly"
6+
7+
# GDELT EventRootCode descriptions for reference:
8+
# 15 = Exhibit military posture
9+
# 16 = Reduce relations
10+
# 17 = Coerce
11+
# 18 = Assault
12+
# 19 = Fight
13+
# 20 = Use unconventional mass violence
14+
15+
# ISO3 country codes for oil-producing countries
16+
# ActionGeo_CountryCode uses ISO2 format in GDELT
17+
OIL_COUNTRY_CODES = {
18+
"RS": "Russia",
19+
"IR": "Iran",
20+
"IZ": "Iraq",
21+
"SA": "Saudi Arabia",
22+
"LY": "Libya",
23+
"VE": "Venezuela",
24+
"AE": "United Arab Emirates",
25+
"KU": "Kuwait",
26+
"NI": "Nigeria",
27+
"AG": "Algeria",
28+
}
29+
30+
# GDELT EventCode descriptions (subset of conflict codes)
31+
EVENT_CODE_MAP = {
32+
"150": "Exhibit military posture",
33+
"151": "Demonstrate military posture",
34+
"152": "Mobilise/increase police power",
35+
"153": "Mobilise armed forces",
36+
"154": "Conduct military exercises",
37+
"155": "Conduct arms build-up",
38+
"160": "Reduce relations",
39+
"170": "Coerce",
40+
"171": "Seize/take possession",
41+
"172": "Conduct hunger strike",
42+
"173": "Conduct strike/boycott",
43+
"174": "Obstruct passage/access",
44+
"175": "Halt negotiations",
45+
"176": "Impose embargo/sanction",
46+
"180": "Assault",
47+
"181": "Physically assault",
48+
"182": "Sexually assault",
49+
"183": "Torture",
50+
"184": "Kill by physical assault",
51+
"185": "Attempt to assassinate",
52+
"186": "Assassinate",
53+
"190": "Use conventional military force",
54+
"191": "Impose blockade",
55+
"192": "Occupy territory",
56+
"193": "Fight with small arms",
57+
"194": "Fight with artillery",
58+
"195": "Employ aerial weapons",
59+
"196": "Violate ceasefire",
60+
"200": "Use unconventional mass violence",
61+
"201": "Engage in mass expulsion",
62+
"202": "Engage in mass killings",
63+
"203": "Engage in ethnic cleansing",
64+
"204": "Use weapons of mass destruction",
65+
}
66+
67+
QUERY = f"""
68+
SELECT
69+
DATE_TRUNC(PARSE_DATE('%Y%m%d', CAST(SQLDATE AS STRING)), WEEK) AS week,
70+
ActionGeo_CountryCode AS country_code,
71+
EventRootCode AS event_root_code,
72+
EventCode AS event_code,
73+
COUNT(*) AS event_count,
74+
SUM(NumMentions) AS total_mentions,
75+
AVG(GoldsteinScale) AS avg_goldstein,
76+
AVG(AvgTone) AS avg_tone
77+
FROM `gdelt-bq.gdeltv2.events`
78+
WHERE SQLDATE BETWEEN 20120101 AND 20261231
79+
AND EventRootCode IN ('15','16','17','18','19','20')
80+
AND ActionGeo_CountryCode IN ({", ".join(f"'{k}'" for k in OIL_COUNTRY_CODES)})
81+
GROUP BY week, country_code, event_root_code, event_code
82+
ORDER BY week
83+
"""
84+
85+
86+
def ingest():
87+
print("Querying GDELT public dataset...")
88+
df = pandas_gbq.read_gbq(QUERY, project_id=PROJECT_ID)
89+
print(f"Fetched {len(df)} rows.")
90+
91+
# Map country code to country name
92+
df["country"] = df["country_code"].map(OIL_COUNTRY_CODES)
93+
94+
# Map event code to description
95+
df["event_code_desc"] = df["event_code"].map(EVENT_CODE_MAP).fillna("Other")
96+
97+
print(df.head(10))
98+
print(f"\nCountries: {df['country'].value_counts().to_dict()}")
99+
print(f"Event root codes: {df['event_root_code'].value_counts().to_dict()}")
100+
101+
print(f"\nUploading to BigQuery: {DESTINATION}...")
102+
pandas_gbq.to_gbq(
103+
df,
104+
destination_table=DESTINATION,
105+
project_id=PROJECT_ID,
106+
if_exists="replace",
107+
)
108+
print(f"Done! Table {DESTINATION} updated.")
109+
110+
111+
if __name__ == "__main__":
112+
ingest()

0 commit comments

Comments
 (0)