Skip to content

Commit 057bbc9

Browse files
committed
feat(gateway): unblock Twitter/X event ingestion
Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com>
1 parent 38090c0 commit 057bbc9

15 files changed

Lines changed: 1098 additions & 4 deletions

File tree

devops/docker/compose/.env.example

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,13 @@
6363
# TROGON_SOURCE_TELEGRAM_STREAM_MAX_AGE_SECS=604800
6464
# TROGON_SOURCE_TELEGRAM_NATS_ACK_TIMEOUT_SECS=10
6565

66+
# --- Twitter/X Source ---
67+
# TROGON_SOURCE_TWITTER_CONSUMER_SECRET=
68+
# TROGON_SOURCE_TWITTER_SUBJECT_PREFIX=twitter
69+
# TROGON_SOURCE_TWITTER_STREAM_NAME=TWITTER
70+
# TROGON_SOURCE_TWITTER_STREAM_MAX_AGE_SECS=604800
71+
# TROGON_SOURCE_TWITTER_NATS_ACK_TIMEOUT_SECS=10
72+
6673
# --- Slack Source ---
6774
# TROGON_SOURCE_SLACK_SIGNING_SECRET=
6875
# TROGON_SOURCE_SLACK_SUBJECT_PREFIX=slack

devops/docker/compose/services/trogon-gateway/README.md

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ prefix:
1212
| GitHub | `/github/webhook` | `TROGON_SOURCE_GITHUB_WEBHOOK_SECRET` |
1313
| Slack | `/slack/webhook` | `TROGON_SOURCE_SLACK_SIGNING_SECRET` |
1414
| Telegram | `/telegram/webhook` | `TROGON_SOURCE_TELEGRAM_WEBHOOK_SECRET` |
15+
| Twitter/X | `/twitter/webhook` | `TROGON_SOURCE_TWITTER_CONSUMER_SECRET` |
1516
| GitLab | `/gitlab/webhook` | `TROGON_SOURCE_GITLAB_WEBHOOK_SECRET` |
1617
| incident.io | `/incidentio/webhook` | `TROGON_SOURCE_INCIDENTIO_SIGNING_SECRET` |
1718
| Linear | `/linear/webhook` | `TROGON_SOURCE_LINEAR_WEBHOOK_SECRET` |
@@ -60,6 +61,13 @@ contain only resource IDs rather than full objects. The gateway forwards the raw
6061
verified payload to NATS and leaves any enrichment or reordering to downstream
6162
consumers.
6263

64+
## Twitter/X webhooks
65+
66+
X uses the app consumer secret for both the `GET /twitter/webhook` CRC challenge
67+
response and the `POST /twitter/webhook` `x-twitter-webhooks-signature`
68+
verification. Configure `TROGON_SOURCE_TWITTER_CONSUMER_SECRET` before you
69+
register the webhook URL with X.
70+
6371
## Notion webhooks
6472

6573
Notion signs webhook payloads with the subscription `verification_token`.
@@ -83,7 +91,8 @@ docker compose --profile dev up
8391

8492
This starts ngrok alongside the gateway. Check `docker compose logs ngrok`
8593
for the public URL. Append the source prefix path when configuring each
86-
platform's webhook settings (e.g. `https://<ngrok-url>/github/webhook`).
94+
platform's webhook settings (e.g. `https://<ngrok-url>/github/webhook` or
95+
`https://<ngrok-url>/twitter/webhook`).
8796

8897
## Verify
8998

rsworkspace/Cargo.lock

Lines changed: 21 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

rsworkspace/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ trogon-source-notion = { path = "crates/trogon-source-notion" }
2424
trogon-source-sentry = { path = "crates/trogon-source-sentry" }
2525
trogon-source-slack = { path = "crates/trogon-source-slack" }
2626
trogon-source-telegram = { path = "crates/trogon-source-telegram" }
27+
trogon-source-twitter = { path = "crates/trogon-source-twitter" }
2728
trogon-std = { path = "crates/trogon-std" }
2829

2930
# ACP

rsworkspace/crates/trogon-gateway/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ trogon-source-notion = { workspace = true }
3030
trogon-source-sentry = { workspace = true }
3131
trogon-source-slack = { workspace = true }
3232
trogon-source-telegram = { workspace = true }
33+
trogon-source-twitter = { workspace = true }
3334
trogon-std = { workspace = true, features = ["clap", "telemetry-http"] }
3435

3536
[dev-dependencies]

rsworkspace/crates/trogon-gateway/README.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ All webhook sources share one HTTP port (`TROGON_GATEWAY_PORT`, default `8080`)
3636
| GitHub | `/github/webhook` |
3737
| Slack | `/slack/webhook` |
3838
| Telegram | `/telegram/webhook` |
39+
| Twitter/X | `/twitter/webhook` |
3940
| GitLab | `/gitlab/webhook` |
4041
| Linear | `/linear/webhook` |
4142

@@ -49,6 +50,7 @@ A source is enabled only when its required setting is present:
4950
| Discord | `TROGON_SOURCE_DISCORD_BOT_TOKEN` |
5051
| Slack | `TROGON_SOURCE_SLACK_SIGNING_SECRET` |
5152
| Telegram | `TROGON_SOURCE_TELEGRAM_WEBHOOK_SECRET` |
53+
| Twitter/X | `TROGON_SOURCE_TWITTER_CONSUMER_SECRET` |
5254
| GitLab | `TROGON_SOURCE_GITLAB_WEBHOOK_SECRET` |
5355
| Linear | `TROGON_SOURCE_LINEAR_WEBHOOK_SECRET` |
5456

@@ -69,7 +71,7 @@ NATS auth is resolved in this priority order:
6971
Per-source optional tuning (with defaults):
7072

7173
- `TROGON_SOURCE_<SOURCE>_SUBJECT_PREFIX` (defaults: `github`, `discord`, `slack`, `telegram`, `gitlab`, `linear`)
72-
- `TROGON_SOURCE_<SOURCE>_STREAM_NAME` (defaults: `GITHUB`, `DISCORD`, `SLACK`, `TELEGRAM`, `GITLAB`, `LINEAR`)
74+
- `TROGON_SOURCE_<SOURCE>_STREAM_NAME` (defaults: `GITHUB`, `DISCORD`, `SLACK`, `TELEGRAM`, `TWITTER`, `GITLAB`, `LINEAR`)
7375
- `TROGON_SOURCE_<SOURCE>_STREAM_MAX_AGE_SECS` (default: `604800`)
7476
- `TROGON_SOURCE_<SOURCE>_NATS_ACK_TIMEOUT_SECS` (default: `10`)
7577

@@ -78,6 +80,7 @@ Source-specific extras:
7880
- `TROGON_SOURCE_DISCORD_GATEWAY_INTENTS`
7981
- `TROGON_SOURCE_SLACK_TIMESTAMP_MAX_DRIFT_SECS` (default: `300`)
8082
- `TROGON_SOURCE_LINEAR_TIMESTAMP_TOLERANCE_SECS` (default: `60`, `0` disables tolerance)
83+
- `TROGON_SOURCE_TWITTER_CONSUMER_SECRET` is used for both CRC responses and `x-twitter-webhooks-signature` validation
8184

8285
## Config file shape
8386

@@ -107,6 +110,9 @@ signing_secret = "slack-secret"
107110
[sources.telegram]
108111
webhook_secret = "telegram-secret"
109112

113+
[sources.twitter]
114+
consumer_secret = "twitter-consumer-secret"
115+
110116
[sources.gitlab]
111117
webhook_secret = "gitlab-secret"
112118

rsworkspace/crates/trogon-gateway/src/config.rs

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use trogon_source_notion::NotionVerificationToken;
1818
use trogon_source_sentry::SentryClientSecret;
1919
use trogon_source_slack::config::SlackSigningSecret;
2020
use trogon_source_telegram::config::TelegramWebhookSecret;
21+
use trogon_source_twitter::config::TwitterConsumerSecret;
2122
use trogon_std::{NonZeroDuration, ZeroDuration};
2223

2324
use crate::source_status::SourceStatus;
@@ -198,6 +199,8 @@ struct SourcesConfig {
198199
#[config(nested)]
199200
telegram: TelegramConfig,
200201
#[config(nested)]
202+
twitter: TwitterConfig,
203+
#[config(nested)]
201204
gitlab: GitlabConfig,
202205
#[config(nested)]
203206
incidentio: IncidentioConfig,
@@ -277,6 +280,22 @@ struct TelegramConfig {
277280
nats_ack_timeout_secs: u64,
278281
}
279282

283+
#[derive(Config)]
284+
struct TwitterConfig {
285+
#[config(env = "TROGON_SOURCE_TWITTER_STATUS")]
286+
status: Option<String>,
287+
#[config(env = "TROGON_SOURCE_TWITTER_CONSUMER_SECRET")]
288+
consumer_secret: Option<String>,
289+
#[config(env = "TROGON_SOURCE_TWITTER_SUBJECT_PREFIX", default = "twitter")]
290+
subject_prefix: String,
291+
#[config(env = "TROGON_SOURCE_TWITTER_STREAM_NAME", default = "TWITTER")]
292+
stream_name: String,
293+
#[config(env = "TROGON_SOURCE_TWITTER_STREAM_MAX_AGE_SECS", default = 604_800)]
294+
stream_max_age_secs: u64,
295+
#[config(env = "TROGON_SOURCE_TWITTER_NATS_ACK_TIMEOUT_SECS", default = 10)]
296+
nats_ack_timeout_secs: u64,
297+
}
298+
280299
#[derive(Config)]
281300
struct GitlabConfig {
282301
#[config(env = "TROGON_SOURCE_GITLAB_STATUS")]
@@ -382,6 +401,7 @@ pub struct ResolvedConfig {
382401
pub discord: Option<trogon_source_discord::DiscordConfig>,
383402
pub slack: Option<trogon_source_slack::SlackConfig>,
384403
pub telegram: Option<trogon_source_telegram::TelegramSourceConfig>,
404+
pub twitter: Option<trogon_source_twitter::TwitterConfig>,
385405
pub gitlab: Option<trogon_source_gitlab::GitlabConfig>,
386406
pub incidentio: Option<trogon_source_incidentio::IncidentioConfig>,
387407
pub linear: Option<trogon_source_linear::LinearConfig>,
@@ -395,6 +415,7 @@ impl ResolvedConfig {
395415
|| self.discord.is_some()
396416
|| self.slack.is_some()
397417
|| self.telegram.is_some()
418+
|| self.twitter.is_some()
398419
|| self.gitlab.is_some()
399420
|| self.incidentio.is_some()
400421
|| self.linear.is_some()
@@ -424,6 +445,7 @@ fn resolve(cfg: GatewayConfig, nats_overrides: &NatsArgs) -> Result<ResolvedConf
424445
let discord = resolve_discord(cfg.sources.discord, &mut errors);
425446
let slack = resolve_slack(cfg.sources.slack, &mut errors);
426447
let telegram = resolve_telegram(cfg.sources.telegram, &mut errors);
448+
let twitter = resolve_twitter(cfg.sources.twitter, &mut errors);
427449
let gitlab = resolve_gitlab(cfg.sources.gitlab, &mut errors);
428450
let incidentio = resolve_incidentio(cfg.sources.incidentio, &mut errors);
429451
let linear = resolve_linear(cfg.sources.linear, &mut errors);
@@ -456,6 +478,7 @@ fn resolve(cfg: GatewayConfig, nats_overrides: &NatsArgs) -> Result<ResolvedConf
456478
discord,
457479
slack,
458480
telegram,
481+
twitter,
459482
gitlab,
460483
incidentio,
461484
linear,
@@ -798,6 +821,84 @@ fn resolve_telegram(
798821
})
799822
}
800823

824+
fn resolve_twitter(
825+
section: TwitterConfig,
826+
errors: &mut Vec<ConfigValidationError>,
827+
) -> Option<trogon_source_twitter::TwitterConfig> {
828+
if !resolve_source_status("twitter", section.status.as_deref(), errors) {
829+
return None;
830+
}
831+
832+
let secret_str = section.consumer_secret?;
833+
let consumer_secret = match TwitterConsumerSecret::new(secret_str) {
834+
Ok(secret) => secret,
835+
Err(error) => {
836+
errors.push(ConfigValidationError::invalid(
837+
"twitter",
838+
"consumer_secret",
839+
error,
840+
));
841+
return None;
842+
}
843+
};
844+
845+
let subject_prefix = match NatsToken::new(section.subject_prefix) {
846+
Ok(token) => token,
847+
Err(error) => {
848+
errors.push(ConfigValidationError::invalid_subject_token(
849+
"twitter",
850+
"subject_prefix",
851+
error,
852+
));
853+
return None;
854+
}
855+
};
856+
857+
let stream_name = match NatsToken::new(section.stream_name) {
858+
Ok(token) => token,
859+
Err(error) => {
860+
errors.push(ConfigValidationError::invalid_subject_token(
861+
"twitter",
862+
"stream_name",
863+
error,
864+
));
865+
return None;
866+
}
867+
};
868+
869+
let nats_ack_timeout = match NonZeroDuration::from_secs(section.nats_ack_timeout_secs) {
870+
Ok(duration) => duration,
871+
Err(error) => {
872+
errors.push(ConfigValidationError::invalid(
873+
"twitter",
874+
"nats_ack_timeout_secs",
875+
error,
876+
));
877+
return None;
878+
}
879+
};
880+
881+
let stream_max_age = match StreamMaxAge::from_secs(section.stream_max_age_secs) {
882+
Ok(age) => age,
883+
Err(error) => {
884+
errors.push(ConfigValidationError::invalid(
885+
"twitter",
886+
"stream_max_age_secs",
887+
error,
888+
));
889+
return None;
890+
}
891+
};
892+
893+
Some(trogon_source_twitter::TwitterConfig {
894+
consumer_secret,
895+
subject_prefix,
896+
stream_name,
897+
stream_max_age,
898+
nats_ack_timeout,
899+
})
900+
}
901+
801902
fn resolve_gitlab(
802903
section: GitlabConfig,
803904
errors: &mut Vec<ConfigValidationError>,
@@ -1305,6 +1406,15 @@ webhook_secret = "{secret}"
13051406
)
13061407
}
13071408

1409+
fn twitter_toml(secret: &str) -> String {
1410+
format!(
1411+
r#"
1412+
[sources.twitter]
1413+
consumer_secret = "{secret}"
1414+
"#
1415+
)
1416+
}
1417+
13081418
fn gitlab_toml(secret: &str) -> String {
13091419
format!(
13101420
r#"
@@ -1536,6 +1646,25 @@ webhook_secret = "telegram-webhook-secret"
15361646
assert!(cfg.telegram.is_none());
15371647
}
15381648

1649+
#[test]
1650+
fn twitter_resolves_with_valid_consumer_secret() {
1651+
let f = write_toml(&twitter_toml("twitter-consumer-secret"));
1652+
let cfg = load(Some(f.path())).expect("load failed");
1653+
assert!(cfg.twitter.is_some());
1654+
}
1655+
1656+
#[test]
1657+
fn twitter_disabled_returns_none() {
1658+
let toml = r#"
1659+
[sources.twitter]
1660+
status = "disabled"
1661+
consumer_secret = "twitter-consumer-secret"
1662+
"#;
1663+
let f = write_toml(toml);
1664+
let cfg = load(Some(f.path())).expect("load failed");
1665+
assert!(cfg.twitter.is_none());
1666+
}
1667+
15391668
#[test]
15401669
fn gitlab_resolves_with_valid_secret() {
15411670
let f = write_toml(&gitlab_toml("gitlab-webhook-secret"));

rsworkspace/crates/trogon-gateway/src/http.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,14 @@ where
4646
info!(source = "telegram", "mounted at /telegram");
4747
}
4848

49+
if let Some(ref cfg) = config.twitter {
50+
app = app.nest(
51+
"/twitter",
52+
trogon_source_twitter::router(publisher.clone(), cfg),
53+
);
54+
info!(source = "twitter", "mounted at /twitter");
55+
}
56+
4957
if let Some(ref cfg) = config.gitlab {
5058
app = app.nest(
5159
"/gitlab",
@@ -134,6 +142,9 @@ signing_secret = "slack-secret"
134142
[sources.telegram]
135143
webhook_secret = "tg-secret"
136144
145+
[sources.twitter]
146+
consumer_secret = "twitter-consumer-secret"
147+
137148
[sources.gitlab]
138149
webhook_secret = "gl-secret"
139150

0 commit comments

Comments
 (0)