include top user agents in daily infra reports

Author: bubelov

src/db/log/migrations/4.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS request_user_agent_date ON request(user_agent, date);

src/db/log/request/blocking_queries.rs

@@ -1,6 +1,9 @@
 use super::schema::{self, Columns};
 use crate::Result;
 use rusqlite::{named_params, Connection};
+use std::time::Instant;
+use time::{Duration, OffsetDateTime};
+use tracing::info;
 
 pub struct InsertArgs {
     pub ip: String,
@@ -115,7 +118,7 @@ pub fn select_daily_infra_report(conn: &Connection) -> Result<DailyInfraReport>
             SELECT COUNT(*), COUNT(DISTINCT {ip})
             FROM {table}
             WHERE {date} >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-24 hours')
-            AND ({user_agent} LIKE 'axios%' OR {user_agent} = 'btcmap.org')
+            AND {user_agent} = 'btcmap.org'
         "#,
         table = schema::TABLE_NAME,
         date = Columns::Date.as_str(),
@@ -130,7 +133,7 @@ pub fn select_daily_infra_report(conn: &Connection) -> Result<DailyInfraReport>
             SELECT COUNT(*), COUNT(DISTINCT {ip})
             FROM {table}
             WHERE {date} >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-24 hours')
-            AND ({user_agent} LIKE 'okhttp%' OR {user_agent} LIKE 'BTC Map Android%')
+            AND {user_agent} LIKE 'BTC Map Android%'
         "#,
         table = schema::TABLE_NAME,
         date = Columns::Date.as_str(),
@@ -167,6 +170,106 @@ pub fn select_daily_infra_report(conn: &Connection) -> Result<DailyInfraReport>
     })
 }
 
+pub struct TopUserAgent {
+    pub user_agent: String,
+    pub count: i64,
+    pub unique_ips: i64,
+}
+
+pub fn select_top_user_agents(conn: &Connection) -> Result<Vec<TopUserAgent>> {
+    let overall_start = Instant::now();
+
+    let since_date = (OffsetDateTime::now_utc() - Duration::hours(24))
+        .format(&time::format_description::well_known::Rfc3339)
+        .unwrap()
+        .to_string();
+
+    let sql_top = format!(
+        r#"
+            SELECT {user_agent}, COUNT(*) as count
+            FROM {table}
+            WHERE {date} >= ?1
+            AND {user_agent} IS NOT NULL
+            GROUP BY {user_agent}
+            ORDER BY count DESC
+            LIMIT 10
+        "#,
+        table = schema::TABLE_NAME,
+        date = Columns::Date.as_str(),
+        user_agent = Columns::UserAgent.as_str(),
+    );
+
+    let top_agents: Vec<(String, i64)> = {
+        let start = Instant::now();
+        let mut stmt = conn.prepare(&sql_top)?;
+        let rows = stmt.query_map([&since_date], |row| Ok((row.get(0)?, row.get(1)?)))?;
+        let result = rows.collect::<Result<Vec<_>, _>>()?;
+        info!(
+            elapsed_ms = start.elapsed().as_millis() as u64,
+            count = result.len(),
+            "top_user_agents: first query (get top 10 user agents)"
+        );
+        result
+    };
+
+    if top_agents.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let placeholders: Vec<String> = top_agents.iter().map(|_| "?".to_string()).collect();
+    let sql_unique = format!(
+        r#"
+            SELECT {user_agent}, COUNT(DISTINCT {ip}) as unique_ips
+            FROM {table}
+            WHERE {date} >= ?1
+            AND {user_agent} IN ({})
+            GROUP BY {user_agent}
+        "#,
+        placeholders.join(", "),
+        table = schema::TABLE_NAME,
+        date = Columns::Date.as_str(),
+        user_agent = Columns::UserAgent.as_str(),
+        ip = Columns::Ip.as_str(),
+    );
+
+    let unique_ips: std::collections::HashMap<String, i64> = {
+        let start = Instant::now();
+        let mut stmt = conn.prepare(&sql_unique)?;
+        let mut params: Vec<&dyn rusqlite::ToSql> = vec![&since_date as &dyn rusqlite::ToSql];
+        params.extend(top_agents.iter().map(|(ua, _)| ua as &dyn rusqlite::ToSql));
+        let rows = stmt.query_map(params.as_slice(), |row| {
+            Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
+        })?;
+        let result = rows
+            .collect::<Result<Vec<(String, i64)>, _>>()?
+            .into_iter()
+            .collect::<std::collections::HashMap<_, _>>();
+        info!(
+            elapsed_ms = start.elapsed().as_millis() as u64,
+            count = result.len(),
+            "top_user_agents: second query (get unique IPs for {} agents)",
+            top_agents.len()
+        );
+        result
+    };
+
+    let result = top_agents
+        .into_iter()
+        .map(|(user_agent, count)| TopUserAgent {
+            user_agent: user_agent.clone(),
+            count,
+            unique_ips: *unique_ips.get(&user_agent).unwrap_or(&0),
+        })
+        .collect();
+
+    info!(
+        elapsed_ms = overall_start.elapsed().as_millis() as u64,
+        "top_user_agents: total"
+    );
+
+    Ok(result)
+}
+
 #[cfg(test)]
 mod test {
     use crate::db::log::request::blocking_queries::InsertArgs;

src/db/log/request/queries.rs

@@ -1,6 +1,6 @@
 use super::super::LogPool;
 use super::blocking_queries;
-use super::blocking_queries::{DailyInfraReport, InsertArgs};
+use super::blocking_queries::{DailyInfraReport, InsertArgs, TopUserAgent};
 use crate::db::log::request::schema::Request;
 use crate::Result;
 
@@ -25,3 +25,10 @@ pub async fn select_daily_infra_report(pool: &LogPool) -> Result<DailyInfraRepor
         .interact(move |conn| blocking_queries::select_daily_infra_report(conn))
         .await?
 }
+
+pub async fn select_top_user_agents(pool: &LogPool) -> Result<Vec<TopUserAgent>> {
+    pool.get()
+        .await?
+        .interact(|conn| blocking_queries::select_top_user_agents(conn))
+        .await?
+}

src/rpc/log/get_daily_infra_report.rs

@@ -10,6 +10,7 @@ pub struct Res {
     pub web: PlatformStats,
     pub android: PlatformStats,
     pub ios: PlatformStats,
+    pub top_user_agents: Vec<TopUserAgent>,
 }
 
 #[derive(Serialize)]
@@ -18,8 +19,16 @@ pub struct PlatformStats {
     pub unique_ips: i64,
 }
 
+#[derive(Serialize)]
+pub struct TopUserAgent {
+    pub user_agent: String,
+    pub count: i64,
+    pub unique_ips: i64,
+}
+
 pub async fn run(pool: &LogPool) -> Result<Res> {
     let report = queries::select_daily_infra_report(pool).await?;
+    let top_user_agents = queries::select_top_user_agents(pool).await?;
     Ok(Res {
         total_requests: report.total_requests,
         unique_ips: report.unique_ips,
@@ -35,5 +44,13 @@ pub async fn run(pool: &LogPool) -> Result<Res> {
             requests: report.ios_requests,
             unique_ips: report.ios_unique_ips,
         },
+        top_user_agents: top_user_agents
+            .into_iter()
+            .map(|ua| TopUserAgent {
+                user_agent: ua.user_agent,
+                count: ua.count,
+                unique_ips: ua.unique_ips,
+            })
+            .collect(),
     })
 }