chore(web-scraper): bump Retrack component version

Author: azasypkin

components/retrack

@@ -16,6 +16,11 @@ interface RetrackTracker {
   tags: string[];
 }
 
+interface RetrackTrackersPage {
+  items: RetrackTracker[];
+  total: number;
+}
+
 async function getRetrackTracker(page: Page, id: string): Promise<RetrackTracker | null> {
   const response = await page.request.get(`${RETRACK_URL}/api/trackers/${id}`);
   if (response.status() === 404) {
@@ -26,9 +31,11 @@ async function getRetrackTracker(page: Page, id: string): Promise<RetrackTracker
 }
 
 async function listRetrackTrackersByTag(page: Page, tag: string): Promise<RetrackTracker[]> {
-  const response = await page.request.get(`${RETRACK_URL}/api/trackers?tag=${encodeURIComponent(tag)}`);
+  const response = await page.request.get(`${RETRACK_URL}/api/trackers?tag=${encodeURIComponent(tag)}&pageSize=100`);
   expect(response.ok()).toBeTruthy();
-  return (await response.json()) as RetrackTracker[];
+  const trackersPage = (await response.json()) as RetrackTrackersPage;
+  expect(trackersPage.items.length).toBeLessThanOrEqual(trackersPage.total);
+  return trackersPage.items;
 }
 
 test.describe('User termination cleans up Retrack trackers', () => {

components/secutils-docs/static/img/docs/guides/csp/create_step1_empty.png

@@ -6,7 +6,7 @@ use crate::{
 };
 use anyhow::{Context, bail};
 use retrack_types::trackers::{
-    Tracker, TrackerCreateParams, TrackerDataRevision, TrackerDataRevisionImportParams,
+    Page, Tracker, TrackerCreateParams, TrackerDataRevision, TrackerDataRevisionImportParams,
     TrackerDataRevisionImportResult, TrackerDebugParams, TrackerExecutionLog,
     TrackerListRevisionsParams, TrackerUpdateParams,
 };
@@ -21,6 +21,8 @@ pub struct RetrackApi<'a, DR: DnsResolver, ET: EmailTransport> {
 }
 
 impl<'a, DR: DnsResolver, ET: EmailTransport> RetrackApi<'a, DR, ET> {
+    const LIST_TRACKERS_PAGE_SIZE: usize = 100;
+
     /// Creates Retrack API.
     pub fn new(api: &'a Api<DR, ET>) -> Self {
         Self { api }
@@ -31,26 +33,46 @@ impl<'a, DR: DnsResolver, ET: EmailTransport> RetrackApi<'a, DR, ET> {
         &self,
         tags: &[Tag],
     ) -> anyhow::Result<Vec<Tracker>> {
-        // Construct tags query string.
-        let tags_query = prepare_tags(tags)
+        let prepared_tags = prepare_tags(tags);
+        let tags_query = prepared_tags
             .iter()
             .map(|tag| format!("tag={}", urlencoding::encode(tag)))
             .collect::<Vec<_>>()
             .join("&");
-        let endpoint = format!("{}api/trackers?{tags_query}", self.api.config.retrack.host);
+        let mut trackers = vec![];
+        let mut page_index = 0;
+
+        loop {
+            let page_query = format!(
+                "page={page_index}&pageSize={}{}{}",
+                Self::LIST_TRACKERS_PAGE_SIZE,
+                if tags_query.is_empty() { "" } else { "&" },
+                tags_query
+            );
+            let endpoint = format!("{}api/trackers?{page_query}", self.api.config.retrack.host);
+            let response = self
+                .api
+                .network
+                .http_client
+                .get(&endpoint)
+                .send()
+                .await
+                .with_context(|| format!("Cannot query trackers ({page_query})."))?;
+            let mut page = response
+                .json::<Page<Tracker>>()
+                .await
+                .context(format!("Cannot deserialize trackers ({page_query})."))?;
 
-        let response = self
-            .api
-            .network
-            .http_client
-            .get(&endpoint)
-            .send()
-            .await
-            .with_context(|| format!("Cannot query trackers ({tags_query})."))?;
-        response
-            .json()
-            .await
-            .context(format!("Cannot deserialize trackers ({tags_query})."))
+            let total = page.total.max(0) as usize;
+            let is_last_page = page.items.is_empty() || trackers.len() + page.items.len() >= total;
+            trackers.append(&mut page.items);
+            if is_last_page {
+                break;
+            }
+            page_index += 1;
+        }
+
+        Ok(trackers)
     }
 
     /// Retrieves Retrack trackers with the specified IDs.

components/secutils-docs/static/img/docs/guides/web_scraping/detect_resources_step10_tracker_created.png

@@ -598,6 +598,7 @@ mod tests {
     use httpmock::MockServer;
     use insta::assert_debug_snapshot;
     use jsonwebtoken::{EncodingKey, Header, encode};
+    use retrack_types::trackers::{Page, Tracker};
     use serde_json::json;
     use sqlx::PgPool;
     use std::collections::HashSet;
@@ -1127,7 +1128,7 @@ mod tests {
             when.method(httpmock::Method::GET).path("/api/trackers");
             then.status(200)
                 .header("Content-Type", "application/json")
-                .json_body(json!([]));
+                .json_body_obj(&Page::new(Vec::<Tracker>::new(), 0));
         });
         config.retrack.host = Url::parse(&retrack_server.base_url())?;
         let api = mock_api_with_config(pool, config).await?;

components/secutils-docs/static/img/docs/guides/web_scraping/detect_resources_step11_initial.png

@@ -374,6 +374,7 @@ mod tests {
     };
     use actix_web::{body::MessageBody, http::StatusCode};
     use httpmock::MockServer;
+    use retrack_types::trackers::{Page, Tracker};
     use serde_json::{Value, json};
     use sqlx::PgPool;
     use std::{sync::Arc, time::Duration};
@@ -706,13 +707,13 @@ mod tests {
                 .json_body(identity_json(destination_id, destination_email));
         });
         // Retrack tracker-list call made by `clone_data` -> `generate_export`. Returning an
-        // empty array lets clone_data succeed, so the failure mode under test (recovery-link
+        // empty page lets clone_data succeed, so the failure mode under test (recovery-link
         // minting) is the one that actually trips the rollback.
         server.mock(|when, then| {
             when.method(httpmock::Method::GET).path("/api/trackers");
             then.status(200)
                 .header("Content-Type", "application/json")
-                .json_body(json!([]));
+                .json_body_obj(&Page::new(Vec::<Tracker>::new(), 0));
         });
         // Rollback's `terminate` bulk-removes the user's Retrack trackers (none here).
         server.mock(|when, then| {
@@ -782,12 +783,12 @@ mod tests {
                 .json_body(identity_json(destination_id, destination_email));
         });
         // Retrack tracker-list call made by `clone_data` -> `generate_export`. Source user
-        // has no trackers so an empty array is the correct response.
+        // has no trackers so an empty page is the correct response.
         server.mock(|when, then| {
             when.method(httpmock::Method::GET).path("/api/trackers");
             then.status(200)
                 .header("Content-Type", "application/json")
-                .json_body(json!([]));
+                .json_body_obj(&Page::new(Vec::<Tracker>::new(), 0));
         });
         let recovery_mock = server.mock(|when, then| {
             when.method(httpmock::Method::POST)