From 27abb6ab49380ce0451b04a0b467e943e8be521e Mon Sep 17 00:00:00 2001 From: Bill Yang Date: Mon, 13 Jul 2026 23:50:26 -0700 Subject: [PATCH 1/3] Implement session replay enhancements and batch processing improvements - Introduced a stable batch ID generation for session replay events, ensuring consistency during retries. - Enhanced event handling by assigning globally increasing sequence numbers across batches. - Updated the session replay recorder to manage pending batches more effectively, improving event flushing and delivery reliability. - Modified the tracking system to wait for earlier requests before sending identify events, ensuring accurate user identification. - Added tests to verify the stability of batch IDs and sequence number assignments during event processing. --- server/public/script-full.js | 122 +++++++++----- server/public/script.js | 2 +- .../analytics-script/sessionReplay.test.ts | 31 ++++ server/src/analytics-script/sessionReplay.ts | 98 +++++++---- server/src/analytics-script/tracking.test.ts | 22 +++ server/src/analytics-script/tracking.ts | 44 +++-- server/src/analytics-script/types.ts | 2 + .../sessionReplay/recordSessionReplay.test.ts | 16 ++ .../api/sessionReplay/recordSessionReplay.ts | 25 +-- .../src/api/sites/batchImportEvents.test.ts | 104 ++++++++++++ server/src/api/sites/batchImportEvents.ts | 91 ++++++----- server/src/api/sites/createSiteImport.ts | 14 +- server/src/api/sites/deleteSiteImport.ts | 3 - server/src/db/clickhouse/clickhouse.ts | 2 + server/src/index.ts | 6 + server/src/lib/clickhouseLimits.ts | 2 + .../src/services/import/importQuotaManager.ts | 93 ----------- .../import/importStatusManager.test.ts | 86 ++++++++++ .../services/import/importStatusManager.ts | 72 +++++++-- .../import/mappers/simpleAnalytics.ts | 11 +- server/src/services/import/mappers/umami.ts | 7 +- .../replay/sessionReplayIngestService.test.ts | 75 ++++++++- .../replay/sessionReplayIngestService.ts | 40 ++++- .../replay/sessionReplayQueryService.ts | 12 +- .../services/storage/r2StorageService.test.ts | 57 +++++++ .../src/services/storage/r2StorageService.ts | 5 +- .../tracker/botBlocking/botEventQueue.ts | 129 +++++++-------- .../services/tracker/identifyService.test.ts | 70 ++++++++ .../src/services/tracker/identifyService.ts | 69 +++----- .../services/tracker/pageviewQueue.test.ts | 72 +++++++++ server/src/services/tracker/pageviewQueue.ts | 66 +++----- .../tracker/reliableBatchQueue.test.ts | 50 ++++++ .../services/tracker/reliableBatchQueue.ts | 152 ++++++++++++++++++ .../src/services/tracker/trackEvent.test.ts | 20 ++- server/src/services/tracker/trackEvent.ts | 12 +- server/src/types/sessionReplay.ts | 2 + 36 files changed, 1247 insertions(+), 437 deletions(-) create mode 100644 server/src/api/sites/batchImportEvents.test.ts create mode 100644 server/src/lib/clickhouseLimits.ts delete mode 100644 server/src/services/import/importQuotaManager.ts create mode 100644 server/src/services/import/importStatusManager.test.ts create mode 100644 server/src/services/storage/r2StorageService.test.ts create mode 100644 server/src/services/tracker/identifyService.test.ts create mode 100644 server/src/services/tracker/pageviewQueue.test.ts create mode 100644 server/src/services/tracker/reliableBatchQueue.test.ts create mode 100644 server/src/services/tracker/reliableBatchQueue.ts diff --git a/server/public/script-full.js b/server/public/script-full.js index 1d9be10d5..bafe6de0b 100644 --- a/server/public/script-full.js +++ b/server/public/script-full.js @@ -250,6 +250,20 @@ // sessionReplay.ts var SAMPLE_STORAGE_KEY = "rybbit-replay-sampled"; + function createReplayBatchId() { + const bytes = new Uint8Array(16); + if (globalThis.crypto?.getRandomValues) { + globalThis.crypto.getRandomValues(bytes); + } else { + for (let index = 0; index < bytes.length; index++) { + bytes[index] = Math.floor(Math.random() * 256); + } + } + bytes[6] = bytes[6] & 15 | 64; + bytes[8] = bytes[8] & 63 | 128; + const hex = [...bytes].map((byte) => byte.toString(16).padStart(2, "0")); + return `${hex.slice(0, 4).join("")}-${hex.slice(4, 6).join("")}-${hex.slice(6, 8).join("")}-${hex.slice(8, 10).join("")}-${hex.slice(10).join("")}`; + } function shouldSampleSession(sampleRate) { if (sampleRate >= 100) return true; if (sampleRate <= 0) return false; @@ -269,6 +283,9 @@ constructor(config, userId, sendBatch) { this.isRecording = false; this.eventBuffer = []; + this.pendingBatches = []; + this.nextSequence = 0; + this.sendLoop = null; this.config = config; this.userId = userId; this.sendBatch = sendBatch; @@ -384,15 +401,15 @@ } this.isRecording = false; this.clearBatchTimer(); - if (this.eventBuffer.length > 0) { - this.flushEvents(); + if (this.eventBuffer.length > 0 || this.pendingBatches.length > 0) { + void this.flushEvents(); } } isActive() { return this.isRecording; } addEvent(event) { - this.eventBuffer.push(event); + this.eventBuffer.push({ ...event, sequence: this.nextSequence++ }); if (this.eventBuffer.length >= this.config.sessionReplayBatchSize) { this.flushEvents(); } @@ -400,8 +417,8 @@ setupBatchTimer() { this.clearBatchTimer(); this.batchTimer = window.setInterval(() => { - if (this.eventBuffer.length > 0) { - this.flushEvents(); + if (this.eventBuffer.length > 0 || this.pendingBatches.length > 0) { + void this.flushEvents(); } }, this.config.sessionReplayBatchInterval); } @@ -412,26 +429,45 @@ } } async flushEvents() { - if (this.eventBuffer.length === 0) { - return; + if (this.eventBuffer.length > 0) { + const batch = { + batchId: createReplayBatchId(), + userId: this.userId, + events: this.eventBuffer, + metadata: { + pageUrl: window.location.href, + viewportWidth: screen.width, + viewportHeight: screen.height, + language: navigator.language + } + }; + this.eventBuffer = []; + this.pendingBatches.push(batch); } - const events = [...this.eventBuffer]; - this.eventBuffer = []; - const batch = { - userId: this.userId, - events, - metadata: { - pageUrl: window.location.href, - viewportWidth: screen.width, - viewportHeight: screen.height, - language: navigator.language - } - }; - try { - await this.sendBatch(batch); - } catch (error) { - this.eventBuffer.unshift(...events); + await this.drainPendingBatches(); + } + drainPendingBatches() { + if (this.sendLoop) { + const activeLoop = this.sendLoop; + return activeLoop.then(() => { + if (this.pendingBatches.length > 0 && this.sendLoop === null) { + return this.drainPendingBatches(); + } + }); } + this.sendLoop = (async () => { + while (this.pendingBatches.length > 0) { + try { + await this.sendBatch(this.pendingBatches[0]); + this.pendingBatches.shift(); + } catch { + return; + } + } + })().finally(() => { + this.sendLoop = null; + }); + return this.sendLoop; } // Update user ID when it changes updateUserId(userId) { @@ -446,7 +482,7 @@ // Handle page navigation for SPAs onPageChange() { if (this.isRecording) { - this.flushEvents(); + void this.flushEvents(); } } // Cleanup on page unload @@ -600,6 +636,7 @@ var Tracker = class { constructor(config) { this.customUserId = null; + this.pendingTrackingRequests = /* @__PURE__ */ new Set(); this.errorDedupeCache = /* @__PURE__ */ new Map(); this.errorDedupeLastCleanup = 0; this.exposedFeatureFlags = /* @__PURE__ */ new Set(); @@ -738,20 +775,25 @@ } return payload; } - async sendTrackingData(payload) { - try { - await fetch(`${this.config.analyticsHost}/track`, { - method: "POST", - headers: { - "Content-Type": "application/json" - }, - body: JSON.stringify(payload), - mode: "cors", - keepalive: true - }); - } catch (error) { - console.error("Failed to send tracking data:", error); - } + sendTrackingData(payload) { + const request = (async () => { + try { + await fetch(`${this.config.analyticsHost}/track`, { + method: "POST", + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify(payload), + mode: "cors", + keepalive: true + }); + } catch (error) { + console.error("Failed to send tracking data:", error); + } + })(); + this.pendingTrackingRequests.add(request); + void request.finally(() => this.pendingTrackingRequests.delete(request)); + return request; } track(eventType, eventName = "", properties = {}) { if (eventType === "custom_event" && (!eventName || typeof eventName !== "string")) { @@ -931,7 +973,9 @@ } catch (e2) { console.warn("Could not persist user ID to localStorage"); } - void this.sendIdentifyEvent(this.customUserId, traits, true).then(() => this.refreshFeatureFlags()); + const identifiedUserId = this.customUserId; + const earlierTrackingRequests = [...this.pendingTrackingRequests]; + void Promise.allSettled(earlierTrackingRequests).then(() => this.sendIdentifyEvent(identifiedUserId, traits, true)).then(() => this.refreshFeatureFlags()); if (this.sessionReplayRecorder) { this.sessionReplayRecorder.updateUserId(this.customUserId); } diff --git a/server/public/script.js b/server/public/script.js index 0c7b28950..3d25bc97e 100644 --- a/server/public/script.js +++ b/server/public/script.js @@ -1 +1 @@ -"use strict";(()=>{var De=Object.defineProperty;var Ue=(n,e,t)=>e in n?De(n,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):n[e]=t;var R=(n,e,t)=>Ue(n,typeof e!="symbol"?e+"":e,t);function He(n){if(n.startsWith("re:")){let l=n.slice(3);if(!l)throw new Error("Empty regex pattern");return new RegExp(l)}let t="__DOUBLE_ASTERISK_TOKEN__",r="__SINGLE_ASTERISK_TOKEN__",s=n.replace(/\*\*/g,t).replace(/\*/g,r).replace(/[.+?^${}()|[\]\\]/g,"\\$&");s=s.replace(new RegExp(`/${t}/`,"g"),"/(?:.+/)?"),s=s.replace(new RegExp(t,"g"),".*"),s=s.replace(/\//g,"\\/");let a=s.replace(new RegExp(r,"g"),"[^/]+");return new RegExp("^"+a+"$")}function $(n,e){for(let t of e)try{if(He(t).test(n))return t}catch(r){console.error(`Invalid pattern: ${t}`,r)}return null}function ce(n,e){let t=null;return(...r)=>{t&&clearTimeout(t),t=setTimeout(()=>n(...r),e)}}function le(n){try{let e=window.location.hostname,t=new URL(n).hostname;return t!==e&&t!==""}catch{return!1}}function T(n,e){if(!n)return e;try{let t=JSON.parse(n);return Array.isArray(e)&&!Array.isArray(t)?e:t}catch(t){return console.error("Error parsing JSON:",t),e}}function ue(){try{if(crypto?.randomUUID)return crypto.randomUUID()}catch{}return`${Date.now().toString(36)}-${Math.random().toString(36).slice(2,12)}`}function We(n){let e=`${n}-visitor-id`;try{let t=localStorage.getItem(e);if(t)return t;let r=ue();return localStorage.setItem(e,r),r}catch{return ue()}}function Ve(n){try{return localStorage.getItem(`${n}-user-id`)||void 0}catch{return}}function $e(n){return n.hash&&n.hash.startsWith("#/")?n.hash.substring(1):n.pathname}async function ze(n,e,t,r){try{let i=new URL(window.location.href),s=await fetch(`${n}/site/${e}/feature-flags/evaluate`,{method:"POST",headers:{"Content-Type":"application/json"},credentials:"omit",body:JSON.stringify({anonymousId:r,identifiedUserId:Ve(t),hostname:i.hostname,pathname:$e(i),querystring:i.search,query:Object.fromEntries(i.searchParams.entries()),referrer:document.referrer,language:navigator.language,screenWidth:screen.width,screenHeight:screen.height})});if(!s.ok)return{};let a=await s.json();return a?.flags&&typeof a.flags=="object"?a.flags:{}}catch{return{}}}async function de(n){let e=n.getAttribute("src");if(!e)return console.error("Script src attribute is missing"),null;let t=e.split("/script.js")[0];if(!t)return console.error("Please provide a valid analytics host"),null;let r=n.getAttribute("data-site-id")||n.getAttribute("site-id");if(!r)return console.error("Please provide a valid site ID using the data-site-id attribute"),null;let i=n.getAttribute("data-namespace")||"rybbit",s=We(i),a=T(n.getAttribute("data-skip-patterns"),[]),l=T(n.getAttribute("data-mask-patterns"),[]),d=T(n.getAttribute("data-replay-mask-text-selectors"),[]),u=n.getAttribute("data-debounce")?Math.max(0,parseInt(n.getAttribute("data-debounce"))):500,g=n.getAttribute("data-replay-batch-size")?Math.max(1,parseInt(n.getAttribute("data-replay-batch-size"))):250,h=n.getAttribute("data-replay-batch-interval")?Math.max(1e3,parseInt(n.getAttribute("data-replay-batch-interval"))):5e3,C=n.getAttribute("data-replay-block-class")||void 0,m=n.getAttribute("data-replay-block-selector")||void 0,o=n.getAttribute("data-replay-ignore-class")||void 0,c=n.getAttribute("data-replay-ignore-selector")||void 0,p=n.getAttribute("data-replay-mask-text-class")||void 0,w=n.getAttribute("data-replay-mask-all-inputs"),S=w!==null?w!=="false":void 0,E=n.getAttribute("data-replay-mask-input-options"),F=E?T(E,{password:!0,email:!0}):void 0,ne=n.getAttribute("data-replay-collect-fonts"),Me=ne!==null?ne!=="false":void 0,ie=n.getAttribute("data-replay-sampling"),xe=ie?T(ie,{}):void 0,se=n.getAttribute("data-replay-slim-dom-options"),Be=se?T(se,{}):void 0,ae=n.getAttribute("data-replay-sample-rate"),Oe=ae?Math.min(100,Math.max(0,parseInt(ae,10))):void 0,Ne=n.getAttribute("data-tag")||"",f={namespace:i,analyticsHost:t,siteId:r,visitorId:s,debounceDuration:u,sessionReplayBatchSize:g,sessionReplayBatchInterval:h,sessionReplayMaskTextSelectors:d,skipPatterns:a,maskPatterns:l,autoTrackPageview:!0,autoTrackSpa:!0,trackQuerystring:!0,trackOutbound:!0,enableWebVitals:!1,trackErrors:!1,enableSessionReplay:!1,trackButtonClicks:!1,trackCopy:!1,trackFormInteractions:!1,tag:Ne,featureFlags:{},sessionReplayBlockClass:C,sessionReplayBlockSelector:m,sessionReplayIgnoreClass:o,sessionReplayIgnoreSelector:c,sessionReplayMaskTextClass:p,sessionReplayMaskAllInputs:S,sessionReplayMaskInputOptions:F,sessionReplayCollectFonts:Me,sessionReplaySampling:xe,sessionReplaySlimDOMOptions:Be,sessionReplaySampleRate:Oe},W=f;try{let V=`${t}/site/tracking-config/${r}`,oe=await fetch(V,{method:"GET",credentials:"omit"});if(oe.ok){let y=await oe.json();W={...f,autoTrackPageview:y.trackInitialPageView??f.autoTrackPageview,autoTrackSpa:y.trackSpaNavigation??f.autoTrackSpa,trackQuerystring:y.trackUrlParams??f.trackQuerystring,trackOutbound:y.trackOutbound??f.trackOutbound,enableWebVitals:y.webVitals??f.enableWebVitals,trackErrors:y.trackErrors??f.trackErrors,enableSessionReplay:y.sessionReplay??f.enableSessionReplay,trackButtonClicks:y.trackButtonClicks??f.trackButtonClicks,trackCopy:y.trackCopy??f.trackCopy,trackFormInteractions:y.trackFormInteractions??f.trackFormInteractions}}else console.warn("Failed to fetch tracking config from API, using defaults")}catch(V){console.warn("Error fetching tracking config:",V)}return W.featureFlags=await ze(t,r,i,s),W}var pe="rybbit-replay-sampled";function je(n){if(n>=100)return!0;if(n<=0)return!1;try{let e=sessionStorage.getItem(pe);if(e!==null)return e==="1";let t=Math.random()*100{let r=document.createElement("script");r.src=`${this.config.analyticsHost}/replay.js`,r.async=!1,r.onload=()=>{e()},r.onerror=()=>t(new Error("Failed to load rrweb")),document.head.appendChild(r)})}startRecording(){if(!(this.isRecording||!window.rrweb||!this.config.enableSessionReplay))try{let e={mousemove:!1,mouseInteraction:{MouseUp:!1,MouseDown:!1,Click:!0,ContextMenu:!1,DblClick:!0,Focus:!0,Blur:!0,TouchStart:!1,TouchEnd:!1},scroll:500,input:"last",media:800},t={script:!1,comment:!0,headFavicon:!0,headWhitespace:!0,headMetaDescKeywords:!0,headMetaSocial:!0,headMetaRobots:!0,headMetaHttpEquiv:!0,headMetaAuthorship:!0,headMetaVerification:!0},r={emit:i=>{this.addEvent({type:i.type,data:i.data,timestamp:i.timestamp||Date.now()})},recordCanvas:!1,checkoutEveryNms:6e4,checkoutEveryNth:500,blockClass:this.config.sessionReplayBlockClass??"rr-block",blockSelector:this.config.sessionReplayBlockSelector??null,ignoreClass:this.config.sessionReplayIgnoreClass??"rr-ignore",ignoreSelector:this.config.sessionReplayIgnoreSelector??null,maskTextClass:this.config.sessionReplayMaskTextClass??"rr-mask",maskAllInputs:this.config.sessionReplayMaskAllInputs??!0,maskInputOptions:this.config.sessionReplayMaskInputOptions??{password:!0,email:!0},collectFonts:this.config.sessionReplayCollectFonts??!0,sampling:this.config.sessionReplaySampling??e,slimDOMOptions:this.config.sessionReplaySlimDOMOptions??t};this.config.sessionReplayMaskTextSelectors&&this.config.sessionReplayMaskTextSelectors.length>0&&(r.maskTextSelector=this.config.sessionReplayMaskTextSelectors.join(", ")),this.stopRecordingFn=window.rrweb.record(r),this.isRecording=!0,this.setupBatchTimer()}catch{}}stopRecording(){this.isRecording&&(this.stopRecordingFn&&this.stopRecordingFn(),this.isRecording=!1,this.clearBatchTimer(),this.eventBuffer.length>0&&this.flushEvents())}isActive(){return this.isRecording}addEvent(e){this.eventBuffer.push(e),this.eventBuffer.length>=this.config.sessionReplayBatchSize&&this.flushEvents()}setupBatchTimer(){this.clearBatchTimer(),this.batchTimer=window.setInterval(()=>{this.eventBuffer.length>0&&this.flushEvents()},this.config.sessionReplayBatchInterval)}clearBatchTimer(){this.batchTimer&&(clearInterval(this.batchTimer),this.batchTimer=void 0)}async flushEvents(){if(this.eventBuffer.length===0)return;let e=[...this.eventBuffer];this.eventBuffer=[];let t={userId:this.userId,events:e,metadata:{pageUrl:window.location.href,viewportWidth:screen.width,viewportHeight:screen.height,language:navigator.language}};try{await this.sendBatch(t)}catch{this.eventBuffer.unshift(...e)}}updateUserId(e){e!==this.userId&&(this.eventBuffer.length>0&&this.flushEvents(),this.userId=e)}onPageChange(){this.isRecording&&this.flushEvents()}cleanup(){this.stopRecording()}};var b={automationApi:1,webdriver:1,zeroOuterDimensions:2,missingChrome:4,swiftShader:8,emptyPlugins:16,defaultViewport800x600:32,defaultViewport1024x768:64,impossibleDimensions:128,outerDimensionsWeird:256,pluginApiAbsence:512},ge=null,Ge=10;function he(){return me().score}function fe(){return me().mask}function me(){return ge??(ge=Ke()),ge}function Ke(){let n=0,e=0;function t(r,i){(e&r)===0&&(e|=r,n+=i)}try{let r=navigator.userAgent,i=/Chrome\//.test(r)&&!/\bwv\b|; wv\)/.test(r),s=/Windows NT|Macintosh|X11|Linux x86_64/.test(r)&&!/Mobile|Android|iPhone|iPad/.test(r),a=Number(window.screen?.width),l=Number(window.screen?.height),d=Number(window.outerWidth),u=Number(window.outerHeight),g=Number(window.innerWidth),h=Number(window.innerHeight),m=["__webdriver_evaluate","__selenium_evaluate","__webdriver_script_function","__webdriver_script_func","__webdriver_script_fn","__fxdriver_evaluate","__driver_unwrapped","__webdriver_unwrapped","__driver_evaluate","__selenium_unwrapped","__fxdriver_unwrapped","_phantom","callPhantom","__nightmare","domAutomation","domAutomationController"].some(c=>c in window||c in document);(navigator.webdriver===!0||m)&&t(b.automationApi,3),(u===0||d===0)&&t(b.zeroOuterDimensions,2),(!Number.isFinite(a)||!Number.isFinite(l)||a<=0||l<=0||a>1e5||l>1e5)&&t(b.impossibleDimensions,3),s&&a===800&&l===600&&t(b.defaultViewport800x600,3),s&&a===1024&&l===768&&t(b.defaultViewport1024x768,3),Number.isFinite(d)&&Number.isFinite(u)&&Number.isFinite(g)&&Number.isFinite(h)&&d>0&&u>0&&g>0&&h>0&&(d+8this.sendSessionReplayBatch(e)),await this.sessionReplayRecorder.initialize()}catch(e){console.error("Failed to initialize session replay:",e)}}async sendSessionReplayBatch(e){try{await fetch(`${this.config.analyticsHost}/session-replay/record/${this.config.siteId}`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e),mode:"cors",keepalive:!1})}catch(t){throw console.error("Failed to send session replay batch:",t),t}}createBasePayload(){let e=new URL(window.location.href),t=e.pathname;if(e.hash&&e.hash.startsWith("#/")&&(t=e.hash.substring(1)),$(t,this.config.skipPatterns))return null;let r=$(t,this.config.maskPatterns);r&&(t=r);let i={site_id:this.config.siteId,hostname:e.hostname,pathname:t,querystring:this.config.trackQuerystring?e.search:"",screenWidth:screen.width,screenHeight:screen.height,language:navigator.language,page_title:document.title,referrer:document.referrer,_bs:he(),_bsm:fe()};this.customUserId&&(i.user_id=this.customUserId),this.config.tag&&(i.tag=this.config.tag);let s=this.getFeatureFlagEventPayload();return Object.keys(s).length>0&&(i.feature_flags=s),i}async sendTrackingData(e){try{await fetch(`${this.config.analyticsHost}/track`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e),mode:"cors",keepalive:!0})}catch(t){console.error("Failed to send tracking data:",t)}}track(e,t="",r={}){if(e==="custom_event"&&(!t||typeof t!="string")){console.error("Event name is required and must be a string for custom events");return}let i=this.createBasePayload();if(!i)return;let a={...i,type:e,event_name:t,properties:["custom_event","outbound","error","button_click","copy","form_submit","input_change"].includes(e)?JSON.stringify(r):void 0};this.sendTrackingData(a)}trackPageview(){this.track("pageview")}trackEvent(e,t={}){this.track("custom_event",e,t)}getFeatureFlag(e,t){let r=this.config.featureFlags?.[e];if(!r)return t;let i=`${e}:${r.version}:${this.serializeFeatureFlagValue(r.value)}`;return this.exposedFeatureFlags.has(i)||(this.exposedFeatureFlags.add(i),this.trackEvent("feature_flag_exposure",{key:e,value:this.serializeFeatureFlagValue(r.value),version:r.version,reason:r.reason})),r.value}getFeatureFlags(){return Object.fromEntries(Object.entries(this.config.featureFlags||{}).map(([e,t])=>[e,t.value]))}getFeatureFlagPayload(e,t){let r=this.config.featureFlags?.[e];return!r||r.payload===void 0?t:r.payload}getFeatureFlagPayloads(){return Object.fromEntries(Object.entries(this.config.featureFlags||{}).filter(([,e])=>e.payload!==void 0).map(([e,t])=>[e,t.payload]))}trackOutbound(e,t="",r="_self"){this.track("outbound","",{url:e,text:t,target:r})}trackWebVitals(e){let t=this.createBasePayload();if(!t)return;let r={...t,type:"performance",event_name:"web-vitals",...e};this.sendTrackingData(r)}trackError(e,t={}){let r=e?.message||"";if(r.includes("ResizeObserver loop completed with undelivered notifications")||r.includes("ResizeObserver loop limit exceeded"))return;let i=window.location.origin,s=t.filename||"",a=e.stack||"";if(s)try{if(new URL(s).origin!==i)return}catch{}else if(a&&!a.includes(i))return;let d=[e.name||"Error",r,t.filename||"",t.lineno??"",t.colno??""].join("|"),u=Date.now(),g=6e4,h=this.errorDedupeCache.get(d);if(h&&u-hg){for(let[o,c]of this.errorDedupeCache.entries())u-c>C&&this.errorDedupeCache.delete(o);this.errorDedupeLastCleanup=u}let m={message:e.message?.substring(0,500)||"Unknown error",stack:a.substring(0,2e3)||""};if(s&&(m.fileName=s),t.lineno){let o=typeof t.lineno=="string"?parseInt(t.lineno,10):t.lineno;o&&o!==0&&(m.lineNumber=o)}if(t.colno){let o=typeof t.colno=="string"?parseInt(t.colno,10):t.colno;o&&o!==0&&(m.columnNumber=o)}for(let o in t)!["lineno","colno"].includes(o)&&t[o]!==void 0&&(m[o]=t[o]);this.track("error",e.name||"Error",m)}trackButtonClick(e){this.track("button_click","",e)}trackCopy(e){this.track("copy","",e)}trackFormSubmit(e){this.track("form_submit","",e)}trackInputChange(e){this.track("input_change","",e)}identify(e,t){if(typeof e!="string"||e.trim()===""){console.error("User ID must be a non-empty string");return}this.customUserId=e.trim();try{localStorage.setItem(`${this.config.namespace}-user-id`,this.customUserId)}catch{console.warn("Could not persist user ID to localStorage")}this.sendIdentifyEvent(this.customUserId,t,!0).then(()=>this.refreshFeatureFlags()),this.sessionReplayRecorder&&this.sessionReplayRecorder.updateUserId(this.customUserId)}setTraits(e){if(!e||typeof e!="object"){console.error("Traits must be an object");return}let t=this.customUserId;if(!t){console.warn("Cannot set traits without identifying user first. Call identify() first.");return}this.sendIdentifyEvent(t,e,!1).then(()=>this.refreshFeatureFlags())}async sendIdentifyEvent(e,t,r=!0){try{await fetch(`${this.config.analyticsHost}/identify`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({site_id:this.config.siteId,user_id:e,traits:t,is_new_identify:r}),mode:"cors",keepalive:!0})}catch(i){console.error("Failed to send identify event:",i)}}clearUserId(){this.customUserId=null;try{localStorage.removeItem(`${this.config.namespace}-user-id`)}catch{}this.sessionReplayRecorder&&this.sessionReplayRecorder.updateUserId(""),this.refreshFeatureFlags()}getUserId(){return this.customUserId}startSessionReplay(){this.sessionReplayRecorder?this.sessionReplayRecorder.startRecording():console.warn("Session replay not initialized")}stopSessionReplay(){this.sessionReplayRecorder&&this.sessionReplayRecorder.stopRecording()}isSessionReplayActive(){return this.sessionReplayRecorder?.isActive()??!1}onPageChange(){this.refreshFeatureFlags(),this.sessionReplayRecorder&&this.sessionReplayRecorder.onPageChange()}cleanup(){this.sessionReplayRecorder&&this.sessionReplayRecorder.cleanup()}};var Te=-1,I=n=>{addEventListener("pageshow",(e=>{e.persisted&&(Te=e.timeStamp,n(e))}),!0)},v=(n,e,t,r)=>{let i,s;return a=>{e.value>=0&&(a||r)&&(s=e.value-(i??0),(s||i===void 0)&&(i=e.value,e.delta=s,e.rating=((l,d)=>l>d[1]?"poor":l>d[0]?"needs-improvement":"good")(e.value,t),n(e)))}},Y=n=>{requestAnimationFrame((()=>requestAnimationFrame((()=>n()))))},Z=()=>{let n=performance.getEntriesByType("navigation")[0];if(n&&n.responseStart>0&&n.responseStartZ()?.activationStart??0,k=(n,e=-1)=>{let t=Z(),r="navigate";return Te>=0?r="back-forward-cache":t&&(document.prerendering||_()>0?r="prerender":document.wasDiscarded?r="restore":t.type&&(r=t.type.replace(/_/g,"-"))),{name:n,value:e,rating:"good",delta:0,entries:[],id:`v5-${Date.now()}-${Math.floor(8999999999999*Math.random())+1e12}`,navigationType:r}},z=new WeakMap;function ee(n,e){return z.get(n)||z.set(n,new e),z.get(n)}var G=class{constructor(){R(this,"t");R(this,"i",0);R(this,"o",[])}h(e){if(e.hadRecentInput)return;let t=this.o[0],r=this.o.at(-1);this.i&&t&&r&&e.startTime-r.startTime<1e3&&e.startTime-t.startTime<5e3?(this.i+=e.value,this.o.push(e)):(this.i=e.value,this.o=[e]),this.t?.(e)}},A=(n,e,t={})=>{try{if(PerformanceObserver.supportedEntryTypes.includes(n)){let r=new PerformanceObserver((i=>{Promise.resolve().then((()=>{e(i.getEntries())}))}));return r.observe({type:n,buffered:!0,...t}),r}}catch{}},te=n=>{let e=!1;return()=>{e||(n(),e=!0)}},P=-1,Ce=new Set,ye=()=>document.visibilityState!=="hidden"||document.prerendering?1/0:0,K=n=>{if(document.visibilityState==="hidden"){if(n.type==="visibilitychange")for(let e of Ce)e();isFinite(P)||(P=n.type==="visibilitychange"?n.timeStamp:0,removeEventListener("prerenderingchange",K,!0))}},B=()=>{if(P<0){let n=_();P=(document.prerendering?void 0:globalThis.performance.getEntriesByType("visibility-state").filter((t=>t.name==="hidden"&&t.startTime>n))[0]?.startTime)??ye(),addEventListener("visibilitychange",K,!0),addEventListener("prerenderingchange",K,!0),I((()=>{setTimeout((()=>{P=ye()}))}))}return{get firstHiddenTime(){return P},onHidden(n){Ce.add(n)}}},O=n=>{document.prerendering?addEventListener("prerenderingchange",(()=>n()),!0):n()},be=[1800,3e3],re=(n,e={})=>{O((()=>{let t=B(),r,i=k("FCP"),s=A("paint",(a=>{for(let l of a)l.name==="first-contentful-paint"&&(s.disconnect(),l.startTime{i=k("FCP"),r=v(n,i,be,e.reportAllChanges),Y((()=>{i.value=performance.now()-a.timeStamp,r(!0)}))})))}))},ve=[.1,.25],Pe=(n,e={})=>{let t=B();re(te((()=>{let r,i=k("CLS",0),s=ee(e,G),a=d=>{for(let u of d)s.h(u);s.i>i.value&&(i.value=s.i,i.entries=s.o,r())},l=A("layout-shift",a);l&&(r=v(n,i,ve,e.reportAllChanges),t.onHidden((()=>{a(l.takeRecords()),r(!0)})),I((()=>{s.i=0,i=k("CLS",0),r=v(n,i,ve,e.reportAllChanges),Y((()=>r()))})),setTimeout(r))})))},Ie=0,j=1/0,x=0,qe=n=>{for(let e of n)e.interactionId&&(j=Math.min(j,e.interactionId),x=Math.max(x,e.interactionId),Ie=x?(x-j)/7+1:0)},J,ke=()=>J?Ie:performance.interactionCount??0,Xe=()=>{"interactionCount"in performance||J||(J=A("event",qe,{type:"event",buffered:!0,durationThreshold:0}))},we=0,q=class{constructor(){R(this,"u",[]);R(this,"l",new Map);R(this,"m");R(this,"p")}v(){we=ke(),this.u.length=0,this.l.clear()}L(){let e=Math.min(this.u.length-1,Math.floor((ke()-we)/50));return this.u[e]}h(e){if(this.m?.(e),!e.interactionId&&e.entryType!=="first-input")return;let t=this.u.at(-1),r=this.l.get(e.interactionId);if(r||this.u.length<10||e.duration>t.P){if(r?e.duration>r.P?(r.entries=[e],r.P=e.duration):e.duration===r.P&&e.startTime===r.entries[0].startTime&&r.entries.push(e):(r={id:e.interactionId,entries:[e],P:e.duration},this.l.set(r.id,r),this.u.push(r)),this.u.sort(((i,s)=>s.P-i.P)),this.u.length>10){let i=this.u.splice(10);for(let s of i)this.l.delete(s.id)}this.p?.(r)}}},_e=n=>{let e=globalThis.requestIdleCallback||setTimeout;document.visibilityState==="hidden"?n():(n=te(n),addEventListener("visibilitychange",n,{once:!0,capture:!0}),e((()=>{n(),removeEventListener("visibilitychange",n,{capture:!0})})))},Re=[200,500],Ae=(n,e={})=>{if(!globalThis.PerformanceEventTiming||!("interactionId"in PerformanceEventTiming.prototype))return;let t=B();O((()=>{Xe();let r,i=k("INP"),s=ee(e,q),a=d=>{_e((()=>{for(let g of d)s.h(g);let u=s.L();u&&u.P!==i.value&&(i.value=u.P,i.entries=u.entries,r())}))},l=A("event",a,{durationThreshold:e.durationThreshold??40});r=v(n,i,Re,e.reportAllChanges),l&&(l.observe({type:"first-input",buffered:!0}),t.onHidden((()=>{a(l.takeRecords()),r(!0)})),I((()=>{s.v(),i=k("INP"),r=v(n,i,Re,e.reportAllChanges)})))}))},X=class{constructor(){R(this,"m")}h(e){this.m?.(e)}},Se=[2500,4e3],Fe=(n,e={})=>{O((()=>{let t=B(),r,i=k("LCP"),s=ee(e,X),a=d=>{e.reportAllChanges||(d=d.slice(-1));for(let u of d)s.h(u),u.startTime{a(l.takeRecords()),l.disconnect(),r(!0)})),u=g=>{g.isTrusted&&(_e(d),removeEventListener(g.type,u,{capture:!0}))};for(let g of["keydown","click","visibilitychange"])addEventListener(g,u,{capture:!0});I((g=>{i=k("LCP"),r=v(n,i,Se,e.reportAllChanges),Y((()=>{i.value=performance.now()-g.timeStamp,r(!0)}))}))}}))},Ee=[800,1800],Q=n=>{document.prerendering?O((()=>Q(n))):document.readyState!=="complete"?addEventListener("load",(()=>Q(n)),!0):setTimeout(n)},Le=(n,e={})=>{let t=k("TTFB"),r=v(n,t,Ee,e.reportAllChanges);Q((()=>{let i=Z();i&&(t.value=Math.max(i.responseStart-_(),0),t.entries=[i],r(!0),I((()=>{t=k("TTFB",0),r=v(n,t,Ee,e.reportAllChanges),r(!0)})))}))};var N=class{constructor(e){this.data={lcp:null,cls:null,inp:null,fcp:null,ttfb:null};this.sent=!1;this.timeout=null;this.onReadyCallback=null;this.onReadyCallback=e}initialize(){try{Fe(this.collectMetric.bind(this)),Pe(this.collectMetric.bind(this)),Ae(this.collectMetric.bind(this)),re(this.collectMetric.bind(this)),Le(this.collectMetric.bind(this)),this.timeout=setTimeout(()=>{this.sent||this.sendData()},2e4),window.addEventListener("beforeunload",()=>{this.sent||this.sendData()})}catch(e){console.warn("Error initializing web vitals tracking:",e)}}collectMetric(e){if(this.sent)return;let t=e.name.toLowerCase();this.data[t]=e.value,Object.values(this.data).every(i=>i!==null)&&this.sendData()}sendData(){this.sent||(this.sent=!0,this.timeout&&(clearTimeout(this.timeout),this.timeout=null),this.onReadyCallback&&this.onReadyCallback(this.data))}getData(){return{...this.data}}};var D=class{constructor(e,t){this.tracker=e,this.config=t}initialize(){document.addEventListener("click",this.handleClick.bind(this),!0)}handleClick(e){let t=e.target;this.config.trackButtonClicks&&this.isButton(t)&&this.trackButtonClick(t)}isButton(e){if(e.tagName==="BUTTON"||e.getAttribute("role")==="button")return!0;if(e.tagName==="INPUT"){let i=e.type?.toLowerCase();if(i==="submit"||i==="button")return!0}let t=e.parentElement,r=0;for(;t&&r<3;){if(t.tagName==="BUTTON"||t.getAttribute("role")==="button")return!0;t=t.parentElement,r++}return!1}trackButtonClick(e){let t=this.findButton(e);if(!t||t.hasAttribute("data-rybbit-event"))return;let r={text:this.getElementText(t),...this.extractDataAttributes(t)};this.tracker.trackButtonClick(r)}extractDataAttributes(e){let t={};for(let r of e.attributes)if(r.name.startsWith("data-rybbit-prop-")){let i=r.name.replace("data-rybbit-prop-","");t[i]=r.value}return t}findButton(e){if(e.tagName==="BUTTON"||e.getAttribute("role")==="button")return e;if(e.tagName==="INPUT"){let i=e.type?.toLowerCase();if(i==="submit"||i==="button")return e}let t=e.parentElement,r=0;for(;t&&r<3;){if(t.tagName==="BUTTON"||t.getAttribute("role")==="button")return t;t=t.parentElement,r++}return null}getElementText(e){let t=e.textContent?.trim().substring(0,100);if(t)return t;let r=e.getAttribute("aria-label")?.trim().substring(0,100);if(r)return r;if(e.tagName==="INPUT"){let s=e.value?.trim().substring(0,100);if(s)return s}let i=e.getAttribute("title")?.trim().substring(0,100);if(i)return i}cleanup(){document.removeEventListener("click",this.handleClick.bind(this),!0)}};var U=class{constructor(e){this.tracker=e}initialize(){document.addEventListener("copy",this.handleCopy.bind(this))}handleCopy(){let e=window.getSelection();if(!e||e.isCollapsed)return;let t=e.toString(),r=t.length;if(r===0)return;let i=e.anchorNode,s=i instanceof HTMLElement?i:i?.parentElement;if(!s)return;let a={text:t.substring(0,500),...r>500&&{textLength:r},sourceElement:s.tagName.toLowerCase()};this.tracker.trackCopy(a)}cleanup(){document.removeEventListener("copy",this.handleCopy.bind(this))}};var H=class{constructor(e,t){this.tracker=e,this.config=t,this.boundHandleSubmit=this.handleSubmit.bind(this),this.boundHandleChange=this.handleChange.bind(this)}initialize(){document.addEventListener("submit",this.boundHandleSubmit,!0),document.addEventListener("change",this.boundHandleChange,!0)}cleanup(){document.removeEventListener("submit",this.boundHandleSubmit,!0),document.removeEventListener("change",this.boundHandleChange,!0)}handleSubmit(e){let t=e.target;if(t.tagName!=="FORM")return;let r={formId:t.id||"",formName:t.name||"",formAction:t.action||"",method:(t.method||"get").toUpperCase(),fieldCount:t.elements.length,ariaLabel:t.getAttribute("aria-label")||void 0,...this.extractDataAttributes(t)};this.tracker.trackFormSubmit(r)}handleChange(e){let t=e.target,r=t.tagName.toUpperCase();if(!["INPUT","SELECT","TEXTAREA"].includes(r)||t.disabled||t.readOnly)return;if(r==="INPUT"){let a=t.type?.toLowerCase();if(a==="hidden"||a==="password")return}let i=t.name||t.id||t.getAttribute("aria-label")||t.placeholder||"",s={element:r.toLowerCase(),inputType:r==="INPUT"?t.type?.toLowerCase():void 0,inputName:i,formId:t.form?.id||void 0,formName:t.form?.name||void 0,...this.extractDataAttributes(t)};this.tracker.trackInputChange(s)}extractDataAttributes(e){let t={};for(let r of e.attributes)if(r.name.startsWith("data-rybbit-prop-")){let i=r.name.replace("data-rybbit-prop-","");t[i]=r.value}return t}};(async function(){let n=document.currentScript;if(!n){console.error("Could not find current script tag");return}let e=n.getAttribute("data-namespace")||"rybbit",t=`disable-${e}`;if(window.__RYBBIT_OPTOUT__||localStorage.getItem(t)!==null){window[e]={pageview:()=>{},event:()=>{},error:()=>{},trackOutbound:()=>{},identify:()=>{},setTraits:()=>{},clearUserId:()=>{},getUserId:()=>null,flag:(o,c)=>c,flagPayload:(o,c)=>c,flags:()=>({}),flagPayloads:()=>({}),onReady:()=>{},startSessionReplay:()=>{},stopSessionReplay:()=>{},isSessionReplayActive:()=>!1};return}let r=[],i=o=>(...c)=>{r.push([o,c])};window[e]={pageview:i("pageview"),event:i("event"),error:i("error"),trackOutbound:i("trackOutbound"),identify:i("identify"),setTraits:i("setTraits"),clearUserId:i("clearUserId"),getUserId:()=>null,flag:(o,c)=>c,flagPayload:(o,c)=>c,flags:()=>({}),flagPayloads:()=>({}),onReady:i("onReady"),startSessionReplay:i("startSessionReplay"),stopSessionReplay:i("stopSessionReplay"),isSessionReplayActive:()=>!1};let s=await de(n);if(!s)return;let a=new M(s);s.enableWebVitals&&new N(c=>{a.trackWebVitals(c)}).initialize();let l=null,d=null,u=null;s.trackButtonClicks&&(l=new D(a,s),l.initialize()),s.trackCopy&&(d=new U(a),d.initialize()),s.trackFormInteractions&&(u=new H(a,s),u.initialize()),s.trackErrors&&(window.addEventListener("error",o=>{a.trackError(o.error||new Error(o.message),{filename:o.filename,lineno:o.lineno,colno:o.colno})}),window.addEventListener("unhandledrejection",o=>{let c=o.reason instanceof Error?o.reason:new Error(String(o.reason));a.trackError(c,{type:"unhandledrejection"})}));let g=()=>a.trackPageview(),h=s.debounceDuration>0?ce(g,s.debounceDuration):g;function C(){if(document.addEventListener("click",function(o){let c=o.target;for(;c&&c!==document.documentElement;){if(c.hasAttribute("data-rybbit-event")){let p=c.getAttribute("data-rybbit-event");if(p){let w={};for(let S of c.attributes)if(S.name.startsWith("data-rybbit-prop-")){let E=S.name.replace("data-rybbit-prop-","");w[E]=S.value}a.trackEvent(p,w)}break}c=c.parentElement}if(s.trackOutbound){let p=o.target.closest("a");p?.href&&le(p.href)&&a.trackOutbound(p.href,p.innerText||p.textContent||"",p.target||"_self")}}),s.autoTrackSpa){let o=history.pushState,c=history.replaceState;history.pushState=function(...p){o.apply(this,p),h(),a.onPageChange()},history.replaceState=function(...p){c.apply(this,p),h(),a.onPageChange()},window.addEventListener("popstate",()=>{h(),a.onPageChange()}),window.addEventListener("hashchange",()=>{h(),a.onPageChange()})}}window[s.namespace]={pageview:()=>a.trackPageview(),event:(o,c={})=>a.trackEvent(o,c),error:(o,c={})=>a.trackError(o,c),trackOutbound:(o,c="",p="_self")=>a.trackOutbound(o,c,p),identify:(o,c)=>a.identify(o,c),setTraits:o=>a.setTraits(o),clearUserId:()=>a.clearUserId(),getUserId:()=>a.getUserId(),flag:(o,c)=>a.getFeatureFlag(o,c),flagPayload:(o,c)=>a.getFeatureFlagPayload(o,c),flags:()=>a.getFeatureFlags(),flagPayloads:()=>a.getFeatureFlagPayloads(),onReady:o=>o(window[s.namespace]),startSessionReplay:()=>a.startSessionReplay(),stopSessionReplay:()=>a.stopSessionReplay(),isSessionReplayActive:()=>a.isSessionReplayActive()};let m=window[s.namespace];for(let[o,c]of r)m[o](...c);C(),window.addEventListener("beforeunload",()=>{l?.cleanup(),d?.cleanup(),a.cleanup()}),s.autoTrackPageview&&a.trackPageview()})();})(); +"use strict";(()=>{var De=Object.defineProperty;var Ue=(r,e,t)=>e in r?De(r,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):r[e]=t;var R=(r,e,t)=>Ue(r,typeof e!="symbol"?e+"":e,t);function He(r){if(r.startsWith("re:")){let l=r.slice(3);if(!l)throw new Error("Empty regex pattern");return new RegExp(l)}let t="__DOUBLE_ASTERISK_TOKEN__",n="__SINGLE_ASTERISK_TOKEN__",s=r.replace(/\*\*/g,t).replace(/\*/g,n).replace(/[.+?^${}()|[\]\\]/g,"\\$&");s=s.replace(new RegExp(`/${t}/`,"g"),"/(?:.+/)?"),s=s.replace(new RegExp(t,"g"),".*"),s=s.replace(/\//g,"\\/");let a=s.replace(new RegExp(n,"g"),"[^/]+");return new RegExp("^"+a+"$")}function j(r,e){for(let t of e)try{if(He(t).test(r))return t}catch(n){console.error(`Invalid pattern: ${t}`,n)}return null}function ce(r,e){let t=null;return(...n)=>{t&&clearTimeout(t),t=setTimeout(()=>r(...n),e)}}function le(r){try{let e=window.location.hostname,t=new URL(r).hostname;return t!==e&&t!==""}catch{return!1}}function E(r,e){if(!r)return e;try{let t=JSON.parse(r);return Array.isArray(e)&&!Array.isArray(t)?e:t}catch(t){return console.error("Error parsing JSON:",t),e}}function ue(){try{if(crypto?.randomUUID)return crypto.randomUUID()}catch{}return`${Date.now().toString(36)}-${Math.random().toString(36).slice(2,12)}`}function We(r){let e=`${r}-visitor-id`;try{let t=localStorage.getItem(e);if(t)return t;let n=ue();return localStorage.setItem(e,n),n}catch{return ue()}}function $e(r){try{return localStorage.getItem(`${r}-user-id`)||void 0}catch{return}}function je(r){return r.hash&&r.hash.startsWith("#/")?r.hash.substring(1):r.pathname}async function Ve(r,e,t,n){try{let i=new URL(window.location.href),s=await fetch(`${r}/site/${e}/feature-flags/evaluate`,{method:"POST",headers:{"Content-Type":"application/json"},credentials:"omit",body:JSON.stringify({anonymousId:n,identifiedUserId:$e(t),hostname:i.hostname,pathname:je(i),querystring:i.search,query:Object.fromEntries(i.searchParams.entries()),referrer:document.referrer,language:navigator.language,screenWidth:screen.width,screenHeight:screen.height})});if(!s.ok)return{};let a=await s.json();return a?.flags&&typeof a.flags=="object"?a.flags:{}}catch{return{}}}async function de(r){let e=r.getAttribute("src");if(!e)return console.error("Script src attribute is missing"),null;let t=e.split("/script.js")[0];if(!t)return console.error("Please provide a valid analytics host"),null;let n=r.getAttribute("data-site-id")||r.getAttribute("site-id");if(!n)return console.error("Please provide a valid site ID using the data-site-id attribute"),null;let i=r.getAttribute("data-namespace")||"rybbit",s=We(i),a=E(r.getAttribute("data-skip-patterns"),[]),l=E(r.getAttribute("data-mask-patterns"),[]),d=E(r.getAttribute("data-replay-mask-text-selectors"),[]),u=r.getAttribute("data-debounce")?Math.max(0,parseInt(r.getAttribute("data-debounce"))):500,h=r.getAttribute("data-replay-batch-size")?Math.max(1,parseInt(r.getAttribute("data-replay-batch-size"))):250,g=r.getAttribute("data-replay-batch-interval")?Math.max(1e3,parseInt(r.getAttribute("data-replay-batch-interval"))):5e3,C=r.getAttribute("data-replay-block-class")||void 0,m=r.getAttribute("data-replay-block-selector")||void 0,o=r.getAttribute("data-replay-ignore-class")||void 0,c=r.getAttribute("data-replay-ignore-selector")||void 0,p=r.getAttribute("data-replay-mask-text-class")||void 0,w=r.getAttribute("data-replay-mask-all-inputs"),S=w!==null?w!=="false":void 0,T=r.getAttribute("data-replay-mask-input-options"),A=T?E(T,{password:!0,email:!0}):void 0,re=r.getAttribute("data-replay-collect-fonts"),xe=re!==null?re!=="false":void 0,ie=r.getAttribute("data-replay-sampling"),Me=ie?E(ie,{}):void 0,se=r.getAttribute("data-replay-slim-dom-options"),Be=se?E(se,{}):void 0,ae=r.getAttribute("data-replay-sample-rate"),Oe=ae?Math.min(100,Math.max(0,parseInt(ae,10))):void 0,Ne=r.getAttribute("data-tag")||"",f={namespace:i,analyticsHost:t,siteId:n,visitorId:s,debounceDuration:u,sessionReplayBatchSize:h,sessionReplayBatchInterval:g,sessionReplayMaskTextSelectors:d,skipPatterns:a,maskPatterns:l,autoTrackPageview:!0,autoTrackSpa:!0,trackQuerystring:!0,trackOutbound:!0,enableWebVitals:!1,trackErrors:!1,enableSessionReplay:!1,trackButtonClicks:!1,trackCopy:!1,trackFormInteractions:!1,tag:Ne,featureFlags:{},sessionReplayBlockClass:C,sessionReplayBlockSelector:m,sessionReplayIgnoreClass:o,sessionReplayIgnoreSelector:c,sessionReplayMaskTextClass:p,sessionReplayMaskAllInputs:S,sessionReplayMaskInputOptions:A,sessionReplayCollectFonts:xe,sessionReplaySampling:Me,sessionReplaySlimDOMOptions:Be,sessionReplaySampleRate:Oe},W=f;try{let $=`${t}/site/tracking-config/${n}`,oe=await fetch($,{method:"GET",credentials:"omit"});if(oe.ok){let y=await oe.json();W={...f,autoTrackPageview:y.trackInitialPageView??f.autoTrackPageview,autoTrackSpa:y.trackSpaNavigation??f.autoTrackSpa,trackQuerystring:y.trackUrlParams??f.trackQuerystring,trackOutbound:y.trackOutbound??f.trackOutbound,enableWebVitals:y.webVitals??f.enableWebVitals,trackErrors:y.trackErrors??f.trackErrors,enableSessionReplay:y.sessionReplay??f.enableSessionReplay,trackButtonClicks:y.trackButtonClicks??f.trackButtonClicks,trackCopy:y.trackCopy??f.trackCopy,trackFormInteractions:y.trackFormInteractions??f.trackFormInteractions}}else console.warn("Failed to fetch tracking config from API, using defaults")}catch($){console.warn("Error fetching tracking config:",$)}return W.featureFlags=await Ve(t,n,i,s),W}var pe="rybbit-replay-sampled";function ze(){let r=new Uint8Array(16);if(globalThis.crypto?.getRandomValues)globalThis.crypto.getRandomValues(r);else for(let t=0;tt.toString(16).padStart(2,"0"));return`${e.slice(0,4).join("")}-${e.slice(4,6).join("")}-${e.slice(6,8).join("")}-${e.slice(8,10).join("")}-${e.slice(10).join("")}`}function qe(r){if(r>=100)return!0;if(r<=0)return!1;try{let e=sessionStorage.getItem(pe);if(e!==null)return e==="1";let t=Math.random()*100{let n=document.createElement("script");n.src=`${this.config.analyticsHost}/replay.js`,n.async=!1,n.onload=()=>{e()},n.onerror=()=>t(new Error("Failed to load rrweb")),document.head.appendChild(n)})}startRecording(){if(!(this.isRecording||!window.rrweb||!this.config.enableSessionReplay))try{let e={mousemove:!1,mouseInteraction:{MouseUp:!1,MouseDown:!1,Click:!0,ContextMenu:!1,DblClick:!0,Focus:!0,Blur:!0,TouchStart:!1,TouchEnd:!1},scroll:500,input:"last",media:800},t={script:!1,comment:!0,headFavicon:!0,headWhitespace:!0,headMetaDescKeywords:!0,headMetaSocial:!0,headMetaRobots:!0,headMetaHttpEquiv:!0,headMetaAuthorship:!0,headMetaVerification:!0},n={emit:i=>{this.addEvent({type:i.type,data:i.data,timestamp:i.timestamp||Date.now()})},recordCanvas:!1,checkoutEveryNms:6e4,checkoutEveryNth:500,blockClass:this.config.sessionReplayBlockClass??"rr-block",blockSelector:this.config.sessionReplayBlockSelector??null,ignoreClass:this.config.sessionReplayIgnoreClass??"rr-ignore",ignoreSelector:this.config.sessionReplayIgnoreSelector??null,maskTextClass:this.config.sessionReplayMaskTextClass??"rr-mask",maskAllInputs:this.config.sessionReplayMaskAllInputs??!0,maskInputOptions:this.config.sessionReplayMaskInputOptions??{password:!0,email:!0},collectFonts:this.config.sessionReplayCollectFonts??!0,sampling:this.config.sessionReplaySampling??e,slimDOMOptions:this.config.sessionReplaySlimDOMOptions??t};this.config.sessionReplayMaskTextSelectors&&this.config.sessionReplayMaskTextSelectors.length>0&&(n.maskTextSelector=this.config.sessionReplayMaskTextSelectors.join(", ")),this.stopRecordingFn=window.rrweb.record(n),this.isRecording=!0,this.setupBatchTimer()}catch{}}stopRecording(){this.isRecording&&(this.stopRecordingFn&&this.stopRecordingFn(),this.isRecording=!1,this.clearBatchTimer(),(this.eventBuffer.length>0||this.pendingBatches.length>0)&&this.flushEvents())}isActive(){return this.isRecording}addEvent(e){this.eventBuffer.push({...e,sequence:this.nextSequence++}),this.eventBuffer.length>=this.config.sessionReplayBatchSize&&this.flushEvents()}setupBatchTimer(){this.clearBatchTimer(),this.batchTimer=window.setInterval(()=>{(this.eventBuffer.length>0||this.pendingBatches.length>0)&&this.flushEvents()},this.config.sessionReplayBatchInterval)}clearBatchTimer(){this.batchTimer&&(clearInterval(this.batchTimer),this.batchTimer=void 0)}async flushEvents(){if(this.eventBuffer.length>0){let e={batchId:ze(),userId:this.userId,events:this.eventBuffer,metadata:{pageUrl:window.location.href,viewportWidth:screen.width,viewportHeight:screen.height,language:navigator.language}};this.eventBuffer=[],this.pendingBatches.push(e)}await this.drainPendingBatches()}drainPendingBatches(){return this.sendLoop?this.sendLoop.then(()=>{if(this.pendingBatches.length>0&&this.sendLoop===null)return this.drainPendingBatches()}):(this.sendLoop=(async()=>{for(;this.pendingBatches.length>0;)try{await this.sendBatch(this.pendingBatches[0]),this.pendingBatches.shift()}catch{return}})().finally(()=>{this.sendLoop=null}),this.sendLoop)}updateUserId(e){e!==this.userId&&(this.eventBuffer.length>0&&this.flushEvents(),this.userId=e)}onPageChange(){this.isRecording&&this.flushEvents()}cleanup(){this.stopRecording()}};var b={automationApi:1,webdriver:1,zeroOuterDimensions:2,missingChrome:4,swiftShader:8,emptyPlugins:16,defaultViewport800x600:32,defaultViewport1024x768:64,impossibleDimensions:128,outerDimensionsWeird:256,pluginApiAbsence:512},he=null,Ge=10;function ge(){return me().score}function fe(){return me().mask}function me(){return he??(he=Ke()),he}function Ke(){let r=0,e=0;function t(n,i){(e&n)===0&&(e|=n,r+=i)}try{let n=navigator.userAgent,i=/Chrome\//.test(n)&&!/\bwv\b|; wv\)/.test(n),s=/Windows NT|Macintosh|X11|Linux x86_64/.test(n)&&!/Mobile|Android|iPhone|iPad/.test(n),a=Number(window.screen?.width),l=Number(window.screen?.height),d=Number(window.outerWidth),u=Number(window.outerHeight),h=Number(window.innerWidth),g=Number(window.innerHeight),m=["__webdriver_evaluate","__selenium_evaluate","__webdriver_script_function","__webdriver_script_func","__webdriver_script_fn","__fxdriver_evaluate","__driver_unwrapped","__webdriver_unwrapped","__driver_evaluate","__selenium_unwrapped","__fxdriver_unwrapped","_phantom","callPhantom","__nightmare","domAutomation","domAutomationController"].some(c=>c in window||c in document);(navigator.webdriver===!0||m)&&t(b.automationApi,3),(u===0||d===0)&&t(b.zeroOuterDimensions,2),(!Number.isFinite(a)||!Number.isFinite(l)||a<=0||l<=0||a>1e5||l>1e5)&&t(b.impossibleDimensions,3),s&&a===800&&l===600&&t(b.defaultViewport800x600,3),s&&a===1024&&l===768&&t(b.defaultViewport1024x768,3),Number.isFinite(d)&&Number.isFinite(u)&&Number.isFinite(h)&&Number.isFinite(g)&&d>0&&u>0&&h>0&&g>0&&(d+8this.sendSessionReplayBatch(e)),await this.sessionReplayRecorder.initialize()}catch(e){console.error("Failed to initialize session replay:",e)}}async sendSessionReplayBatch(e){try{await fetch(`${this.config.analyticsHost}/session-replay/record/${this.config.siteId}`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e),mode:"cors",keepalive:!1})}catch(t){throw console.error("Failed to send session replay batch:",t),t}}createBasePayload(){let e=new URL(window.location.href),t=e.pathname;if(e.hash&&e.hash.startsWith("#/")&&(t=e.hash.substring(1)),j(t,this.config.skipPatterns))return null;let n=j(t,this.config.maskPatterns);n&&(t=n);let i={site_id:this.config.siteId,hostname:e.hostname,pathname:t,querystring:this.config.trackQuerystring?e.search:"",screenWidth:screen.width,screenHeight:screen.height,language:navigator.language,page_title:document.title,referrer:document.referrer,_bs:ge(),_bsm:fe()};this.customUserId&&(i.user_id=this.customUserId),this.config.tag&&(i.tag=this.config.tag);let s=this.getFeatureFlagEventPayload();return Object.keys(s).length>0&&(i.feature_flags=s),i}sendTrackingData(e){let t=(async()=>{try{await fetch(`${this.config.analyticsHost}/track`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e),mode:"cors",keepalive:!0})}catch(n){console.error("Failed to send tracking data:",n)}})();return this.pendingTrackingRequests.add(t),t.finally(()=>this.pendingTrackingRequests.delete(t)),t}track(e,t="",n={}){if(e==="custom_event"&&(!t||typeof t!="string")){console.error("Event name is required and must be a string for custom events");return}let i=this.createBasePayload();if(!i)return;let a={...i,type:e,event_name:t,properties:["custom_event","outbound","error","button_click","copy","form_submit","input_change"].includes(e)?JSON.stringify(n):void 0};this.sendTrackingData(a)}trackPageview(){this.track("pageview")}trackEvent(e,t={}){this.track("custom_event",e,t)}getFeatureFlag(e,t){let n=this.config.featureFlags?.[e];if(!n)return t;let i=`${e}:${n.version}:${this.serializeFeatureFlagValue(n.value)}`;return this.exposedFeatureFlags.has(i)||(this.exposedFeatureFlags.add(i),this.trackEvent("feature_flag_exposure",{key:e,value:this.serializeFeatureFlagValue(n.value),version:n.version,reason:n.reason})),n.value}getFeatureFlags(){return Object.fromEntries(Object.entries(this.config.featureFlags||{}).map(([e,t])=>[e,t.value]))}getFeatureFlagPayload(e,t){let n=this.config.featureFlags?.[e];return!n||n.payload===void 0?t:n.payload}getFeatureFlagPayloads(){return Object.fromEntries(Object.entries(this.config.featureFlags||{}).filter(([,e])=>e.payload!==void 0).map(([e,t])=>[e,t.payload]))}trackOutbound(e,t="",n="_self"){this.track("outbound","",{url:e,text:t,target:n})}trackWebVitals(e){let t=this.createBasePayload();if(!t)return;let n={...t,type:"performance",event_name:"web-vitals",...e};this.sendTrackingData(n)}trackError(e,t={}){let n=e?.message||"";if(n.includes("ResizeObserver loop completed with undelivered notifications")||n.includes("ResizeObserver loop limit exceeded"))return;let i=window.location.origin,s=t.filename||"",a=e.stack||"";if(s)try{if(new URL(s).origin!==i)return}catch{}else if(a&&!a.includes(i))return;let d=[e.name||"Error",n,t.filename||"",t.lineno??"",t.colno??""].join("|"),u=Date.now(),h=6e4,g=this.errorDedupeCache.get(d);if(g&&u-gh){for(let[o,c]of this.errorDedupeCache.entries())u-c>C&&this.errorDedupeCache.delete(o);this.errorDedupeLastCleanup=u}let m={message:e.message?.substring(0,500)||"Unknown error",stack:a.substring(0,2e3)||""};if(s&&(m.fileName=s),t.lineno){let o=typeof t.lineno=="string"?parseInt(t.lineno,10):t.lineno;o&&o!==0&&(m.lineNumber=o)}if(t.colno){let o=typeof t.colno=="string"?parseInt(t.colno,10):t.colno;o&&o!==0&&(m.columnNumber=o)}for(let o in t)!["lineno","colno"].includes(o)&&t[o]!==void 0&&(m[o]=t[o]);this.track("error",e.name||"Error",m)}trackButtonClick(e){this.track("button_click","",e)}trackCopy(e){this.track("copy","",e)}trackFormSubmit(e){this.track("form_submit","",e)}trackInputChange(e){this.track("input_change","",e)}identify(e,t){if(typeof e!="string"||e.trim()===""){console.error("User ID must be a non-empty string");return}this.customUserId=e.trim();try{localStorage.setItem(`${this.config.namespace}-user-id`,this.customUserId)}catch{console.warn("Could not persist user ID to localStorage")}let n=this.customUserId,i=[...this.pendingTrackingRequests];Promise.allSettled(i).then(()=>this.sendIdentifyEvent(n,t,!0)).then(()=>this.refreshFeatureFlags()),this.sessionReplayRecorder&&this.sessionReplayRecorder.updateUserId(this.customUserId)}setTraits(e){if(!e||typeof e!="object"){console.error("Traits must be an object");return}let t=this.customUserId;if(!t){console.warn("Cannot set traits without identifying user first. Call identify() first.");return}this.sendIdentifyEvent(t,e,!1).then(()=>this.refreshFeatureFlags())}async sendIdentifyEvent(e,t,n=!0){try{await fetch(`${this.config.analyticsHost}/identify`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({site_id:this.config.siteId,user_id:e,traits:t,is_new_identify:n}),mode:"cors",keepalive:!0})}catch(i){console.error("Failed to send identify event:",i)}}clearUserId(){this.customUserId=null;try{localStorage.removeItem(`${this.config.namespace}-user-id`)}catch{}this.sessionReplayRecorder&&this.sessionReplayRecorder.updateUserId(""),this.refreshFeatureFlags()}getUserId(){return this.customUserId}startSessionReplay(){this.sessionReplayRecorder?this.sessionReplayRecorder.startRecording():console.warn("Session replay not initialized")}stopSessionReplay(){this.sessionReplayRecorder&&this.sessionReplayRecorder.stopRecording()}isSessionReplayActive(){return this.sessionReplayRecorder?.isActive()??!1}onPageChange(){this.refreshFeatureFlags(),this.sessionReplayRecorder&&this.sessionReplayRecorder.onPageChange()}cleanup(){this.sessionReplayRecorder&&this.sessionReplayRecorder.cleanup()}};var Ee=-1,I=r=>{addEventListener("pageshow",(e=>{e.persisted&&(Ee=e.timeStamp,r(e))}),!0)},v=(r,e,t,n)=>{let i,s;return a=>{e.value>=0&&(a||n)&&(s=e.value-(i??0),(s||i===void 0)&&(i=e.value,e.delta=s,e.rating=((l,d)=>l>d[1]?"poor":l>d[0]?"needs-improvement":"good")(e.value,t),r(e)))}},Y=r=>{requestAnimationFrame((()=>requestAnimationFrame((()=>r()))))},Z=()=>{let r=performance.getEntriesByType("navigation")[0];if(r&&r.responseStart>0&&r.responseStartZ()?.activationStart??0,k=(r,e=-1)=>{let t=Z(),n="navigate";return Ee>=0?n="back-forward-cache":t&&(document.prerendering||_()>0?n="prerender":document.wasDiscarded?n="restore":t.type&&(n=t.type.replace(/_/g,"-"))),{name:r,value:e,rating:"good",delta:0,entries:[],id:`v5-${Date.now()}-${Math.floor(8999999999999*Math.random())+1e12}`,navigationType:n}},V=new WeakMap;function ee(r,e){return V.get(r)||V.set(r,new e),V.get(r)}var q=class{constructor(){R(this,"t");R(this,"i",0);R(this,"o",[])}h(e){if(e.hadRecentInput)return;let t=this.o[0],n=this.o.at(-1);this.i&&t&&n&&e.startTime-n.startTime<1e3&&e.startTime-t.startTime<5e3?(this.i+=e.value,this.o.push(e)):(this.i=e.value,this.o=[e]),this.t?.(e)}},L=(r,e,t={})=>{try{if(PerformanceObserver.supportedEntryTypes.includes(r)){let n=new PerformanceObserver((i=>{Promise.resolve().then((()=>{e(i.getEntries())}))}));return n.observe({type:r,buffered:!0,...t}),n}}catch{}},te=r=>{let e=!1;return()=>{e||(r(),e=!0)}},P=-1,Ce=new Set,ye=()=>document.visibilityState!=="hidden"||document.prerendering?1/0:0,G=r=>{if(document.visibilityState==="hidden"){if(r.type==="visibilitychange")for(let e of Ce)e();isFinite(P)||(P=r.type==="visibilitychange"?r.timeStamp:0,removeEventListener("prerenderingchange",G,!0))}},B=()=>{if(P<0){let r=_();P=(document.prerendering?void 0:globalThis.performance.getEntriesByType("visibility-state").filter((t=>t.name==="hidden"&&t.startTime>r))[0]?.startTime)??ye(),addEventListener("visibilitychange",G,!0),addEventListener("prerenderingchange",G,!0),I((()=>{setTimeout((()=>{P=ye()}))}))}return{get firstHiddenTime(){return P},onHidden(r){Ce.add(r)}}},O=r=>{document.prerendering?addEventListener("prerenderingchange",(()=>r()),!0):r()},be=[1800,3e3],ne=(r,e={})=>{O((()=>{let t=B(),n,i=k("FCP"),s=L("paint",(a=>{for(let l of a)l.name==="first-contentful-paint"&&(s.disconnect(),l.startTime{i=k("FCP"),n=v(r,i,be,e.reportAllChanges),Y((()=>{i.value=performance.now()-a.timeStamp,n(!0)}))})))}))},ve=[.1,.25],Pe=(r,e={})=>{let t=B();ne(te((()=>{let n,i=k("CLS",0),s=ee(e,q),a=d=>{for(let u of d)s.h(u);s.i>i.value&&(i.value=s.i,i.entries=s.o,n())},l=L("layout-shift",a);l&&(n=v(r,i,ve,e.reportAllChanges),t.onHidden((()=>{a(l.takeRecords()),n(!0)})),I((()=>{s.i=0,i=k("CLS",0),n=v(r,i,ve,e.reportAllChanges),Y((()=>n()))})),setTimeout(n))})))},Ie=0,z=1/0,M=0,Xe=r=>{for(let e of r)e.interactionId&&(z=Math.min(z,e.interactionId),M=Math.max(M,e.interactionId),Ie=M?(M-z)/7+1:0)},K,ke=()=>K?Ie:performance.interactionCount??0,Qe=()=>{"interactionCount"in performance||K||(K=L("event",Xe,{type:"event",buffered:!0,durationThreshold:0}))},we=0,J=class{constructor(){R(this,"u",[]);R(this,"l",new Map);R(this,"m");R(this,"p")}v(){we=ke(),this.u.length=0,this.l.clear()}L(){let e=Math.min(this.u.length-1,Math.floor((ke()-we)/50));return this.u[e]}h(e){if(this.m?.(e),!e.interactionId&&e.entryType!=="first-input")return;let t=this.u.at(-1),n=this.l.get(e.interactionId);if(n||this.u.length<10||e.duration>t.P){if(n?e.duration>n.P?(n.entries=[e],n.P=e.duration):e.duration===n.P&&e.startTime===n.entries[0].startTime&&n.entries.push(e):(n={id:e.interactionId,entries:[e],P:e.duration},this.l.set(n.id,n),this.u.push(n)),this.u.sort(((i,s)=>s.P-i.P)),this.u.length>10){let i=this.u.splice(10);for(let s of i)this.l.delete(s.id)}this.p?.(n)}}},_e=r=>{let e=globalThis.requestIdleCallback||setTimeout;document.visibilityState==="hidden"?r():(r=te(r),addEventListener("visibilitychange",r,{once:!0,capture:!0}),e((()=>{r(),removeEventListener("visibilitychange",r,{capture:!0})})))},Re=[200,500],Le=(r,e={})=>{if(!globalThis.PerformanceEventTiming||!("interactionId"in PerformanceEventTiming.prototype))return;let t=B();O((()=>{Qe();let n,i=k("INP"),s=ee(e,J),a=d=>{_e((()=>{for(let h of d)s.h(h);let u=s.L();u&&u.P!==i.value&&(i.value=u.P,i.entries=u.entries,n())}))},l=L("event",a,{durationThreshold:e.durationThreshold??40});n=v(r,i,Re,e.reportAllChanges),l&&(l.observe({type:"first-input",buffered:!0}),t.onHidden((()=>{a(l.takeRecords()),n(!0)})),I((()=>{s.v(),i=k("INP"),n=v(r,i,Re,e.reportAllChanges)})))}))},X=class{constructor(){R(this,"m")}h(e){this.m?.(e)}},Se=[2500,4e3],Ae=(r,e={})=>{O((()=>{let t=B(),n,i=k("LCP"),s=ee(e,X),a=d=>{e.reportAllChanges||(d=d.slice(-1));for(let u of d)s.h(u),u.startTime{a(l.takeRecords()),l.disconnect(),n(!0)})),u=h=>{h.isTrusted&&(_e(d),removeEventListener(h.type,u,{capture:!0}))};for(let h of["keydown","click","visibilitychange"])addEventListener(h,u,{capture:!0});I((h=>{i=k("LCP"),n=v(r,i,Se,e.reportAllChanges),Y((()=>{i.value=performance.now()-h.timeStamp,n(!0)}))}))}}))},Te=[800,1800],Q=r=>{document.prerendering?O((()=>Q(r))):document.readyState!=="complete"?addEventListener("load",(()=>Q(r)),!0):setTimeout(r)},Fe=(r,e={})=>{let t=k("TTFB"),n=v(r,t,Te,e.reportAllChanges);Q((()=>{let i=Z();i&&(t.value=Math.max(i.responseStart-_(),0),t.entries=[i],n(!0),I((()=>{t=k("TTFB",0),n=v(r,t,Te,e.reportAllChanges),n(!0)})))}))};var N=class{constructor(e){this.data={lcp:null,cls:null,inp:null,fcp:null,ttfb:null};this.sent=!1;this.timeout=null;this.onReadyCallback=null;this.onReadyCallback=e}initialize(){try{Ae(this.collectMetric.bind(this)),Pe(this.collectMetric.bind(this)),Le(this.collectMetric.bind(this)),ne(this.collectMetric.bind(this)),Fe(this.collectMetric.bind(this)),this.timeout=setTimeout(()=>{this.sent||this.sendData()},2e4),window.addEventListener("beforeunload",()=>{this.sent||this.sendData()})}catch(e){console.warn("Error initializing web vitals tracking:",e)}}collectMetric(e){if(this.sent)return;let t=e.name.toLowerCase();this.data[t]=e.value,Object.values(this.data).every(i=>i!==null)&&this.sendData()}sendData(){this.sent||(this.sent=!0,this.timeout&&(clearTimeout(this.timeout),this.timeout=null),this.onReadyCallback&&this.onReadyCallback(this.data))}getData(){return{...this.data}}};var D=class{constructor(e,t){this.tracker=e,this.config=t}initialize(){document.addEventListener("click",this.handleClick.bind(this),!0)}handleClick(e){let t=e.target;this.config.trackButtonClicks&&this.isButton(t)&&this.trackButtonClick(t)}isButton(e){if(e.tagName==="BUTTON"||e.getAttribute("role")==="button")return!0;if(e.tagName==="INPUT"){let i=e.type?.toLowerCase();if(i==="submit"||i==="button")return!0}let t=e.parentElement,n=0;for(;t&&n<3;){if(t.tagName==="BUTTON"||t.getAttribute("role")==="button")return!0;t=t.parentElement,n++}return!1}trackButtonClick(e){let t=this.findButton(e);if(!t||t.hasAttribute("data-rybbit-event"))return;let n={text:this.getElementText(t),...this.extractDataAttributes(t)};this.tracker.trackButtonClick(n)}extractDataAttributes(e){let t={};for(let n of e.attributes)if(n.name.startsWith("data-rybbit-prop-")){let i=n.name.replace("data-rybbit-prop-","");t[i]=n.value}return t}findButton(e){if(e.tagName==="BUTTON"||e.getAttribute("role")==="button")return e;if(e.tagName==="INPUT"){let i=e.type?.toLowerCase();if(i==="submit"||i==="button")return e}let t=e.parentElement,n=0;for(;t&&n<3;){if(t.tagName==="BUTTON"||t.getAttribute("role")==="button")return t;t=t.parentElement,n++}return null}getElementText(e){let t=e.textContent?.trim().substring(0,100);if(t)return t;let n=e.getAttribute("aria-label")?.trim().substring(0,100);if(n)return n;if(e.tagName==="INPUT"){let s=e.value?.trim().substring(0,100);if(s)return s}let i=e.getAttribute("title")?.trim().substring(0,100);if(i)return i}cleanup(){document.removeEventListener("click",this.handleClick.bind(this),!0)}};var U=class{constructor(e){this.tracker=e}initialize(){document.addEventListener("copy",this.handleCopy.bind(this))}handleCopy(){let e=window.getSelection();if(!e||e.isCollapsed)return;let t=e.toString(),n=t.length;if(n===0)return;let i=e.anchorNode,s=i instanceof HTMLElement?i:i?.parentElement;if(!s)return;let a={text:t.substring(0,500),...n>500&&{textLength:n},sourceElement:s.tagName.toLowerCase()};this.tracker.trackCopy(a)}cleanup(){document.removeEventListener("copy",this.handleCopy.bind(this))}};var H=class{constructor(e,t){this.tracker=e,this.config=t,this.boundHandleSubmit=this.handleSubmit.bind(this),this.boundHandleChange=this.handleChange.bind(this)}initialize(){document.addEventListener("submit",this.boundHandleSubmit,!0),document.addEventListener("change",this.boundHandleChange,!0)}cleanup(){document.removeEventListener("submit",this.boundHandleSubmit,!0),document.removeEventListener("change",this.boundHandleChange,!0)}handleSubmit(e){let t=e.target;if(t.tagName!=="FORM")return;let n={formId:t.id||"",formName:t.name||"",formAction:t.action||"",method:(t.method||"get").toUpperCase(),fieldCount:t.elements.length,ariaLabel:t.getAttribute("aria-label")||void 0,...this.extractDataAttributes(t)};this.tracker.trackFormSubmit(n)}handleChange(e){let t=e.target,n=t.tagName.toUpperCase();if(!["INPUT","SELECT","TEXTAREA"].includes(n)||t.disabled||t.readOnly)return;if(n==="INPUT"){let a=t.type?.toLowerCase();if(a==="hidden"||a==="password")return}let i=t.name||t.id||t.getAttribute("aria-label")||t.placeholder||"",s={element:n.toLowerCase(),inputType:n==="INPUT"?t.type?.toLowerCase():void 0,inputName:i,formId:t.form?.id||void 0,formName:t.form?.name||void 0,...this.extractDataAttributes(t)};this.tracker.trackInputChange(s)}extractDataAttributes(e){let t={};for(let n of e.attributes)if(n.name.startsWith("data-rybbit-prop-")){let i=n.name.replace("data-rybbit-prop-","");t[i]=n.value}return t}};(async function(){let r=document.currentScript;if(!r){console.error("Could not find current script tag");return}let e=r.getAttribute("data-namespace")||"rybbit",t=`disable-${e}`;if(window.__RYBBIT_OPTOUT__||localStorage.getItem(t)!==null){window[e]={pageview:()=>{},event:()=>{},error:()=>{},trackOutbound:()=>{},identify:()=>{},setTraits:()=>{},clearUserId:()=>{},getUserId:()=>null,flag:(o,c)=>c,flagPayload:(o,c)=>c,flags:()=>({}),flagPayloads:()=>({}),onReady:()=>{},startSessionReplay:()=>{},stopSessionReplay:()=>{},isSessionReplayActive:()=>!1};return}let n=[],i=o=>(...c)=>{n.push([o,c])};window[e]={pageview:i("pageview"),event:i("event"),error:i("error"),trackOutbound:i("trackOutbound"),identify:i("identify"),setTraits:i("setTraits"),clearUserId:i("clearUserId"),getUserId:()=>null,flag:(o,c)=>c,flagPayload:(o,c)=>c,flags:()=>({}),flagPayloads:()=>({}),onReady:i("onReady"),startSessionReplay:i("startSessionReplay"),stopSessionReplay:i("stopSessionReplay"),isSessionReplayActive:()=>!1};let s=await de(r);if(!s)return;let a=new x(s);s.enableWebVitals&&new N(c=>{a.trackWebVitals(c)}).initialize();let l=null,d=null,u=null;s.trackButtonClicks&&(l=new D(a,s),l.initialize()),s.trackCopy&&(d=new U(a),d.initialize()),s.trackFormInteractions&&(u=new H(a,s),u.initialize()),s.trackErrors&&(window.addEventListener("error",o=>{a.trackError(o.error||new Error(o.message),{filename:o.filename,lineno:o.lineno,colno:o.colno})}),window.addEventListener("unhandledrejection",o=>{let c=o.reason instanceof Error?o.reason:new Error(String(o.reason));a.trackError(c,{type:"unhandledrejection"})}));let h=()=>a.trackPageview(),g=s.debounceDuration>0?ce(h,s.debounceDuration):h;function C(){if(document.addEventListener("click",function(o){let c=o.target;for(;c&&c!==document.documentElement;){if(c.hasAttribute("data-rybbit-event")){let p=c.getAttribute("data-rybbit-event");if(p){let w={};for(let S of c.attributes)if(S.name.startsWith("data-rybbit-prop-")){let T=S.name.replace("data-rybbit-prop-","");w[T]=S.value}a.trackEvent(p,w)}break}c=c.parentElement}if(s.trackOutbound){let p=o.target.closest("a");p?.href&&le(p.href)&&a.trackOutbound(p.href,p.innerText||p.textContent||"",p.target||"_self")}}),s.autoTrackSpa){let o=history.pushState,c=history.replaceState;history.pushState=function(...p){o.apply(this,p),g(),a.onPageChange()},history.replaceState=function(...p){c.apply(this,p),g(),a.onPageChange()},window.addEventListener("popstate",()=>{g(),a.onPageChange()}),window.addEventListener("hashchange",()=>{g(),a.onPageChange()})}}window[s.namespace]={pageview:()=>a.trackPageview(),event:(o,c={})=>a.trackEvent(o,c),error:(o,c={})=>a.trackError(o,c),trackOutbound:(o,c="",p="_self")=>a.trackOutbound(o,c,p),identify:(o,c)=>a.identify(o,c),setTraits:o=>a.setTraits(o),clearUserId:()=>a.clearUserId(),getUserId:()=>a.getUserId(),flag:(o,c)=>a.getFeatureFlag(o,c),flagPayload:(o,c)=>a.getFeatureFlagPayload(o,c),flags:()=>a.getFeatureFlags(),flagPayloads:()=>a.getFeatureFlagPayloads(),onReady:o=>o(window[s.namespace]),startSessionReplay:()=>a.startSessionReplay(),stopSessionReplay:()=>a.stopSessionReplay(),isSessionReplayActive:()=>a.isSessionReplayActive()};let m=window[s.namespace];for(let[o,c]of n)m[o](...c);C(),window.addEventListener("beforeunload",()=>{l?.cleanup(),d?.cleanup(),a.cleanup()}),s.autoTrackPageview&&a.trackPageview()})();})(); diff --git a/server/src/analytics-script/sessionReplay.test.ts b/server/src/analytics-script/sessionReplay.test.ts index 157f3ca0f..ea0512409 100644 --- a/server/src/analytics-script/sessionReplay.test.ts +++ b/server/src/analytics-script/sessionReplay.test.ts @@ -70,4 +70,35 @@ describe("SessionReplayRecorder identity", () => { events: [{ data: { user: "employee-bob" } }], }); }); + + it("uses a stable batch id when retrying a failed delivery", async () => { + sendBatch.mockRejectedValueOnce(new Error("network failed")).mockResolvedValueOnce(undefined); + emit({ type: 2, data: { retry: true }, timestamp: 1_700_000_000_000 }); + + recorder.onPageChange(); + await vi.waitFor(() => expect(sendBatch).toHaveBeenCalledTimes(1)); + const firstAttempt = sendBatch.mock.calls[0][0]; + + recorder.onPageChange(); + await vi.waitFor(() => expect(sendBatch).toHaveBeenCalledTimes(2)); + const secondAttempt = sendBatch.mock.calls[1][0]; + + expect(firstAttempt.batchId).toMatch(/^[0-9a-f-]{36}$/i); + expect(secondAttempt.batchId).toBe(firstAttempt.batchId); + expect(secondAttempt.events).toEqual(firstAttempt.events); + }); + + it("assigns globally increasing sequence numbers across batches", async () => { + (config as ScriptConfig).sessionReplayBatchSize = 1; + recorder.cleanup(); + recorder = new SessionReplayRecorder(config, "employee-alice", sendBatch); + await recorder.initialize(); + + emit({ type: 3, data: { order: 1 }, timestamp: 1_700_000_000_000 }); + emit({ type: 3, data: { order: 2 }, timestamp: 1_700_000_000_000 }); + + await vi.waitFor(() => expect(sendBatch).toHaveBeenCalledTimes(2)); + expect(sendBatch.mock.calls.map(call => call[0].events[0].sequence)).toEqual([0, 1]); + expect(new Set(sendBatch.mock.calls.map(call => call[0].batchId)).size).toBe(2); + }); }); diff --git a/server/src/analytics-script/sessionReplay.ts b/server/src/analytics-script/sessionReplay.ts index 1b8d0c738..1f6966f77 100644 --- a/server/src/analytics-script/sessionReplay.ts +++ b/server/src/analytics-script/sessionReplay.ts @@ -2,6 +2,21 @@ import { ScriptConfig, SessionReplayEvent, SessionReplayBatch } from "./types.js const SAMPLE_STORAGE_KEY = "rybbit-replay-sampled"; +function createReplayBatchId(): string { + const bytes = new Uint8Array(16); + if (globalThis.crypto?.getRandomValues) { + globalThis.crypto.getRandomValues(bytes); + } else { + for (let index = 0; index < bytes.length; index++) { + bytes[index] = Math.floor(Math.random() * 256); + } + } + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + const hex = [...bytes].map(byte => byte.toString(16).padStart(2, "0")); + return `${hex.slice(0, 4).join("")}-${hex.slice(4, 6).join("")}-${hex.slice(6, 8).join("")}-${hex.slice(8, 10).join("")}-${hex.slice(10).join("")}`; +} + /** * Determines if this session should have replay enabled based on sample rate. * Uses sessionStorage to persist the decision for the entire browser session. @@ -60,6 +75,9 @@ export class SessionReplayRecorder { private stopRecordingFn?: () => void; private userId: string; private eventBuffer: SessionReplayEvent[] = []; + private pendingBatches: SessionReplayBatch[] = []; + private nextSequence = 0; + private sendLoop: Promise | null = null; private batchTimer?: number; private sendBatch: (batch: SessionReplayBatch) => Promise; @@ -156,11 +174,11 @@ export class SessionReplayRecorder { checkoutEveryNms: 60000, // Checkout every 60 seconds checkoutEveryNth: 500, // Checkout every 500 events // Use config values with fallbacks to defaults - blockClass: this.config.sessionReplayBlockClass ?? 'rr-block', + blockClass: this.config.sessionReplayBlockClass ?? "rr-block", blockSelector: this.config.sessionReplayBlockSelector ?? null, - ignoreClass: this.config.sessionReplayIgnoreClass ?? 'rr-ignore', + ignoreClass: this.config.sessionReplayIgnoreClass ?? "rr-ignore", ignoreSelector: this.config.sessionReplayIgnoreSelector ?? null, - maskTextClass: this.config.sessionReplayMaskTextClass ?? 'rr-mask', + maskTextClass: this.config.sessionReplayMaskTextClass ?? "rr-mask", maskAllInputs: this.config.sessionReplayMaskAllInputs ?? true, maskInputOptions: this.config.sessionReplayMaskInputOptions ?? { password: true, email: true }, collectFonts: this.config.sessionReplayCollectFonts ?? true, @@ -170,7 +188,7 @@ export class SessionReplayRecorder { // Add custom text masking selectors if configured if (this.config.sessionReplayMaskTextSelectors && this.config.sessionReplayMaskTextSelectors.length > 0) { - recordingOptions.maskTextSelector = this.config.sessionReplayMaskTextSelectors.join(', '); + recordingOptions.maskTextSelector = this.config.sessionReplayMaskTextSelectors.join(", "); } this.stopRecordingFn = window.rrweb.record(recordingOptions); @@ -195,8 +213,8 @@ export class SessionReplayRecorder { this.clearBatchTimer(); // Send any remaining events - if (this.eventBuffer.length > 0) { - this.flushEvents(); + if (this.eventBuffer.length > 0 || this.pendingBatches.length > 0) { + void this.flushEvents(); } } @@ -205,7 +223,7 @@ export class SessionReplayRecorder { } private addEvent(event: SessionReplayEvent): void { - this.eventBuffer.push(event); + this.eventBuffer.push({ ...event, sequence: this.nextSequence++ }); // Auto-flush if buffer is full if (this.eventBuffer.length >= this.config.sessionReplayBatchSize) { @@ -216,8 +234,8 @@ export class SessionReplayRecorder { private setupBatchTimer(): void { this.clearBatchTimer(); this.batchTimer = window.setInterval(() => { - if (this.eventBuffer.length > 0) { - this.flushEvents(); + if (this.eventBuffer.length > 0 || this.pendingBatches.length > 0) { + void this.flushEvents(); } }, this.config.sessionReplayBatchInterval); } @@ -230,30 +248,50 @@ export class SessionReplayRecorder { } private async flushEvents(): Promise { - if (this.eventBuffer.length === 0) { - return; + if (this.eventBuffer.length > 0) { + const batch: SessionReplayBatch = { + batchId: createReplayBatchId(), + userId: this.userId, + events: this.eventBuffer, + metadata: { + pageUrl: window.location.href, + viewportWidth: screen.width, + viewportHeight: screen.height, + language: navigator.language, + }, + }; + this.eventBuffer = []; + this.pendingBatches.push(batch); } - const events = [...this.eventBuffer]; - this.eventBuffer = []; - - const batch: SessionReplayBatch = { - userId: this.userId, - events, - metadata: { - pageUrl: window.location.href, - viewportWidth: screen.width, - viewportHeight: screen.height, - language: navigator.language, - }, - }; + await this.drainPendingBatches(); + } - try { - await this.sendBatch(batch); - } catch (error) { - // Re-queue the events for retry since this batch failed - this.eventBuffer.unshift(...events); + private drainPendingBatches(): Promise { + if (this.sendLoop) { + const activeLoop = this.sendLoop; + return activeLoop.then(() => { + if (this.pendingBatches.length > 0 && this.sendLoop === null) { + return this.drainPendingBatches(); + } + }); } + + this.sendLoop = (async () => { + while (this.pendingBatches.length > 0) { + try { + await this.sendBatch(this.pendingBatches[0]); + this.pendingBatches.shift(); + } catch { + // Keep the exact batch (including identity and batch id) for retry. + return; + } + } + })().finally(() => { + this.sendLoop = null; + }); + + return this.sendLoop; } // Update user ID when it changes @@ -276,7 +314,7 @@ export class SessionReplayRecorder { public onPageChange(): void { if (this.isRecording) { // Flush current events before page change - this.flushEvents(); + void this.flushEvents(); } } diff --git a/server/src/analytics-script/tracking.test.ts b/server/src/analytics-script/tracking.test.ts index e6f5707f1..443b33b0f 100644 --- a/server/src/analytics-script/tracking.test.ts +++ b/server/src/analytics-script/tracking.test.ts @@ -487,5 +487,27 @@ describe("Tracker", () => { consoleSpy.mockRestore(); }); + + it("waits for earlier tracking requests before sending identify", async () => { + let finishTracking!: () => void; + const trackingResponse = new Promise(resolve => { + finishTracking = () => resolve({ ok: true } as Response); + }); + vi.mocked(global.fetch) + .mockReturnValueOnce(trackingResponse) + .mockResolvedValueOnce({ ok: true } as Response) + .mockResolvedValue({ ok: true, json: async () => ({ flags: {} }) } as Response); + + tracker.trackPageview(); + tracker.identify("user-ordered"); + + expect(global.fetch).toHaveBeenCalledTimes(1); + expect(vi.mocked(global.fetch).mock.calls[0][0]).toBe("https://analytics.example.com/track"); + + finishTracking(); + await vi.waitFor(() => + expect(global.fetch).toHaveBeenCalledWith("https://analytics.example.com/identify", expect.any(Object)) + ); + }); }); }); diff --git a/server/src/analytics-script/tracking.ts b/server/src/analytics-script/tracking.ts index 998174288..afc7de418 100644 --- a/server/src/analytics-script/tracking.ts +++ b/server/src/analytics-script/tracking.ts @@ -17,6 +17,7 @@ export class Tracker { private config: ScriptConfig; private customUserId: string | null = null; private sessionReplayRecorder?: SessionReplayRecorder; + private pendingTrackingRequests = new Set>(); private errorDedupeCache: Map = new Map(); private errorDedupeLastCleanup = 0; private exposedFeatureFlags = new Set(); @@ -181,20 +182,26 @@ export class Tracker { return payload; } - async sendTrackingData(payload: TrackingPayload): Promise { - try { - await fetch(`${this.config.analyticsHost}/track`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - mode: "cors", - keepalive: true, - }); - } catch (error) { - console.error("Failed to send tracking data:", error); - } + sendTrackingData(payload: TrackingPayload): Promise { + const request = (async () => { + try { + await fetch(`${this.config.analyticsHost}/track`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + mode: "cors", + keepalive: true, + }); + } catch (error) { + console.error("Failed to send tracking data:", error); + } + })(); + + this.pendingTrackingRequests.add(request); + void request.finally(() => this.pendingTrackingRequests.delete(request)); + return request; } track(eventType: TrackingPayload["type"], eventName: string = "", properties: Record = {}): void { @@ -432,8 +439,13 @@ export class Tracker { console.warn("Could not persist user ID to localStorage"); } - // Send identify event to server (creates alias and stores traits) - void this.sendIdentifyEvent(this.customUserId, traits, true).then(() => this.refreshFeatureFlags()); + // The server only acknowledges tracking once ClickHouse has stored it. Wait + // for older anonymous events before requesting their identify backfill. + const identifiedUserId = this.customUserId; + const earlierTrackingRequests = [...this.pendingTrackingRequests]; + void Promise.allSettled(earlierTrackingRequests) + .then(() => this.sendIdentifyEvent(identifiedUserId, traits, true)) + .then(() => this.refreshFeatureFlags()); // Update session replay recorder with new user ID if (this.sessionReplayRecorder) { diff --git a/server/src/analytics-script/types.ts b/server/src/analytics-script/types.ts index c2fb34d7b..6abc5534b 100644 --- a/server/src/analytics-script/types.ts +++ b/server/src/analytics-script/types.ts @@ -155,9 +155,11 @@ export interface SessionReplayEvent { type: string | number; data: any; timestamp: number; + sequence?: number; } export interface SessionReplayBatch { + batchId: string; userId: string; events: SessionReplayEvent[]; metadata?: { diff --git a/server/src/api/sessionReplay/recordSessionReplay.test.ts b/server/src/api/sessionReplay/recordSessionReplay.test.ts index 71a9b9e66..c1caa34cf 100644 --- a/server/src/api/sessionReplay/recordSessionReplay.test.ts +++ b/server/src/api/sessionReplay/recordSessionReplay.test.ts @@ -220,4 +220,20 @@ describe("recordSessionReplay exclusions", () => { message: "Session replay not recorded - user agent excluded", }); }); + + it("rejects viewport dimensions that cannot fit ClickHouse UInt16 columns", async () => { + const reply = createReply(); + await recordSessionReplay( + createRequest({ + body: { + ...baseBody, + metadata: { ...baseBody.metadata!, viewportWidth: 65_536 }, + }, + }), + reply + ); + + expect(reply.status).toHaveBeenCalledWith(400); + expect(mocks.recordEvents).not.toHaveBeenCalled(); + }); }); diff --git a/server/src/api/sessionReplay/recordSessionReplay.ts b/server/src/api/sessionReplay/recordSessionReplay.ts index 93b58a555..0a0e1b785 100644 --- a/server/src/api/sessionReplay/recordSessionReplay.ts +++ b/server/src/api/sessionReplay/recordSessionReplay.ts @@ -7,21 +7,26 @@ import { RecordSessionReplayRequest } from "../../types/sessionReplay.js"; import { resolveClientIp } from "../../services/tracker/resolveClientIp.js"; import { logger } from "../../lib/logger/logger.js"; import { getLocation } from "../../db/geolocation/geolocation.js"; +import { CLICKHOUSE_UINT16_CAPACITY, CLICKHOUSE_UINT16_MAX } from "../../lib/clickhouseLimits.js"; const recordSessionReplaySchema = z.object({ - userId: z.string(), - events: z.array( - z.object({ - type: z.union([z.string(), z.number()]), - data: z.any(), - timestamp: z.number(), - }) - ), + batchId: z.string().uuid().optional(), + userId: z.string().max(255), + events: z + .array( + z.object({ + type: z.union([z.string(), z.number()]), + data: z.any(), + timestamp: z.number().int().nonnegative(), + sequence: z.number().int().nonnegative().max(0xffff_ffff).optional(), + }) + ) + .max(CLICKHOUSE_UINT16_CAPACITY), metadata: z .object({ pageUrl: z.string(), - viewportWidth: z.number().optional(), - viewportHeight: z.number().optional(), + viewportWidth: z.number().int().nonnegative().max(CLICKHOUSE_UINT16_MAX).optional(), + viewportHeight: z.number().int().nonnegative().max(CLICKHOUSE_UINT16_MAX).optional(), language: z.string().optional(), }) .optional(), diff --git a/server/src/api/sites/batchImportEvents.test.ts b/server/src/api/sites/batchImportEvents.test.ts new file mode 100644 index 000000000..98b309092 --- /dev/null +++ b/server/src/api/sites/batchImportEvents.test.ts @@ -0,0 +1,104 @@ +import type { FastifyReply, FastifyRequest } from "fastify"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + completeImport: vi.fn(), + getImportById: vi.fn(), + insert: vi.fn(), + updateImportProgress: vi.fn(), + withOrganizationImportLock: vi.fn(async (_organizationId: string, work: () => Promise) => work()), +})); + +const dbChain = { + from: vi.fn(), + leftJoin: vi.fn(), + where: vi.fn(), + limit: vi.fn(), +}; +dbChain.from.mockReturnValue(dbChain); +dbChain.leftJoin.mockReturnValue(dbChain); +dbChain.where.mockReturnValue(dbChain); + +vi.mock("../../db/clickhouse/clickhouse.js", () => ({ clickhouse: { insert: mocks.insert } })); +vi.mock("../../db/postgres/postgres.js", () => ({ db: { select: vi.fn(() => dbChain) } })); +vi.mock("../../lib/const.js", () => ({ IS_CLOUD: false })); +vi.mock("../../services/import/importStatusManager.js", () => ({ + completeImport: mocks.completeImport, + getImportById: mocks.getImportById, + updateImportProgress: mocks.updateImportProgress, + withOrganizationImportLock: mocks.withOrganizationImportLock, +})); +vi.mock("../../services/import/importQuotaTracker.js", () => ({ + ImportQuotaTracker: { + create: vi.fn(async () => ({ canImportBatch: () => [0] })), + }, +})); + +import { batchImportEvents } from "./batchImportEvents.js"; + +const event = { + session_id: "4f7371e6-41f5-41af-9b57-82a1c8cfeaf8", + hostname: "example.com", + browser: "chrome", + os: "windows 10", + device: "desktop", + screen: "1920x1080", + language: "en-US", + country: "US", + region: "US-CA", + city: "Los Angeles", + url_path: "/", + url_query: "", + referrer_path: "", + referrer_domain: "", + page_title: "Home", + event_type: "1", + event_name: "", + distinct_id: "visitor-1", + created_at: "2026-07-12 12:00:00", +}; + +function replyStub(): FastifyReply { + const reply = { + status: vi.fn(function (this: typeof reply) { + return this; + }), + send: vi.fn(function (this: typeof reply) { + return this; + }), + }; + return reply as unknown as FastifyReply; +} + +describe("batchImportEvents failure semantics", () => { + beforeEach(() => { + vi.clearAllMocks(); + dbChain.from.mockReturnValue(dbChain); + dbChain.leftJoin.mockReturnValue(dbChain); + dbChain.where.mockReturnValue(dbChain); + dbChain.limit.mockResolvedValue([{ organizationId: "org-1", stripeCustomerId: null }]); + mocks.getImportById.mockResolvedValue({ + importId: "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", + siteId: 42, + organizationId: "org-1", + platform: "umami", + completedAt: null, + }); + mocks.updateImportProgress.mockResolvedValue(undefined); + mocks.completeImport.mockResolvedValue(undefined); + }); + + it("does not mark a final batch complete when ClickHouse rejects it", async () => { + mocks.insert.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + const request = { + params: { siteId: 42, importId: "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4" }, + body: { events: [event], isLastBatch: true }, + } as unknown as FastifyRequest; + const reply = replyStub(); + + await batchImportEvents(request as any, reply); + + expect(reply.status).toHaveBeenCalledWith(500); + expect(mocks.completeImport).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts index b6a2eec4c..618b82486 100644 --- a/server/src/api/sites/batchImportEvents.ts +++ b/server/src/api/sites/batchImportEvents.ts @@ -1,11 +1,16 @@ import { FastifyReply, FastifyRequest } from "fastify"; import { z } from "zod"; import { clickhouse } from "../../db/clickhouse/clickhouse.js"; -import { updateImportProgress, completeImport, getImportById } from "../../services/import/importStatusManager.js"; +import { + updateImportProgress, + completeImport, + getImportById, + withOrganizationImportLock, +} from "../../services/import/importStatusManager.js"; import { UmamiEvent, UmamiImportMapper } from "../../services/import/mappers/umami.js"; import { SimpleAnalyticsEvent, SimpleAnalyticsImportMapper } from "../../services/import/mappers/simpleAnalytics.js"; import { PlausibleEvent, PlausibleImportMapper } from "../../services/import/mappers/plausible.js"; -import { importQuotaManager } from "../../services/import/importQuotaManager.js"; +import { ImportQuotaTracker } from "../../services/import/importQuotaTracker.js"; import { db } from "../../db/postgres/postgres.js"; import { organization, sites } from "../../db/postgres/schema.js"; import { eq } from "drizzle-orm"; @@ -70,9 +75,10 @@ export async function batchImportEvents(request: FastifyRequest e.timestamp); - const allowedIndices = quotaTracker.canImportBatch(timestamps); - - const eventsWithinQuota = allowedIndices.map(i => transformedEvents[i]); - const skippedDueToQuota = transformedEvents.length - eventsWithinQuota.length; - - if (eventsWithinQuota.length > 0) { - await clickhouse.insert({ - table: "events", - values: eventsWithinQuota, - format: "JSONEachRow", - }); - } - - await updateImportProgress(importId, eventsWithinQuota.length, skippedDueToQuota, invalidEventCount); + let transformedEvents; + if (importRecord.platform === "umami") { + transformedEvents = UmamiImportMapper.transform(events as UmamiEvent[], siteId, importId); + } else if (importRecord.platform === "simple_analytics") { + transformedEvents = SimpleAnalyticsImportMapper.transform(events as SimpleAnalyticsEvent[], siteId, importId); + } else if (importRecord.platform === "plausible") { + transformedEvents = PlausibleImportMapper.transform(events as PlausibleEvent[], siteId, importId); + } else { + return reply.status(400).send({ error: "Unsupported platform" }); + } + const invalidEventCount = events.length - transformedEvents.length; - if (isLastBatch) { - await completeImport(importId); - importQuotaManager.completeImport(siteRecord.organizationId); - } + try { + await withOrganizationImportLock(organizationId, async () => { + const currentImport = await getImportById(importId); + if (!currentImport || currentImport.completedAt !== null) { + throw new Error("IMPORT_ALREADY_COMPLETED"); + } + + // Recompute from ClickHouse while holding the cross-worker lock. The + // tracker is discarded on failure, so reservations cannot leak. + const quotaTracker = await ImportQuotaTracker.create(organizationId); + const timestamps = transformedEvents.map(event => event.timestamp); + const allowedIndices = quotaTracker.canImportBatch(timestamps); + const eventsWithinQuota = allowedIndices.map(index => transformedEvents[index]); + const skippedDueToQuota = transformedEvents.length - eventsWithinQuota.length; + + if (eventsWithinQuota.length > 0) { + await clickhouse.insert({ + table: "events", + values: eventsWithinQuota, + format: "JSONEachRow", + }); + } + + await updateImportProgress(importId, eventsWithinQuota.length, skippedDueToQuota, invalidEventCount); + + if (isLastBatch) { + await completeImport(importId); + } + }); return reply.send(); } catch (insertError) { const errorMessage = insertError instanceof Error ? insertError.message : "Unknown error"; console.error("Failed to insert events:", errorMessage); - if (isLastBatch) { - await completeImport(importId); - importQuotaManager.completeImport(siteRecord.organizationId); + if (errorMessage === "IMPORT_ALREADY_COMPLETED") { + return reply.status(409).send({ error: "Import is already complete" }); } return reply.status(500).send({ diff --git a/server/src/api/sites/createSiteImport.ts b/server/src/api/sites/createSiteImport.ts index 56c26925f..bfb1cf5ae 100644 --- a/server/src/api/sites/createSiteImport.ts +++ b/server/src/api/sites/createSiteImport.ts @@ -1,6 +1,6 @@ import { FastifyReply, FastifyRequest } from "fastify"; -import { importQuotaManager } from "../../services/import/importQuotaManager.js"; import { createImport } from "../../services/import/importStatusManager.js"; +import { ImportQuotaTracker } from "../../services/import/importQuotaTracker.js"; import { DateTime } from "luxon"; import { z } from "zod"; import { db } from "../../db/postgres/postgres.js"; @@ -65,12 +65,8 @@ export async function createSiteImport(request: FastifyRequest { CREATE TABLE IF NOT EXISTS session_replay_events ( site_id UInt16, session_id String, + batch_id String DEFAULT '', user_id String, timestamp DateTime64(3), event_type LowCardinality(String), @@ -218,6 +219,7 @@ export const initializeClickhouse = async () => { await clickhouse.exec({ query: ` ALTER TABLE session_replay_events + ADD COLUMN IF NOT EXISTS batch_id String DEFAULT '', ADD COLUMN IF NOT EXISTS event_data_key Nullable(String), -- R2 storage key for cloud deployments ADD COLUMN IF NOT EXISTS batch_index Nullable(UInt16), -- Index within the R2 batch ADD COLUMN IF NOT EXISTS identified_user_id String DEFAULT '' diff --git a/server/src/index.ts b/server/src/index.ts index a2dbb864d..39af13726 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -171,8 +171,11 @@ import { auth } from "./lib/auth.js"; import { createCorsOptionsDelegate, createRejectUntrustedOriginHook } from "./lib/cors.js"; import { IS_CLOUD } from "./lib/const.js"; import { reengagementService } from "./services/reengagement/reengagementService.js"; +import { sessionsService } from "./services/sessions/sessionsService.js"; import { telemetryService } from "./services/telemetryService.js"; +import { botEventQueue } from "./services/tracker/botBlocking/botEventQueue.js"; import { handleIdentify } from "./services/tracker/identifyService.js"; +import { pageviewQueue } from "./services/tracker/pageviewQueue.js"; import { trackEvent } from "./services/tracker/trackEvent.js"; import { usageService } from "./services/usageService.js"; import { weeklyReportService } from "./services/weekyReports/weeklyReportService.js"; @@ -571,6 +574,9 @@ const shutdown = async (signal: string) => { await server.close(); server.log.info("Server closed"); + await Promise.all([pageviewQueue.close(), botEventQueue.close(), sessionsService.close()]); + server.log.info("Ingestion queues drained"); + // Shutdown uptime service // await uptimeService.shutdown(); // server.log.info("Uptime service shut down"); diff --git a/server/src/lib/clickhouseLimits.ts b/server/src/lib/clickhouseLimits.ts new file mode 100644 index 000000000..acbe41ec7 --- /dev/null +++ b/server/src/lib/clickhouseLimits.ts @@ -0,0 +1,2 @@ +export const CLICKHOUSE_UINT16_MAX = 65_535; +export const CLICKHOUSE_UINT16_CAPACITY = CLICKHOUSE_UINT16_MAX + 1; diff --git a/server/src/services/import/importQuotaManager.ts b/server/src/services/import/importQuotaManager.ts deleted file mode 100644 index fb52a7510..000000000 --- a/server/src/services/import/importQuotaManager.ts +++ /dev/null @@ -1,93 +0,0 @@ -import cluster from "node:cluster"; -import { ImportQuotaTracker } from "./importQuotaTracker.js"; -import { IS_CLOUD } from "../../lib/const.js"; - -interface CachedTracker { - tracker: ImportQuotaTracker; - lastAccessed: number; -} - -class ImportQuotaManager { - private trackers = new Map(); - private activeImports = new Map(); // stores startedAt timestamp - private trackerCreationLocks = new Map>(); // prevents duplicate tracker creation - - private readonly CONCURRENT_IMPORT_LIMIT = 1; - private readonly TRACKER_TTL_MS = 30 * 60 * 1000; // 30 minutes - private readonly IMPORT_TIMEOUT_MS = 2 * 60 * 60 * 1000; // 2 hours - - async getTracker(organizationId: string): Promise { - const now = Date.now(); - const cached = this.trackers.get(organizationId); - - if (cached && now - cached.lastAccessed < this.TRACKER_TTL_MS) { - cached.lastAccessed = now; - return cached.tracker; - } - - // Check if tracker is currently being created - const existingCreation = this.trackerCreationLocks.get(organizationId); - if (existingCreation) { - return await existingCreation; - } - - // Create lock for this organization's tracker creation - const creationPromise = (async () => { - try { - const tracker = await ImportQuotaTracker.create(organizationId); - this.trackers.set(organizationId, { tracker, lastAccessed: now }); - return tracker; - } finally { - this.trackerCreationLocks.delete(organizationId); - } - })(); - - this.trackerCreationLocks.set(organizationId, creationPromise); - return await creationPromise; - } - - startImport(organizationId: string): boolean { - if (!IS_CLOUD) return true; - - const now = Date.now(); - const startedAt = this.activeImports.get(organizationId); - - // If an import exists and hasn't timed out, limit reached - if (startedAt && now - startedAt < this.IMPORT_TIMEOUT_MS && this.CONCURRENT_IMPORT_LIMIT === 1) { - return false; - } - - // Otherwise start a new import - this.activeImports.set(organizationId, now); - return true; - } - - completeImport(organizationId: string): void { - this.activeImports.delete(organizationId); - } - - cleanup(): void { - const now = Date.now(); - - // Cleanup stale trackers - for (const [orgId, cached] of this.trackers) { - if (now - cached.lastAccessed > this.TRACKER_TTL_MS) { - this.trackers.delete(orgId); - } - } - - // Cleanup timed-out imports - for (const [orgId, startedAt] of this.activeImports) { - if (now - startedAt > this.IMPORT_TIMEOUT_MS) { - this.activeImports.delete(orgId); - } - } - } -} - -export const importQuotaManager = new ImportQuotaManager(); - -// Background cleanup loop -if (IS_CLOUD && !cluster.isWorker) { - setInterval(() => importQuotaManager.cleanup(), 15 * 60 * 1000); -} diff --git a/server/src/services/import/importStatusManager.test.ts b/server/src/services/import/importStatusManager.test.ts new file mode 100644 index 000000000..ddc01854f --- /dev/null +++ b/server/src/services/import/importStatusManager.test.ts @@ -0,0 +1,86 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + const findActive = vi.fn(); + const forUpdate = vi.fn(); + const insertReturning = vi.fn(); + const transaction = vi.fn(); + const updateWhere = vi.fn(); + + const lockChain = { from: vi.fn(), where: vi.fn(), for: forUpdate }; + lockChain.from.mockReturnValue(lockChain); + lockChain.where.mockReturnValue(lockChain); + + const updateChain = { set: vi.fn(), where: updateWhere }; + updateChain.set.mockReturnValue(updateChain); + + const insertChain = { values: vi.fn(), returning: insertReturning }; + insertChain.values.mockReturnValue(insertChain); + + const db = { + insert: vi.fn(() => insertChain), + query: { importStatus: { findFirst: findActive } }, + transaction, + update: vi.fn(() => updateChain), + }; + + return { + db, + findActive, + forUpdate, + insertChain, + insertReturning, + lockChain, + transaction, + updateChain, + updateWhere, + }; +}); + +vi.mock("../../db/postgres/postgres.js", () => ({ db: mocks.db })); + +import { createImport } from "./importStatusManager.js"; + +describe("cluster-safe import creation", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.lockChain.from.mockReturnValue(mocks.lockChain); + mocks.lockChain.where.mockReturnValue(mocks.lockChain); + mocks.updateChain.set.mockReturnValue(mocks.updateChain); + mocks.insertChain.values.mockReturnValue(mocks.insertChain); + mocks.forUpdate.mockResolvedValue([{ id: "org-1" }]); + mocks.updateWhere.mockResolvedValue(undefined); + mocks.transaction.mockImplementation(async callback => callback({ select: () => mocks.lockChain })); + }); + + it("locks the organization row and rejects a second active import", async () => { + mocks.findActive.mockResolvedValue({ importId: "existing" }); + + const result = await createImport({ + siteId: 42, + organizationId: "org-1", + platform: "umami", + enforceSingleActive: true, + }); + + expect(result).toBeNull(); + expect(mocks.forUpdate).toHaveBeenCalledWith("update"); + expect(mocks.db.insert).not.toHaveBeenCalled(); + }); + + it("creates the import while holding the same organization lock", async () => { + mocks.findActive.mockResolvedValue(undefined); + mocks.insertReturning.mockResolvedValue([{ importId: "new-import" }]); + + const result = await createImport({ + siteId: 42, + organizationId: "org-1", + platform: "umami", + enforceSingleActive: true, + }); + + expect(result).toEqual({ importId: "new-import" }); + expect(mocks.forUpdate).toHaveBeenCalledWith("update"); + expect(mocks.db.insert).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/services/import/importStatusManager.ts b/server/src/services/import/importStatusManager.ts index 7c0d35067..d0d8f24b0 100644 --- a/server/src/services/import/importStatusManager.ts +++ b/server/src/services/import/importStatusManager.ts @@ -1,25 +1,73 @@ import { db } from "../../db/postgres/postgres.js"; -import { eq, desc, sql } from "drizzle-orm"; -import { importPlatforms, importStatus } from "../../db/postgres/schema.js"; +import { and, desc, eq, isNull, lt, sql } from "drizzle-orm"; +import { importPlatforms, importStatus, organization } from "../../db/postgres/schema.js"; import { DateTime } from "luxon"; export type SelectImportStatus = typeof importStatus.$inferSelect; +const IMPORT_TIMEOUT_HOURS = 2; + +/** Serialize import state changes across every worker using the organization row. */ +export async function withOrganizationImportLock(organizationId: string, work: () => Promise): Promise { + return db.transaction(async tx => { + const [lockedOrganization] = await tx + .select({ id: organization.id }) + .from(organization) + .where(eq(organization.id, organizationId)) + .for("update"); + + if (!lockedOrganization) { + throw new Error(`Organization ${organizationId} not found`); + } + + return work(); + }); +} + export async function createImport(data: { siteId: number; organizationId: string; platform: (typeof importPlatforms)[number]; -}): Promise<{ importId: string }> { - const [result] = await db - .insert(importStatus) - .values({ - siteId: data.siteId, - organizationId: data.organizationId, - platform: data.platform, - }) - .returning({ importId: importStatus.importId }); + enforceSingleActive?: boolean; +}): Promise<{ importId: string } | null> { + const insertImport = async () => { + const [result] = await db + .insert(importStatus) + .values({ + siteId: data.siteId, + organizationId: data.organizationId, + platform: data.platform, + }) + .returning({ importId: importStatus.importId }); - return result; + return result; + }; + + if (!data.enforceSingleActive) return insertImport(); + + return withOrganizationImportLock(data.organizationId, async () => { + const now = DateTime.utc(); + const staleCutoff = now.minus({ hours: IMPORT_TIMEOUT_HOURS }).toSQL({ includeOffset: false }); + if (!staleCutoff) throw new Error("Failed to calculate import timeout"); + + await db + .update(importStatus) + .set({ completedAt: now.toISO() }) + .where( + and( + eq(importStatus.organizationId, data.organizationId), + isNull(importStatus.completedAt), + lt(importStatus.startedAt, staleCutoff) + ) + ); + + const activeImport = await db.query.importStatus.findFirst({ + where: and(eq(importStatus.organizationId, data.organizationId), isNull(importStatus.completedAt)), + }); + if (activeImport) return null; + + return insertImport(); + }); } export async function updateImportProgress( diff --git a/server/src/services/import/mappers/simpleAnalytics.ts b/server/src/services/import/mappers/simpleAnalytics.ts index 01e9c91d0..f8d86bfac 100644 --- a/server/src/services/import/mappers/simpleAnalytics.ts +++ b/server/src/services/import/mappers/simpleAnalytics.ts @@ -6,6 +6,7 @@ import { UAParser } from "ua-parser-js"; import { DateTime } from "luxon"; import { getDeviceType } from "../../../utils.js"; import { deriveKeyOnlySchema } from "./utils.js"; +import { CLICKHOUSE_UINT16_MAX } from "../../../lib/clickhouseLimits.js"; export type SimpleAnalyticsEvent = z.input; @@ -26,8 +27,14 @@ export class SimpleAnalyticsImportMapper { .string() .max(2048) .transform(querystring => (querystring ? `?${querystring}` : "")), - screen_height: z.string().regex(/^\d+$/), - screen_width: z.string().regex(/^\d+$/), + screen_height: z + .string() + .regex(/^\d+$/) + .refine(value => Number(value) <= CLICKHOUSE_UINT16_MAX), + screen_width: z + .string() + .regex(/^\d+$/) + .refine(value => Number(value) <= CLICKHOUSE_UINT16_MAX), session_id: z.string().uuid(), user_agent: z.string().max(1024), uuid: z.string().uuid(), diff --git a/server/src/services/import/mappers/umami.ts b/server/src/services/import/mappers/umami.ts index 5e2f81d8a..3ab883e0a 100644 --- a/server/src/services/import/mappers/umami.ts +++ b/server/src/services/import/mappers/umami.ts @@ -3,6 +3,7 @@ import { getChannel } from "../../tracker/getChannel.js"; import { RybbitEvent } from "./rybbit.js"; import { z } from "zod"; import { deriveKeyOnlySchema } from "./utils.js"; +import { CLICKHOUSE_UINT16_MAX } from "../../../lib/clickhouseLimits.js"; export type UmamiEvent = z.input; @@ -77,7 +78,11 @@ export class UmamiImportMapper { screen: z .string() .regex(/^\d{1,5}x\d{1,5}$/) - .or(z.literal("")), + .or(z.literal("")) + .refine(value => { + if (!value) return true; + return value.split("x").every(dimension => Number(dimension) <= CLICKHOUSE_UINT16_MAX); + }), language: z.string().max(35), country: z .string() diff --git a/server/src/services/replay/sessionReplayIngestService.test.ts b/server/src/services/replay/sessionReplayIngestService.test.ts index 2f405181d..028a0e87f 100644 --- a/server/src/services/replay/sessionReplayIngestService.test.ts +++ b/server/src/services/replay/sessionReplayIngestService.test.ts @@ -4,12 +4,16 @@ import type { RecordSessionReplayRequest } from "../../types/sessionReplay.js"; const mocks = vi.hoisted(() => ({ generateUserId: vi.fn(), insert: vi.fn(), + query: vi.fn(), + r2Enabled: vi.fn(), + storeBatch: vi.fn(), updateSession: vi.fn(), })); vi.mock("../../db/clickhouse/clickhouse.js", () => ({ clickhouse: { insert: mocks.insert, + query: mocks.query, }, })); @@ -27,7 +31,8 @@ vi.mock("../userId/userIdService.js", () => ({ vi.mock("../storage/r2StorageService.js", () => ({ r2Storage: { - isEnabled: () => false, + isEnabled: mocks.r2Enabled, + storeBatch: mocks.storeBatch, }, })); @@ -61,6 +66,9 @@ describe("SessionReplayIngestService identity", () => { }) ); mocks.insert.mockResolvedValue(undefined); + mocks.query.mockResolvedValue({ json: async () => [] }); + mocks.r2Enabled.mockReturnValue(false); + mocks.storeBatch.mockResolvedValue("unused"); }); it("separates identified replay users behind a shared proxy", async () => { @@ -100,4 +108,69 @@ describe("SessionReplayIngestService identity", () => { siteId: 42, }); }); + + it("stores a stable batch id and the recorder's global sequence number", async () => { + const service = new SessionReplayIngestService(); + await service.recordEvents( + 42, + { + batchId: "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", + userId: "employee-alice", + events: [{ type: 3, data: { ordered: true }, timestamp: 1_700_000_000_000, sequence: 73 }], + }, + requestMeta + ); + + expect(mocks.insert).toHaveBeenCalledWith( + expect.objectContaining({ + values: [ + expect.objectContaining({ + batch_id: "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", + batch_index: 0, + sequence_number: 73, + }), + ], + }) + ); + }); + + it("does not insert a replay batch whose indices are already stored", async () => { + mocks.query.mockResolvedValue({ json: async () => [{ batch_index: 0 }] }); + const service = new SessionReplayIngestService(); + + await service.recordEvents( + 42, + { + batchId: "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", + userId: "employee-alice", + events: [{ type: 3, data: { ordered: true }, timestamp: 1_700_000_000_000, sequence: 73 }], + }, + requestMeta + ); + + expect(mocks.insert).not.toHaveBeenCalled(); + }); + + it("uses the stable batch id in the R2 object key", async () => { + mocks.r2Enabled.mockReturnValue(true); + mocks.storeBatch.mockResolvedValue("42/session/batch.json.zst"); + const service = new SessionReplayIngestService(); + + await service.recordEvents( + 42, + { + batchId: "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", + userId: "employee-alice", + events: [{ type: 3, data: { ordered: true }, timestamp: 1_700_000_000_000, sequence: 73 }], + }, + requestMeta + ); + + expect(mocks.storeBatch).toHaveBeenCalledWith( + 42, + "session-shared-fingerprint-employee-alice", + "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", + [{ ordered: true }] + ); + }); }); diff --git a/server/src/services/replay/sessionReplayIngestService.ts b/server/src/services/replay/sessionReplayIngestService.ts index 9ca52c4b6..32ac73a9d 100644 --- a/server/src/services/replay/sessionReplayIngestService.ts +++ b/server/src/services/replay/sessionReplayIngestService.ts @@ -1,3 +1,4 @@ +import crypto from "crypto"; import { DateTime } from "luxon"; import { clickhouse } from "../../db/clickhouse/clickhouse.js"; import { RecordSessionReplayRequest } from "../../types/sessionReplay.js"; @@ -26,6 +27,12 @@ export class SessionReplayIngestService { requestMeta?: RequestMetadata ): Promise { const { userId: clientUserId, events, metadata } = request; + const batchId = + request.batchId ?? + crypto + .createHash("sha256") + .update(JSON.stringify({ siteId, userId: clientUserId, events, metadata })) + .digest("hex"); // Always generate device fingerprint (anonymous user ID) server-side const deviceFingerprint = await userIdService.generateUserId( @@ -50,17 +57,36 @@ export class SessionReplayIngestService { siteId, }); + const existingBatchResult = await clickhouse.query({ + query: ` + SELECT batch_index + FROM session_replay_events + WHERE site_id = {siteId:UInt16} + AND session_id = {sessionId:String} + AND batch_id = {batchId:String} + `, + query_params: { siteId, sessionId, batchId }, + format: "JSONEachRow", + }); + const existingBatchRows = await processResults<{ batch_index: number | null }>(existingBatchResult); + const storedBatchIndices = new Set( + existingBatchRows.flatMap(row => (row.batch_index === null ? [] : [row.batch_index])) + ); + const missingEvents = events + .map((event, index) => ({ event, index })) + .filter(({ index }) => !storedBatchIndices.has(index)); + // Check if R2 storage is enabled for cloud deployments let r2BatchKey: string | null = null; let eventDataArray: any[] = []; - if (r2Storage.isEnabled()) { + if (missingEvents.length > 0 && r2Storage.isEnabled()) { // Extract event data for R2 storage eventDataArray = events.map(event => event.data); try { // Store event data batch in R2 - r2BatchKey = await r2Storage.storeBatch(siteId, sessionId, eventDataArray); + r2BatchKey = await r2Storage.storeBatch(siteId, sessionId, batchId, eventDataArray); } catch (error) { console.error("Failed to store in R2, falling back to ClickHouse:", error); r2BatchKey = null; @@ -68,7 +94,7 @@ export class SessionReplayIngestService { } // Prepare events for batch insert - const eventsToInsert = events.map((event, index) => { + const eventsToInsert = missingEvents.map(({ event, index }) => { const serializedData = JSON.stringify(event.data); if (r2BatchKey) { @@ -76,6 +102,7 @@ export class SessionReplayIngestService { return { site_id: siteId, session_id: sessionId, + batch_id: batchId, user_id: userId, identified_user_id: identifiedUserId, timestamp: event.timestamp, @@ -83,7 +110,7 @@ export class SessionReplayIngestService { event_data: "", // Empty string when using R2 event_data_key: r2BatchKey, batch_index: index, - sequence_number: index, + sequence_number: event.sequence ?? index, event_size_bytes: serializedData.length, viewport_width: metadata?.viewportWidth || null, viewport_height: metadata?.viewportHeight || null, @@ -94,14 +121,15 @@ export class SessionReplayIngestService { return { site_id: siteId, session_id: sessionId, + batch_id: batchId, user_id: userId, identified_user_id: identifiedUserId, timestamp: event.timestamp, event_type: event.type, event_data: serializedData, event_data_key: null, - batch_index: null, - sequence_number: index, + batch_index: index, + sequence_number: event.sequence ?? index, event_size_bytes: serializedData.length, viewport_width: metadata?.viewportWidth || null, viewport_height: metadata?.viewportHeight || null, diff --git a/server/src/services/replay/sessionReplayQueryService.ts b/server/src/services/replay/sessionReplayQueryService.ts index 2fdafc00b..1720afbc0 100644 --- a/server/src/services/replay/sessionReplayQueryService.ts +++ b/server/src/services/replay/sessionReplayQueryService.ts @@ -144,7 +144,8 @@ export class SessionReplayQueryService { event_type as type, event_data as data, event_data_key, - batch_index + batch_index, + sequence_number FROM session_replay_events WHERE site_id = {siteId:UInt16} AND session_id = {sessionId:String} @@ -160,6 +161,7 @@ export class SessionReplayQueryService { data: string; event_data_key: string | null; batch_index: number | null; + sequence_number: number; }; const eventsResults = await processResults(eventsResult); @@ -196,6 +198,7 @@ export class SessionReplayQueryService { timestamp: event.timestamp, type: event.type, data: JSON.parse(event.data), + sequence: event.sequence_number, }); } } @@ -225,11 +228,12 @@ export class SessionReplayQueryService { for (const { batchEvents, data } of r2Results) { if (data) { for (const event of batchEvents) { - if (event.batch_index !== null && data[event.batch_index]) { + if (event.batch_index !== null && event.batch_index < data.length) { events.push({ timestamp: event.timestamp, type: event.type, data: data[event.batch_index], + sequence: event.sequence_number, }); } } @@ -237,10 +241,10 @@ export class SessionReplayQueryService { } // Sort events by timestamp (in case batches were processed out of order) - events.sort((a, b) => a.timestamp - b.timestamp); + events.sort((a, b) => a.timestamp - b.timestamp || a.sequence - b.sequence); return { - events, + events: events.map(({ sequence: _sequence, ...event }) => event), metadata, }; } diff --git a/server/src/services/storage/r2StorageService.test.ts b/server/src/services/storage/r2StorageService.test.ts new file mode 100644 index 000000000..e152fd0e8 --- /dev/null +++ b/server/src/services/storage/r2StorageService.test.ts @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + process.env.R2_ACCESS_KEY_ID = "test-key"; + process.env.R2_SECRET_ACCESS_KEY = "test-secret"; + process.env.R2_ACCOUNT_ID = "test-account"; + process.env.R2_BUCKET_NAME = "test-bucket"; + return { sentCommands: [] as any[] }; +}); + +vi.mock("../../lib/const.js", () => ({ IS_CLOUD: true })); +vi.mock("../../lib/logger/logger.js", () => ({ + createServiceLogger: () => ({ info: vi.fn(), debug: vi.fn() }), +})); +vi.mock("@smithy/node-http-handler", () => ({ + NodeHttpHandler: class { + handle = vi.fn(); + }, +})); +vi.mock("@aws-sdk/client-s3", () => { + class Command { + constructor(public input: any) {} + } + return { + S3Client: class { + async send(command: any) { + mocks.sentCommands.push(command); + return {}; + } + }, + PutObjectCommand: Command, + GetObjectCommand: Command, + DeleteObjectCommand: Command, + }; +}); +vi.mock("@mongodb-js/zstd", () => ({ + compress: async (buffer: Buffer) => buffer, + decompress: async (buffer: Buffer) => buffer, +})); + +import { r2Storage } from "./r2StorageService.js"; + +describe("R2 replay batch keys", () => { + afterEach(() => { + mocks.sentCommands.length = 0; + }); + + it("uses the stable batch id rather than a collision-prone millisecond timestamp", async () => { + await r2Storage.storeBatch(42, "session-1", "batch-a", [{ batch: 1 }]); + await r2Storage.storeBatch(42, "session-1", "batch-b", [{ batch: 2 }]); + + expect(mocks.sentCommands.map(command => command.input.Key)).toEqual([ + "42/session-1/batch-a.json.zst", + "42/session-1/batch-b.json.zst", + ]); + }); +}); diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.ts index 87bf07df2..33b9746f4 100644 --- a/server/src/services/storage/r2StorageService.ts +++ b/server/src/services/storage/r2StorageService.ts @@ -69,13 +69,12 @@ class R2StorageService { * Store a batch of event data in R2 * Returns the storage key if successful, null if R2 is disabled */ - async storeBatch(siteId: number, sessionId: string, eventDataArray: any[]): Promise { + async storeBatch(siteId: number, sessionId: string, batchId: string, eventDataArray: any[]): Promise { if (!this.enabled || !this.client) { return null; } - const timestamp = Date.now(); - const key = `${siteId}/${sessionId}/${timestamp}.json.zst`; + const key = `${siteId}/${sessionId}/${batchId}.json.zst`; try { // Compress with zstd - much faster decompression than brotli diff --git a/server/src/services/tracker/botBlocking/botEventQueue.ts b/server/src/services/tracker/botBlocking/botEventQueue.ts index 207ce1618..f8275db3c 100644 --- a/server/src/services/tracker/botBlocking/botEventQueue.ts +++ b/server/src/services/tracker/botBlocking/botEventQueue.ts @@ -1,8 +1,8 @@ import { DateTime } from "luxon"; import { clickhouse } from "../../../db/clickhouse/clickhouse.js"; import { getLocation } from "../../../db/geolocation/geolocation.js"; -import { createServiceLogger } from "../../../lib/logger/logger.js"; import { getDeviceType } from "../../../utils.js"; +import { ReliableBatchQueue } from "../reliableBatchQueue.js"; import { clearSelfReferrer, type TotalTrackingPayload } from "../utils.js"; import type { BotEventProperties } from "./index.js"; @@ -12,84 +12,63 @@ type BotEventPayload = TotalTrackingPayload & }; const BOT_EVENT_BATCH_SIZE = 5000; -const BOT_EVENT_FLUSH_INTERVAL_MS = 1000; +const BOT_EVENT_FLUSH_INTERVAL_MS = 250; -class BotEventQueue { - private queue: BotEventPayload[] = []; - private batchSize = BOT_EVENT_BATCH_SIZE; - private interval = BOT_EVENT_FLUSH_INTERVAL_MS; - private processing = false; - private logger = createServiceLogger("bot-event-queue"); +async function processBotEventBatch(batch: BotEventPayload[]): Promise { + const ips = [...new Set(batch.map(event => event.ipAddress))]; + const geoData = await getLocation(ips); - constructor() { - setInterval(() => this.processQueue(), this.interval); - } + const processedBotEvents = batch.map(event => { + const dataForIp = geoData?.[event.ipAddress]; - async add(botEvent: BotEventPayload) { - this.queue.push(botEvent); - } + const countryCode = dataForIp?.countryIso || ""; + const regionCode = dataForIp?.region || ""; + const referrer = clearSelfReferrer(event.referrer || "", event.hostname || ""); - private async processQueue() { - if (this.processing || this.queue.length === 0) return; - this.processing = true; + return { + site_id: event.site_id, + timestamp: DateTime.fromISO(event.timestamp).toFormat("yyyy-MM-dd HH:mm:ss"), + session_id: event.sessionId, + user_id: event.userId, + hostname: event.hostname || "", + pathname: event.pathname || "", + querystring: event.querystring || "", + referrer, + browser: event.ua.browser.name || "", + browser_version: event.ua.browser.major || "", + operating_system: event.ua.os.name || "", + operating_system_version: event.ua.os.version || "", + country: countryCode, + region: countryCode && regionCode ? countryCode + "-" + regionCode : "", + city: dataForIp?.city || "", + lat: dataForIp?.latitude || 0, + lon: dataForIp?.longitude || 0, + screen_width: event.screenWidth || 0, + screen_height: event.screenHeight || 0, + device_type: getDeviceType(event.screenWidth, event.screenHeight, event.ua), + type: event.type || "pageview", + asn: event.botAsn ?? null, + asn_org: event.botAsnOrg || "", + detected_ua_pattern: event.detectedUaPattern || false, + detected_header_heuristics: event.detectedHeaderHeuristics || false, + detected_client_signals: event.detectedClientSignals || false, + detected_bot_asn: event.detectedBotAsn || false, + detected_rate_anomaly: event.detectedRateAnomaly || false, + matched_ua_pattern: event.matchedUaPattern || "", + bot_category: event.botCategory || "", + }; + }); - const batch = this.queue.splice(0, this.batchSize); - const ips = [...new Set(batch.map(event => event.ipAddress))]; - const geoData = await getLocation(ips); - - const processedBotEvents = batch.map(event => { - const dataForIp = geoData?.[event.ipAddress]; - - const countryCode = dataForIp?.countryIso || ""; - const regionCode = dataForIp?.region || ""; - const referrer = clearSelfReferrer(event.referrer || "", event.hostname || ""); - - return { - site_id: event.site_id, - timestamp: DateTime.fromISO(event.timestamp).toFormat("yyyy-MM-dd HH:mm:ss"), - session_id: event.sessionId, - user_id: event.userId, - hostname: event.hostname || "", - pathname: event.pathname || "", - querystring: event.querystring || "", - referrer, - browser: event.ua.browser.name || "", - browser_version: event.ua.browser.major || "", - operating_system: event.ua.os.name || "", - operating_system_version: event.ua.os.version || "", - country: countryCode, - region: countryCode && regionCode ? countryCode + "-" + regionCode : "", - city: dataForIp?.city || "", - lat: dataForIp?.latitude || 0, - lon: dataForIp?.longitude || 0, - screen_width: event.screenWidth || 0, - screen_height: event.screenHeight || 0, - device_type: getDeviceType(event.screenWidth, event.screenHeight, event.ua), - type: event.type || "pageview", - asn: event.botAsn ?? null, - asn_org: event.botAsnOrg || "", - detected_ua_pattern: event.detectedUaPattern || false, - detected_header_heuristics: event.detectedHeaderHeuristics || false, - detected_client_signals: event.detectedClientSignals || false, - detected_bot_asn: event.detectedBotAsn || false, - detected_rate_anomaly: event.detectedRateAnomaly || false, - matched_ua_pattern: event.matchedUaPattern || "", - bot_category: event.botCategory || "", - }; - }); - - try { - await clickhouse.insert({ - table: "bot_events", - values: processedBotEvents, - format: "JSONEachRow", - }); - } catch (error) { - this.logger.error(error, "Error processing bot event queue"); - } finally { - this.processing = false; - } - } + await clickhouse.insert({ + table: "bot_events", + values: processedBotEvents, + format: "JSONEachRow", + }); } -export const botEventQueue = new BotEventQueue(); +export const botEventQueue = new ReliableBatchQueue({ + name: "bot-event-queue", + batchSize: BOT_EVENT_BATCH_SIZE, + flushIntervalMs: BOT_EVENT_FLUSH_INTERVAL_MS, + processBatch: processBotEventBatch, +}); diff --git a/server/src/services/tracker/identifyService.test.ts b/server/src/services/tracker/identifyService.test.ts new file mode 100644 index 000000000..b2fafaa13 --- /dev/null +++ b/server/src/services/tracker/identifyService.test.ts @@ -0,0 +1,70 @@ +import type { FastifyReply, FastifyRequest } from "fastify"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + command: vi.fn(), + generateUserId: vi.fn(), + getConfig: vi.fn(), + transaction: vi.fn(), +})); + +vi.mock("../../db/clickhouse/clickhouse.js", () => ({ + clickhouse: { command: mocks.command }, +})); + +vi.mock("../../db/postgres/postgres.js", () => ({ db: { transaction: mocks.transaction } })); +vi.mock("../../lib/siteConfig.js", () => ({ siteConfig: { getConfig: mocks.getConfig } })); +vi.mock("../userId/userIdService.js", () => ({ + userIdService: { generateUserId: mocks.generateUserId }, +})); + +import { backfillIdentifiedUserId, handleIdentify } from "./identifyService.js"; + +function replyStub(): FastifyReply { + const reply = { + status: vi.fn(function (this: typeof reply) { + return this; + }), + send: vi.fn(function (this: typeof reply) { + return this; + }), + }; + return reply as unknown as FastifyReply; +} + +describe("identified-user backfill delivery", () => { + beforeEach(() => { + vi.clearAllMocks(); + const insertChain = { + values: vi.fn(), + onConflictDoNothing: vi.fn().mockResolvedValue(undefined), + onConflictDoUpdate: vi.fn().mockResolvedValue(undefined), + }; + insertChain.values.mockReturnValue(insertChain); + mocks.transaction.mockImplementation(async callback => callback({ insert: () => insertChain })); + mocks.getConfig.mockResolvedValue({ siteId: 42 }); + mocks.generateUserId.mockResolvedValue("anonymous"); + mocks.command.mockResolvedValue(undefined); + }); + + it("surfaces a ClickHouse mutation failure so identify can be retried", async () => { + mocks.command.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + + await expect(backfillIdentifiedUserId(42, "anonymous", "employee")).rejects.toThrow("ClickHouse unavailable"); + }); + + it("returns an error when the awaited backfill fails", async () => { + mocks.command.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + const request = { + body: { site_id: "site-1", user_id: "employee", is_new_identify: true }, + headers: { "user-agent": "Browser" }, + ip: "198.51.100.10", + } as unknown as FastifyRequest; + const reply = replyStub(); + + await handleIdentify(request, reply); + + expect(reply.status).toHaveBeenCalledWith(500); + expect(mocks.transaction).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/services/tracker/identifyService.ts b/server/src/services/tracker/identifyService.ts index a424a1467..51b62fca3 100644 --- a/server/src/services/tracker/identifyService.ts +++ b/server/src/services/tracker/identifyService.ts @@ -1,5 +1,5 @@ import { FastifyReply, FastifyRequest } from "fastify"; -import { eq, and, sql } from "drizzle-orm"; +import { sql } from "drizzle-orm"; import { z } from "zod"; import { db } from "../../db/postgres/postgres.js"; import { clickhouse } from "../../db/clickhouse/clickhouse.js"; @@ -18,7 +18,7 @@ const MAX_TRAITS_SIZE = 2048; const identifyPayloadSchema = z.object({ site_id: z.string().min(1), anonymous_id: z.string().min(1).max(255).optional(), - user_id: z.string().min(1).max(255), + user_id: z.string().trim().min(1).max(255), ip_address: z.string().ip().optional(), user_agent: z.string().max(512).optional(), traits: z @@ -68,6 +68,7 @@ export async function backfillIdentifiedUserId( logger.info({ siteId, anonymousId, userId }, "Backfilled identified_user_id in ClickHouse"); } catch (error) { logger.error({ siteId, anonymousId, userId, error }, "Error backfilling identified_user_id"); + throw error; } } @@ -104,50 +105,22 @@ export async function handleIdentify(request: FastifyRequest, reply: FastifyRepl siteId ); - // Create alias if this is a new identify call (links anonymous_id to user_id) - if (is_new_identify) { - // Ensure a user_profiles row exists at identify time so the user is - // discoverable via search/inventory queries even when no traits are set, - // and so createdAt reflects identification time rather than first setTraits. - try { - await db.insert(userProfiles).values({ siteId, userId: user_id }).onConflictDoNothing(); - } catch (error) { - logger.error({ siteId, userId: user_id, error }, "Error creating user profile shell"); - } - - try { - // Check if alias already exists - const existingAlias = await db - .select() - .from(userAliases) - .where(and(eq(userAliases.siteId, siteId), eq(userAliases.anonymousId, anonymousId))) - .limit(1); - - if (existingAlias.length === 0) { - // Create new alias - await db.insert(userAliases).values({ - siteId, - anonymousId, - userId: user_id, + await db.transaction(async tx => { + if (is_new_identify) { + // Keep profile and alias creation atomic. The conflict update also handles + // simultaneous identify calls for the same anonymous visitor. + await tx.insert(userProfiles).values({ siteId, userId: user_id }).onConflictDoNothing(); + await tx + .insert(userAliases) + .values({ siteId, anonymousId, userId: user_id }) + .onConflictDoUpdate({ + target: [userAliases.siteId, userAliases.anonymousId], + set: { userId: user_id }, }); - // Fire-and-forget: backfill identified_user_id on past anonymous events - backfillIdentifiedUserId(siteId, anonymousId, user_id); - } else if (existingAlias[0].userId !== user_id) { - // Update alias to point to new user - await db - .update(userAliases) - .set({ userId: user_id }) - .where(and(eq(userAliases.siteId, siteId), eq(userAliases.anonymousId, anonymousId))); - } - } catch (error) { - // Handle unique constraint violation gracefully (race condition) - logger.debug({ siteId, anonymousId, userId: user_id, error }, "Alias may already exist"); } - } - // Atomic upsert: merge non-null traits and remove keys explicitly set to null. - if (traits && Object.keys(traits).length > 0) { - try { + // Atomic upsert: merge non-null traits and remove keys explicitly set to null. + if (traits && Object.keys(traits).length > 0) { const filteredTraits = Object.fromEntries(Object.entries(traits).filter(([_, v]) => v !== null)); const nullKeys = Object.entries(traits) .filter(([_, v]) => v === null) @@ -160,7 +133,7 @@ export async function handleIdentify(request: FastifyRequest, reply: FastifyRepl ? sql`(${userProfiles.traits} - ${nullKeys}::text[]) || ${JSON.stringify(filteredTraits)}::jsonb` : sql`${userProfiles.traits} || ${JSON.stringify(filteredTraits)}::jsonb`; - await db + await tx .insert(userProfiles) .values({ siteId, userId: user_id, traits: filteredTraits }) .onConflictDoUpdate({ @@ -170,9 +143,13 @@ export async function handleIdentify(request: FastifyRequest, reply: FastifyRepl updatedAt: sql`now()`, }, }); - } catch (error) { - logger.error({ siteId, userId: user_id, error }, "Error updating user profile"); } + }); + + if (is_new_identify) { + // Await the mutation so failures are visible and a retried identify call + // re-runs the idempotent `identified_user_id = ''` backfill. + await backfillIdentifiedUserId(siteId, anonymousId, user_id); } return reply.status(200).send({ diff --git a/server/src/services/tracker/pageviewQueue.test.ts b/server/src/services/tracker/pageviewQueue.test.ts new file mode 100644 index 000000000..d60394099 --- /dev/null +++ b/server/src/services/tracker/pageviewQueue.test.ts @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + getLocation: vi.fn(), + insert: vi.fn(), +})); + +vi.mock("../../db/geolocation/geolocation.js", () => ({ getLocation: mocks.getLocation })); +vi.mock("../../db/clickhouse/clickhouse.js", () => ({ clickhouse: { insert: mocks.insert } })); + +import { pageviewQueue } from "./pageviewQueue.js"; + +const payload = { + site_id: 42, + timestamp: "2026-07-12T12:00:00.123Z", + sessionId: "session-1", + userId: "visitor-1", + identifiedUserId: "", + ipAddress: "198.51.100.10", + ua: { browser: {}, os: {}, device: {}, engine: {}, cpu: {}, ua: "" }, + hostname: "example.com", + pathname: "/", +} as any; + +describe("PageviewQueue delivery", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.getLocation.mockResolvedValue({}); + mocks.insert.mockResolvedValue(undefined); + (pageviewQueue as any).queue = []; + (pageviewQueue as any).processing = false; + (pageviewQueue as any).closing = false; + }); + + it("does not acknowledge an event until ClickHouse stores its batch", async () => { + let acknowledged = false; + const delivery = pageviewQueue.add(payload).then(() => { + acknowledged = true; + }); + + await Promise.resolve(); + expect(acknowledged).toBe(false); + + await (pageviewQueue as any).processQueue(); + await delivery; + expect(acknowledged).toBe(true); + }); + + it("requeues a transiently failed batch instead of dropping it", async () => { + mocks.insert.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + const delivery = pageviewQueue.add(payload); + + await (pageviewQueue as any).processQueue(); + expect((pageviewQueue as any).queue).toHaveLength(1); + + await (pageviewQueue as any).processQueue({ ignoreBackoff: true }); + await expect(delivery).resolves.toBeUndefined(); + expect(mocks.insert).toHaveBeenCalledTimes(2); + }); + + it("recovers when enrichment fails before insertion", async () => { + mocks.getLocation.mockRejectedValueOnce(new Error("GeoIP unavailable")); + const delivery = pageviewQueue.add(payload); + + await expect((pageviewQueue as any).processQueue()).resolves.toBeUndefined(); + expect((pageviewQueue as any).processing).toBe(false); + expect((pageviewQueue as any).queue).toHaveLength(1); + + await (pageviewQueue as any).processQueue({ ignoreBackoff: true }); + await expect(delivery).resolves.toBeUndefined(); + }); +}); diff --git a/server/src/services/tracker/pageviewQueue.ts b/server/src/services/tracker/pageviewQueue.ts index 1b901d26c..49805cc80 100644 --- a/server/src/services/tracker/pageviewQueue.ts +++ b/server/src/services/tracker/pageviewQueue.ts @@ -1,17 +1,17 @@ import { DateTime } from "luxon"; import { clickhouse } from "../../db/clickhouse/clickhouse.js"; import { getLocation } from "../../db/geolocation/geolocation.js"; -import { createServiceLogger } from "../../lib/logger/logger.js"; import { getDeviceType } from "../../utils.js"; import { getChannel } from "./getChannel.js"; import { clearSelfReferrer, getAllUrlParams, TotalTrackingPayload } from "./utils.js"; +import { ReliableBatchQueue } from "./reliableBatchQueue.js"; type TotalPayload = TotalTrackingPayload & { sessionId: string; }; const PAGEVIEW_BATCH_SIZE = 5000; -const PAGEVIEW_FLUSH_INTERVAL_MS = 1000; +const PAGEVIEW_FLUSH_INTERVAL_MS = 250; const getParsedProperties = (properties: string | undefined) => { try { @@ -21,39 +21,20 @@ const getParsedProperties = (properties: string | undefined) => { } }; -class PageviewQueue { - private queue: TotalPayload[] = []; - private batchSize = PAGEVIEW_BATCH_SIZE; - private interval = PAGEVIEW_FLUSH_INTERVAL_MS; - private processing = false; - private logger = createServiceLogger("pageview-queue"); +async function processPageviewBatch(batch: TotalPayload[]): Promise { + const ips = [...new Set(batch.map(pv => pv.ipAddress))]; - constructor() { - // Start processing interval - setInterval(() => this.processQueue(), this.interval); - } - - async add(pageview: TotalPayload) { - this.queue.push(pageview); - } - - private async processQueue() { - if (this.processing || this.queue.length === 0) return; - this.processing = true; + const geoData = await getLocation(ips); - // Get batch of pageviews - const batch = this.queue.splice(0, this.batchSize); - const ips = [...new Set(batch.map(pv => pv.ipAddress))]; - - const geoData = await getLocation(ips); - - // Process each pageview with its geo data - const processedPageviews = batch.filter(pv => { + // Process each pageview with its geo data + const processedPageviews = batch + .filter(pv => { if (pv.site_id == 9133 && pv.screenWidth == 800 && pv.screenHeight == 600) { return false; } return true; - }).map(pv => { + }) + .map(pv => { const dataForIp = geoData?.[pv.ipAddress]; const countryCode = dataForIp?.countryIso || ""; @@ -69,7 +50,6 @@ class PageviewQueue { // Get all URL parameters for the url_parameters map const allUrlParams = getAllUrlParams(pv.querystring || ""); - return { site_id: pv.site_id, timestamp: DateTime.fromISO(pv.timestamp).toFormat("yyyy-MM-dd HH:mm:ss"), @@ -115,21 +95,17 @@ class PageviewQueue { }; }); - // this.logger.info({ count: processedPageviews.length }, "Bulk insert to ClickHouse"); - // Bulk insert into database - try { - await clickhouse.insert({ - table: "events", - values: processedPageviews, - format: "JSONEachRow", - }); - } catch (error) { - this.logger.error(error, "Error processing pageview queue"); - } finally { - this.processing = false; - } - } + await clickhouse.insert({ + table: "events", + values: processedPageviews, + format: "JSONEachRow", + }); } // Create singleton instance -export const pageviewQueue = new PageviewQueue(); +export const pageviewQueue = new ReliableBatchQueue({ + name: "pageview-queue", + batchSize: PAGEVIEW_BATCH_SIZE, + flushIntervalMs: PAGEVIEW_FLUSH_INTERVAL_MS, + processBatch: processPageviewBatch, +}); diff --git a/server/src/services/tracker/reliableBatchQueue.test.ts b/server/src/services/tracker/reliableBatchQueue.test.ts new file mode 100644 index 000000000..81e03a17a --- /dev/null +++ b/server/src/services/tracker/reliableBatchQueue.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, vi } from "vitest"; +import { QueueFullError, ReliableBatchQueue } from "./reliableBatchQueue.js"; + +function createQueue(processBatch: (batch: number[]) => Promise, maxQueueSize = 10) { + return new ReliableBatchQueue({ + name: "test-ingestion-queue", + batchSize: 10, + flushIntervalMs: 60_000, + maxQueueSize, + processBatch, + }); +} + +describe("ReliableBatchQueue", () => { + it("rejects the caller instead of acknowledging after retries are exhausted", async () => { + const processBatch = vi.fn().mockRejectedValue(new Error("storage unavailable")); + const queue = createQueue(processBatch); + const delivery = queue.add(1); + const rejection = expect(delivery).rejects.toThrow("storage unavailable"); + + await queue.processQueue({ ignoreBackoff: true }); + await queue.processQueue({ ignoreBackoff: true }); + await queue.processQueue({ ignoreBackoff: true }); + + await rejection; + expect(processBatch).toHaveBeenCalledTimes(3); + await queue.close(); + }); + + it("drains pending deliveries during graceful shutdown", async () => { + const processBatch = vi.fn().mockResolvedValue(undefined); + const queue = createQueue(processBatch); + const delivery = queue.add(1); + + await queue.close(); + + await expect(delivery).resolves.toBeUndefined(); + expect(processBatch).toHaveBeenCalledWith([1]); + }); + + it("applies bounded backpressure", async () => { + const processBatch = vi.fn().mockResolvedValue(undefined); + const queue = createQueue(processBatch, 1); + const firstDelivery = queue.add(1); + + await expect(queue.add(2)).rejects.toBeInstanceOf(QueueFullError); + await queue.close(); + await expect(firstDelivery).resolves.toBeUndefined(); + }); +}); diff --git a/server/src/services/tracker/reliableBatchQueue.ts b/server/src/services/tracker/reliableBatchQueue.ts new file mode 100644 index 000000000..8b4935d9f --- /dev/null +++ b/server/src/services/tracker/reliableBatchQueue.ts @@ -0,0 +1,152 @@ +import { createServiceLogger } from "../../lib/logger/logger.js"; + +type QueueEntry = { + value: T; + attempts: number; + resolve: () => void; + reject: (error: Error) => void; +}; + +type ProcessQueueOptions = { + ignoreBackoff?: boolean; +}; + +type ReliableBatchQueueOptions = { + batchSize: number; + flushIntervalMs: number; + maxAttempts?: number; + maxQueueSize?: number; + name: string; + processBatch: (batch: T[]) => Promise; +}; + +const DEFAULT_MAX_ATTEMPTS = 3; +const DEFAULT_MAX_QUEUE_SIZE = 50_000; +const MAX_RETRY_DELAY_MS = 1_000; + +export class QueueClosedError extends Error { + constructor(queueName: string) { + super(`${queueName} is shutting down`); + } +} + +export class QueueFullError extends Error { + constructor(queueName: string, maxQueueSize: number) { + super(`${queueName} reached its ${maxQueueSize}-event capacity`); + } +} + +/** + * Small request-backed batching primitive: callers are resolved only after the + * batch reaches durable storage. Failed batches stay queued for bounded retries; + * once retries are exhausted callers receive an error instead of a false 200. + */ +export class ReliableBatchQueue { + private queue: QueueEntry[] = []; + private processing = false; + private closing = false; + private nextAttemptAt = 0; + private intervalHandle: ReturnType; + private activeProcess: Promise | null = null; + private readonly logger; + private readonly maxAttempts: number; + private readonly maxQueueSize: number; + + constructor(private readonly options: ReliableBatchQueueOptions) { + this.logger = createServiceLogger(options.name); + this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS; + this.maxQueueSize = options.maxQueueSize ?? DEFAULT_MAX_QUEUE_SIZE; + this.intervalHandle = setInterval(() => void this.processQueue(), options.flushIntervalMs); + } + + add(value: T): Promise { + if (this.closing) { + return Promise.reject(new QueueClosedError(this.options.name)); + } + if (this.queue.length >= this.maxQueueSize) { + return Promise.reject(new QueueFullError(this.options.name, this.maxQueueSize)); + } + + const delivery = new Promise((resolve, reject) => { + this.queue.push({ value, attempts: 0, resolve, reject }); + }); + + if (this.queue.length >= this.options.batchSize) { + void this.processQueue(); + } + + return delivery; + } + + async processQueue(processOptions: ProcessQueueOptions = {}): Promise { + if (this.activeProcess) { + await this.activeProcess; + return; + } + if (this.queue.length === 0) return; + if (!processOptions.ignoreBackoff && Date.now() < this.nextAttemptAt) return; + + this.activeProcess = this.runBatch(); + try { + await this.activeProcess; + } finally { + this.activeProcess = null; + } + } + + private async runBatch(): Promise { + this.processing = true; + const entries = this.queue.splice(0, this.options.batchSize); + + try { + await this.options.processBatch(entries.map(entry => entry.value)); + this.nextAttemptAt = 0; + for (const entry of entries) entry.resolve(); + } catch (cause) { + const error = cause instanceof Error ? cause : new Error(String(cause)); + const retryEntries: QueueEntry[] = []; + + for (const entry of entries) { + entry.attempts += 1; + if (entry.attempts < this.maxAttempts) { + retryEntries.push(entry); + } else { + entry.reject(error); + } + } + + if (retryEntries.length > 0) { + this.queue.unshift(...retryEntries); + const attempt = Math.max(...retryEntries.map(entry => entry.attempts)); + this.nextAttemptAt = Date.now() + Math.min(100 * 2 ** (attempt - 1), MAX_RETRY_DELAY_MS); + } + + this.logger.error( + { + err: error, + batchSize: entries.length, + requeued: retryEntries.length, + }, + "Failed to persist ingestion batch" + ); + } finally { + this.processing = false; + } + } + + /** Stop the timer and synchronously exhaust pending batches before shutdown. */ + async close(): Promise { + if (this.closing) { + if (this.activeProcess) await this.activeProcess; + return; + } + + this.closing = true; + clearInterval(this.intervalHandle); + + if (this.activeProcess) await this.activeProcess; + while (this.queue.length > 0) { + await this.processQueue({ ignoreBackoff: true }); + } + } +} diff --git a/server/src/services/tracker/trackEvent.test.ts b/server/src/services/tracker/trackEvent.test.ts index fb6c64102..37f8bd12a 100644 --- a/server/src/services/tracker/trackEvent.test.ts +++ b/server/src/services/tracker/trackEvent.test.ts @@ -53,7 +53,7 @@ vi.mock("./utils.js", () => ({ createBasePayload: mocks.createBasePayload, })); -import { trackEvent } from "./trackEvent.js"; +import { trackEvent, trackingPayloadSchema } from "./trackEvent.js"; const siteConfiguration = { id: "site_abc", @@ -120,3 +120,21 @@ describe("trackEvent session identity", () => { ); }); }); + +describe("tracking payload storage bounds", () => { + const payload = { + type: "pageview" as const, + site_id: "site_abc", + }; + + it("accepts the largest ClickHouse UInt16 screen dimensions", () => { + expect(trackingPayloadSchema.safeParse({ ...payload, screenWidth: 65_535, screenHeight: 65_535 }).success).toBe( + true + ); + }); + + it("rejects dimensions that cannot be stored in ClickHouse", () => { + expect(trackingPayloadSchema.safeParse({ ...payload, screenWidth: 65_536 }).success).toBe(false); + expect(trackingPayloadSchema.safeParse({ ...payload, screenHeight: 65_536 }).success).toBe(false); + }); +}); diff --git a/server/src/services/tracker/trackEvent.ts b/server/src/services/tracker/trackEvent.ts index aa1e51429..9b7e7592a 100644 --- a/server/src/services/tracker/trackEvent.ts +++ b/server/src/services/tracker/trackEvent.ts @@ -11,6 +11,8 @@ import { checkApiKey } from "../../lib/auth-utils.js"; import { botEventQueue } from "./botBlocking/botEventQueue.js"; import { checkBotBlocking } from "./botBlocking/index.js"; import { resolveTrackingIdentity } from "./requestIdentity.js"; +import { CLICKHOUSE_UINT16_MAX } from "../../lib/clickhouseLimits.js"; +import { QueueFullError } from "./reliableBatchQueue.js"; // Shared fields for all event types const baseEventFields = { @@ -18,8 +20,8 @@ const baseEventFields = { hostname: z.string().max(253).optional(), pathname: z.string().max(2048).optional(), querystring: z.string().max(2048).optional(), - screenWidth: z.number().int().nonnegative().optional(), - screenHeight: z.number().int().nonnegative().optional(), + screenWidth: z.number().int().nonnegative().max(CLICKHOUSE_UINT16_MAX).optional(), + screenHeight: z.number().int().nonnegative().max(CLICKHOUSE_UINT16_MAX).optional(), language: z.string().max(35).optional(), page_title: z.string().max(512).optional(), referrer: z.string().max(2048).optional(), @@ -440,6 +442,12 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) { details: error.flatten(), }); } + if (error instanceof QueueFullError) { + return reply.status(503).send({ + success: false, + error: "Ingestion queue is at capacity", + }); + } return reply.status(500).send({ success: false, error: "Failed to track event", diff --git a/server/src/types/sessionReplay.ts b/server/src/types/sessionReplay.ts index ba4b61c03..bdc500a93 100644 --- a/server/src/types/sessionReplay.ts +++ b/server/src/types/sessionReplay.ts @@ -46,11 +46,13 @@ export interface SessionReplayMetadata { } export interface RecordSessionReplayRequest { + batchId?: string; userId: string; events: Array<{ type: string | number; data: any; timestamp: number; + sequence?: number; }>; metadata?: { pageUrl: string; From 7ed16780d827f5165dab5a912c2b459469ffabda Mon Sep 17 00:00:00 2001 From: Bill Yang Date: Tue, 14 Jul 2026 00:39:18 -0700 Subject: [PATCH 2/3] Enhance session replay error handling and improve test coverage - Updated the session replay batch sending logic to throw an error when the server responds with an HTTP error, ensuring better error management. - Added a test case to verify that session replay delivery is correctly rejected when an HTTP error occurs, improving the robustness of the tracking system. - Refactored the session replay batch sending method to include response validation, enhancing overall reliability. --- server/public/script-full.js | 5 +- server/public/script.js | 2 +- server/src/analytics-script/tracking.test.ts | 29 ++++++ server/src/analytics-script/tracking.ts | 5 +- .../src/api/sites/batchImportEvents.test.ts | 43 +++++++-- server/src/api/sites/batchImportEvents.ts | 33 +++++-- server/src/api/sites/createSiteImport.ts | 4 +- server/src/db/clickhouse/clickhouse.ts | 23 +++++ .../import/importQuotaTracker.test.ts | 94 +++++++++++++++++++ .../src/services/import/importQuotaTracker.ts | 69 +++++++++++--- .../import/importStatusManager.test.ts | 22 ++++- .../services/import/importStatusManager.ts | 45 +++++---- .../replay/sessionReplayIngestService.test.ts | 26 ++++- .../replay/sessionReplayIngestService.ts | 60 ++++++++---- .../tracker/botBlocking/botEventQueue.ts | 7 +- .../services/tracker/pageviewQueue.test.ts | 15 ++- server/src/services/tracker/pageviewQueue.ts | 7 +- .../tracker/reliableBatchQueue.test.ts | 30 +++++- .../services/tracker/reliableBatchQueue.ts | 78 ++++++++++----- 19 files changed, 489 insertions(+), 108 deletions(-) create mode 100644 server/src/services/import/importQuotaTracker.test.ts diff --git a/server/public/script-full.js b/server/public/script-full.js index bafe6de0b..bf18e6143 100644 --- a/server/public/script-full.js +++ b/server/public/script-full.js @@ -722,7 +722,7 @@ } async sendSessionReplayBatch(batch) { try { - await fetch(`${this.config.analyticsHost}/session-replay/record/${this.config.siteId}`, { + const response = await fetch(`${this.config.analyticsHost}/session-replay/record/${this.config.siteId}`, { method: "POST", headers: { "Content-Type": "application/json" @@ -732,6 +732,9 @@ keepalive: false // Disable keepalive for large session replay requests }); + if (!response.ok) { + throw new Error(`Session replay delivery failed: ${response.status} ${response.statusText}`.trim()); + } } catch (error) { console.error("Failed to send session replay batch:", error); throw error; diff --git a/server/public/script.js b/server/public/script.js index 3d25bc97e..5837169ee 100644 --- a/server/public/script.js +++ b/server/public/script.js @@ -1 +1 @@ -"use strict";(()=>{var De=Object.defineProperty;var Ue=(r,e,t)=>e in r?De(r,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):r[e]=t;var R=(r,e,t)=>Ue(r,typeof e!="symbol"?e+"":e,t);function He(r){if(r.startsWith("re:")){let l=r.slice(3);if(!l)throw new Error("Empty regex pattern");return new RegExp(l)}let t="__DOUBLE_ASTERISK_TOKEN__",n="__SINGLE_ASTERISK_TOKEN__",s=r.replace(/\*\*/g,t).replace(/\*/g,n).replace(/[.+?^${}()|[\]\\]/g,"\\$&");s=s.replace(new RegExp(`/${t}/`,"g"),"/(?:.+/)?"),s=s.replace(new RegExp(t,"g"),".*"),s=s.replace(/\//g,"\\/");let a=s.replace(new RegExp(n,"g"),"[^/]+");return new RegExp("^"+a+"$")}function j(r,e){for(let t of e)try{if(He(t).test(r))return t}catch(n){console.error(`Invalid pattern: ${t}`,n)}return null}function ce(r,e){let t=null;return(...n)=>{t&&clearTimeout(t),t=setTimeout(()=>r(...n),e)}}function le(r){try{let e=window.location.hostname,t=new URL(r).hostname;return t!==e&&t!==""}catch{return!1}}function E(r,e){if(!r)return e;try{let t=JSON.parse(r);return Array.isArray(e)&&!Array.isArray(t)?e:t}catch(t){return console.error("Error parsing JSON:",t),e}}function ue(){try{if(crypto?.randomUUID)return crypto.randomUUID()}catch{}return`${Date.now().toString(36)}-${Math.random().toString(36).slice(2,12)}`}function We(r){let e=`${r}-visitor-id`;try{let t=localStorage.getItem(e);if(t)return t;let n=ue();return localStorage.setItem(e,n),n}catch{return ue()}}function $e(r){try{return localStorage.getItem(`${r}-user-id`)||void 0}catch{return}}function je(r){return r.hash&&r.hash.startsWith("#/")?r.hash.substring(1):r.pathname}async function Ve(r,e,t,n){try{let i=new URL(window.location.href),s=await fetch(`${r}/site/${e}/feature-flags/evaluate`,{method:"POST",headers:{"Content-Type":"application/json"},credentials:"omit",body:JSON.stringify({anonymousId:n,identifiedUserId:$e(t),hostname:i.hostname,pathname:je(i),querystring:i.search,query:Object.fromEntries(i.searchParams.entries()),referrer:document.referrer,language:navigator.language,screenWidth:screen.width,screenHeight:screen.height})});if(!s.ok)return{};let a=await s.json();return a?.flags&&typeof a.flags=="object"?a.flags:{}}catch{return{}}}async function de(r){let e=r.getAttribute("src");if(!e)return console.error("Script src attribute is missing"),null;let t=e.split("/script.js")[0];if(!t)return console.error("Please provide a valid analytics host"),null;let n=r.getAttribute("data-site-id")||r.getAttribute("site-id");if(!n)return console.error("Please provide a valid site ID using the data-site-id attribute"),null;let i=r.getAttribute("data-namespace")||"rybbit",s=We(i),a=E(r.getAttribute("data-skip-patterns"),[]),l=E(r.getAttribute("data-mask-patterns"),[]),d=E(r.getAttribute("data-replay-mask-text-selectors"),[]),u=r.getAttribute("data-debounce")?Math.max(0,parseInt(r.getAttribute("data-debounce"))):500,h=r.getAttribute("data-replay-batch-size")?Math.max(1,parseInt(r.getAttribute("data-replay-batch-size"))):250,g=r.getAttribute("data-replay-batch-interval")?Math.max(1e3,parseInt(r.getAttribute("data-replay-batch-interval"))):5e3,C=r.getAttribute("data-replay-block-class")||void 0,m=r.getAttribute("data-replay-block-selector")||void 0,o=r.getAttribute("data-replay-ignore-class")||void 0,c=r.getAttribute("data-replay-ignore-selector")||void 0,p=r.getAttribute("data-replay-mask-text-class")||void 0,w=r.getAttribute("data-replay-mask-all-inputs"),S=w!==null?w!=="false":void 0,T=r.getAttribute("data-replay-mask-input-options"),A=T?E(T,{password:!0,email:!0}):void 0,re=r.getAttribute("data-replay-collect-fonts"),xe=re!==null?re!=="false":void 0,ie=r.getAttribute("data-replay-sampling"),Me=ie?E(ie,{}):void 0,se=r.getAttribute("data-replay-slim-dom-options"),Be=se?E(se,{}):void 0,ae=r.getAttribute("data-replay-sample-rate"),Oe=ae?Math.min(100,Math.max(0,parseInt(ae,10))):void 0,Ne=r.getAttribute("data-tag")||"",f={namespace:i,analyticsHost:t,siteId:n,visitorId:s,debounceDuration:u,sessionReplayBatchSize:h,sessionReplayBatchInterval:g,sessionReplayMaskTextSelectors:d,skipPatterns:a,maskPatterns:l,autoTrackPageview:!0,autoTrackSpa:!0,trackQuerystring:!0,trackOutbound:!0,enableWebVitals:!1,trackErrors:!1,enableSessionReplay:!1,trackButtonClicks:!1,trackCopy:!1,trackFormInteractions:!1,tag:Ne,featureFlags:{},sessionReplayBlockClass:C,sessionReplayBlockSelector:m,sessionReplayIgnoreClass:o,sessionReplayIgnoreSelector:c,sessionReplayMaskTextClass:p,sessionReplayMaskAllInputs:S,sessionReplayMaskInputOptions:A,sessionReplayCollectFonts:xe,sessionReplaySampling:Me,sessionReplaySlimDOMOptions:Be,sessionReplaySampleRate:Oe},W=f;try{let $=`${t}/site/tracking-config/${n}`,oe=await fetch($,{method:"GET",credentials:"omit"});if(oe.ok){let y=await oe.json();W={...f,autoTrackPageview:y.trackInitialPageView??f.autoTrackPageview,autoTrackSpa:y.trackSpaNavigation??f.autoTrackSpa,trackQuerystring:y.trackUrlParams??f.trackQuerystring,trackOutbound:y.trackOutbound??f.trackOutbound,enableWebVitals:y.webVitals??f.enableWebVitals,trackErrors:y.trackErrors??f.trackErrors,enableSessionReplay:y.sessionReplay??f.enableSessionReplay,trackButtonClicks:y.trackButtonClicks??f.trackButtonClicks,trackCopy:y.trackCopy??f.trackCopy,trackFormInteractions:y.trackFormInteractions??f.trackFormInteractions}}else console.warn("Failed to fetch tracking config from API, using defaults")}catch($){console.warn("Error fetching tracking config:",$)}return W.featureFlags=await Ve(t,n,i,s),W}var pe="rybbit-replay-sampled";function ze(){let r=new Uint8Array(16);if(globalThis.crypto?.getRandomValues)globalThis.crypto.getRandomValues(r);else for(let t=0;tt.toString(16).padStart(2,"0"));return`${e.slice(0,4).join("")}-${e.slice(4,6).join("")}-${e.slice(6,8).join("")}-${e.slice(8,10).join("")}-${e.slice(10).join("")}`}function qe(r){if(r>=100)return!0;if(r<=0)return!1;try{let e=sessionStorage.getItem(pe);if(e!==null)return e==="1";let t=Math.random()*100{let n=document.createElement("script");n.src=`${this.config.analyticsHost}/replay.js`,n.async=!1,n.onload=()=>{e()},n.onerror=()=>t(new Error("Failed to load rrweb")),document.head.appendChild(n)})}startRecording(){if(!(this.isRecording||!window.rrweb||!this.config.enableSessionReplay))try{let e={mousemove:!1,mouseInteraction:{MouseUp:!1,MouseDown:!1,Click:!0,ContextMenu:!1,DblClick:!0,Focus:!0,Blur:!0,TouchStart:!1,TouchEnd:!1},scroll:500,input:"last",media:800},t={script:!1,comment:!0,headFavicon:!0,headWhitespace:!0,headMetaDescKeywords:!0,headMetaSocial:!0,headMetaRobots:!0,headMetaHttpEquiv:!0,headMetaAuthorship:!0,headMetaVerification:!0},n={emit:i=>{this.addEvent({type:i.type,data:i.data,timestamp:i.timestamp||Date.now()})},recordCanvas:!1,checkoutEveryNms:6e4,checkoutEveryNth:500,blockClass:this.config.sessionReplayBlockClass??"rr-block",blockSelector:this.config.sessionReplayBlockSelector??null,ignoreClass:this.config.sessionReplayIgnoreClass??"rr-ignore",ignoreSelector:this.config.sessionReplayIgnoreSelector??null,maskTextClass:this.config.sessionReplayMaskTextClass??"rr-mask",maskAllInputs:this.config.sessionReplayMaskAllInputs??!0,maskInputOptions:this.config.sessionReplayMaskInputOptions??{password:!0,email:!0},collectFonts:this.config.sessionReplayCollectFonts??!0,sampling:this.config.sessionReplaySampling??e,slimDOMOptions:this.config.sessionReplaySlimDOMOptions??t};this.config.sessionReplayMaskTextSelectors&&this.config.sessionReplayMaskTextSelectors.length>0&&(n.maskTextSelector=this.config.sessionReplayMaskTextSelectors.join(", ")),this.stopRecordingFn=window.rrweb.record(n),this.isRecording=!0,this.setupBatchTimer()}catch{}}stopRecording(){this.isRecording&&(this.stopRecordingFn&&this.stopRecordingFn(),this.isRecording=!1,this.clearBatchTimer(),(this.eventBuffer.length>0||this.pendingBatches.length>0)&&this.flushEvents())}isActive(){return this.isRecording}addEvent(e){this.eventBuffer.push({...e,sequence:this.nextSequence++}),this.eventBuffer.length>=this.config.sessionReplayBatchSize&&this.flushEvents()}setupBatchTimer(){this.clearBatchTimer(),this.batchTimer=window.setInterval(()=>{(this.eventBuffer.length>0||this.pendingBatches.length>0)&&this.flushEvents()},this.config.sessionReplayBatchInterval)}clearBatchTimer(){this.batchTimer&&(clearInterval(this.batchTimer),this.batchTimer=void 0)}async flushEvents(){if(this.eventBuffer.length>0){let e={batchId:ze(),userId:this.userId,events:this.eventBuffer,metadata:{pageUrl:window.location.href,viewportWidth:screen.width,viewportHeight:screen.height,language:navigator.language}};this.eventBuffer=[],this.pendingBatches.push(e)}await this.drainPendingBatches()}drainPendingBatches(){return this.sendLoop?this.sendLoop.then(()=>{if(this.pendingBatches.length>0&&this.sendLoop===null)return this.drainPendingBatches()}):(this.sendLoop=(async()=>{for(;this.pendingBatches.length>0;)try{await this.sendBatch(this.pendingBatches[0]),this.pendingBatches.shift()}catch{return}})().finally(()=>{this.sendLoop=null}),this.sendLoop)}updateUserId(e){e!==this.userId&&(this.eventBuffer.length>0&&this.flushEvents(),this.userId=e)}onPageChange(){this.isRecording&&this.flushEvents()}cleanup(){this.stopRecording()}};var b={automationApi:1,webdriver:1,zeroOuterDimensions:2,missingChrome:4,swiftShader:8,emptyPlugins:16,defaultViewport800x600:32,defaultViewport1024x768:64,impossibleDimensions:128,outerDimensionsWeird:256,pluginApiAbsence:512},he=null,Ge=10;function ge(){return me().score}function fe(){return me().mask}function me(){return he??(he=Ke()),he}function Ke(){let r=0,e=0;function t(n,i){(e&n)===0&&(e|=n,r+=i)}try{let n=navigator.userAgent,i=/Chrome\//.test(n)&&!/\bwv\b|; wv\)/.test(n),s=/Windows NT|Macintosh|X11|Linux x86_64/.test(n)&&!/Mobile|Android|iPhone|iPad/.test(n),a=Number(window.screen?.width),l=Number(window.screen?.height),d=Number(window.outerWidth),u=Number(window.outerHeight),h=Number(window.innerWidth),g=Number(window.innerHeight),m=["__webdriver_evaluate","__selenium_evaluate","__webdriver_script_function","__webdriver_script_func","__webdriver_script_fn","__fxdriver_evaluate","__driver_unwrapped","__webdriver_unwrapped","__driver_evaluate","__selenium_unwrapped","__fxdriver_unwrapped","_phantom","callPhantom","__nightmare","domAutomation","domAutomationController"].some(c=>c in window||c in document);(navigator.webdriver===!0||m)&&t(b.automationApi,3),(u===0||d===0)&&t(b.zeroOuterDimensions,2),(!Number.isFinite(a)||!Number.isFinite(l)||a<=0||l<=0||a>1e5||l>1e5)&&t(b.impossibleDimensions,3),s&&a===800&&l===600&&t(b.defaultViewport800x600,3),s&&a===1024&&l===768&&t(b.defaultViewport1024x768,3),Number.isFinite(d)&&Number.isFinite(u)&&Number.isFinite(h)&&Number.isFinite(g)&&d>0&&u>0&&h>0&&g>0&&(d+8this.sendSessionReplayBatch(e)),await this.sessionReplayRecorder.initialize()}catch(e){console.error("Failed to initialize session replay:",e)}}async sendSessionReplayBatch(e){try{await fetch(`${this.config.analyticsHost}/session-replay/record/${this.config.siteId}`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e),mode:"cors",keepalive:!1})}catch(t){throw console.error("Failed to send session replay batch:",t),t}}createBasePayload(){let e=new URL(window.location.href),t=e.pathname;if(e.hash&&e.hash.startsWith("#/")&&(t=e.hash.substring(1)),j(t,this.config.skipPatterns))return null;let n=j(t,this.config.maskPatterns);n&&(t=n);let i={site_id:this.config.siteId,hostname:e.hostname,pathname:t,querystring:this.config.trackQuerystring?e.search:"",screenWidth:screen.width,screenHeight:screen.height,language:navigator.language,page_title:document.title,referrer:document.referrer,_bs:ge(),_bsm:fe()};this.customUserId&&(i.user_id=this.customUserId),this.config.tag&&(i.tag=this.config.tag);let s=this.getFeatureFlagEventPayload();return Object.keys(s).length>0&&(i.feature_flags=s),i}sendTrackingData(e){let t=(async()=>{try{await fetch(`${this.config.analyticsHost}/track`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e),mode:"cors",keepalive:!0})}catch(n){console.error("Failed to send tracking data:",n)}})();return this.pendingTrackingRequests.add(t),t.finally(()=>this.pendingTrackingRequests.delete(t)),t}track(e,t="",n={}){if(e==="custom_event"&&(!t||typeof t!="string")){console.error("Event name is required and must be a string for custom events");return}let i=this.createBasePayload();if(!i)return;let a={...i,type:e,event_name:t,properties:["custom_event","outbound","error","button_click","copy","form_submit","input_change"].includes(e)?JSON.stringify(n):void 0};this.sendTrackingData(a)}trackPageview(){this.track("pageview")}trackEvent(e,t={}){this.track("custom_event",e,t)}getFeatureFlag(e,t){let n=this.config.featureFlags?.[e];if(!n)return t;let i=`${e}:${n.version}:${this.serializeFeatureFlagValue(n.value)}`;return this.exposedFeatureFlags.has(i)||(this.exposedFeatureFlags.add(i),this.trackEvent("feature_flag_exposure",{key:e,value:this.serializeFeatureFlagValue(n.value),version:n.version,reason:n.reason})),n.value}getFeatureFlags(){return Object.fromEntries(Object.entries(this.config.featureFlags||{}).map(([e,t])=>[e,t.value]))}getFeatureFlagPayload(e,t){let n=this.config.featureFlags?.[e];return!n||n.payload===void 0?t:n.payload}getFeatureFlagPayloads(){return Object.fromEntries(Object.entries(this.config.featureFlags||{}).filter(([,e])=>e.payload!==void 0).map(([e,t])=>[e,t.payload]))}trackOutbound(e,t="",n="_self"){this.track("outbound","",{url:e,text:t,target:n})}trackWebVitals(e){let t=this.createBasePayload();if(!t)return;let n={...t,type:"performance",event_name:"web-vitals",...e};this.sendTrackingData(n)}trackError(e,t={}){let n=e?.message||"";if(n.includes("ResizeObserver loop completed with undelivered notifications")||n.includes("ResizeObserver loop limit exceeded"))return;let i=window.location.origin,s=t.filename||"",a=e.stack||"";if(s)try{if(new URL(s).origin!==i)return}catch{}else if(a&&!a.includes(i))return;let d=[e.name||"Error",n,t.filename||"",t.lineno??"",t.colno??""].join("|"),u=Date.now(),h=6e4,g=this.errorDedupeCache.get(d);if(g&&u-gh){for(let[o,c]of this.errorDedupeCache.entries())u-c>C&&this.errorDedupeCache.delete(o);this.errorDedupeLastCleanup=u}let m={message:e.message?.substring(0,500)||"Unknown error",stack:a.substring(0,2e3)||""};if(s&&(m.fileName=s),t.lineno){let o=typeof t.lineno=="string"?parseInt(t.lineno,10):t.lineno;o&&o!==0&&(m.lineNumber=o)}if(t.colno){let o=typeof t.colno=="string"?parseInt(t.colno,10):t.colno;o&&o!==0&&(m.columnNumber=o)}for(let o in t)!["lineno","colno"].includes(o)&&t[o]!==void 0&&(m[o]=t[o]);this.track("error",e.name||"Error",m)}trackButtonClick(e){this.track("button_click","",e)}trackCopy(e){this.track("copy","",e)}trackFormSubmit(e){this.track("form_submit","",e)}trackInputChange(e){this.track("input_change","",e)}identify(e,t){if(typeof e!="string"||e.trim()===""){console.error("User ID must be a non-empty string");return}this.customUserId=e.trim();try{localStorage.setItem(`${this.config.namespace}-user-id`,this.customUserId)}catch{console.warn("Could not persist user ID to localStorage")}let n=this.customUserId,i=[...this.pendingTrackingRequests];Promise.allSettled(i).then(()=>this.sendIdentifyEvent(n,t,!0)).then(()=>this.refreshFeatureFlags()),this.sessionReplayRecorder&&this.sessionReplayRecorder.updateUserId(this.customUserId)}setTraits(e){if(!e||typeof e!="object"){console.error("Traits must be an object");return}let t=this.customUserId;if(!t){console.warn("Cannot set traits without identifying user first. Call identify() first.");return}this.sendIdentifyEvent(t,e,!1).then(()=>this.refreshFeatureFlags())}async sendIdentifyEvent(e,t,n=!0){try{await fetch(`${this.config.analyticsHost}/identify`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({site_id:this.config.siteId,user_id:e,traits:t,is_new_identify:n}),mode:"cors",keepalive:!0})}catch(i){console.error("Failed to send identify event:",i)}}clearUserId(){this.customUserId=null;try{localStorage.removeItem(`${this.config.namespace}-user-id`)}catch{}this.sessionReplayRecorder&&this.sessionReplayRecorder.updateUserId(""),this.refreshFeatureFlags()}getUserId(){return this.customUserId}startSessionReplay(){this.sessionReplayRecorder?this.sessionReplayRecorder.startRecording():console.warn("Session replay not initialized")}stopSessionReplay(){this.sessionReplayRecorder&&this.sessionReplayRecorder.stopRecording()}isSessionReplayActive(){return this.sessionReplayRecorder?.isActive()??!1}onPageChange(){this.refreshFeatureFlags(),this.sessionReplayRecorder&&this.sessionReplayRecorder.onPageChange()}cleanup(){this.sessionReplayRecorder&&this.sessionReplayRecorder.cleanup()}};var Ee=-1,I=r=>{addEventListener("pageshow",(e=>{e.persisted&&(Ee=e.timeStamp,r(e))}),!0)},v=(r,e,t,n)=>{let i,s;return a=>{e.value>=0&&(a||n)&&(s=e.value-(i??0),(s||i===void 0)&&(i=e.value,e.delta=s,e.rating=((l,d)=>l>d[1]?"poor":l>d[0]?"needs-improvement":"good")(e.value,t),r(e)))}},Y=r=>{requestAnimationFrame((()=>requestAnimationFrame((()=>r()))))},Z=()=>{let r=performance.getEntriesByType("navigation")[0];if(r&&r.responseStart>0&&r.responseStartZ()?.activationStart??0,k=(r,e=-1)=>{let t=Z(),n="navigate";return Ee>=0?n="back-forward-cache":t&&(document.prerendering||_()>0?n="prerender":document.wasDiscarded?n="restore":t.type&&(n=t.type.replace(/_/g,"-"))),{name:r,value:e,rating:"good",delta:0,entries:[],id:`v5-${Date.now()}-${Math.floor(8999999999999*Math.random())+1e12}`,navigationType:n}},V=new WeakMap;function ee(r,e){return V.get(r)||V.set(r,new e),V.get(r)}var q=class{constructor(){R(this,"t");R(this,"i",0);R(this,"o",[])}h(e){if(e.hadRecentInput)return;let t=this.o[0],n=this.o.at(-1);this.i&&t&&n&&e.startTime-n.startTime<1e3&&e.startTime-t.startTime<5e3?(this.i+=e.value,this.o.push(e)):(this.i=e.value,this.o=[e]),this.t?.(e)}},L=(r,e,t={})=>{try{if(PerformanceObserver.supportedEntryTypes.includes(r)){let n=new PerformanceObserver((i=>{Promise.resolve().then((()=>{e(i.getEntries())}))}));return n.observe({type:r,buffered:!0,...t}),n}}catch{}},te=r=>{let e=!1;return()=>{e||(r(),e=!0)}},P=-1,Ce=new Set,ye=()=>document.visibilityState!=="hidden"||document.prerendering?1/0:0,G=r=>{if(document.visibilityState==="hidden"){if(r.type==="visibilitychange")for(let e of Ce)e();isFinite(P)||(P=r.type==="visibilitychange"?r.timeStamp:0,removeEventListener("prerenderingchange",G,!0))}},B=()=>{if(P<0){let r=_();P=(document.prerendering?void 0:globalThis.performance.getEntriesByType("visibility-state").filter((t=>t.name==="hidden"&&t.startTime>r))[0]?.startTime)??ye(),addEventListener("visibilitychange",G,!0),addEventListener("prerenderingchange",G,!0),I((()=>{setTimeout((()=>{P=ye()}))}))}return{get firstHiddenTime(){return P},onHidden(r){Ce.add(r)}}},O=r=>{document.prerendering?addEventListener("prerenderingchange",(()=>r()),!0):r()},be=[1800,3e3],ne=(r,e={})=>{O((()=>{let t=B(),n,i=k("FCP"),s=L("paint",(a=>{for(let l of a)l.name==="first-contentful-paint"&&(s.disconnect(),l.startTime{i=k("FCP"),n=v(r,i,be,e.reportAllChanges),Y((()=>{i.value=performance.now()-a.timeStamp,n(!0)}))})))}))},ve=[.1,.25],Pe=(r,e={})=>{let t=B();ne(te((()=>{let n,i=k("CLS",0),s=ee(e,q),a=d=>{for(let u of d)s.h(u);s.i>i.value&&(i.value=s.i,i.entries=s.o,n())},l=L("layout-shift",a);l&&(n=v(r,i,ve,e.reportAllChanges),t.onHidden((()=>{a(l.takeRecords()),n(!0)})),I((()=>{s.i=0,i=k("CLS",0),n=v(r,i,ve,e.reportAllChanges),Y((()=>n()))})),setTimeout(n))})))},Ie=0,z=1/0,M=0,Xe=r=>{for(let e of r)e.interactionId&&(z=Math.min(z,e.interactionId),M=Math.max(M,e.interactionId),Ie=M?(M-z)/7+1:0)},K,ke=()=>K?Ie:performance.interactionCount??0,Qe=()=>{"interactionCount"in performance||K||(K=L("event",Xe,{type:"event",buffered:!0,durationThreshold:0}))},we=0,J=class{constructor(){R(this,"u",[]);R(this,"l",new Map);R(this,"m");R(this,"p")}v(){we=ke(),this.u.length=0,this.l.clear()}L(){let e=Math.min(this.u.length-1,Math.floor((ke()-we)/50));return this.u[e]}h(e){if(this.m?.(e),!e.interactionId&&e.entryType!=="first-input")return;let t=this.u.at(-1),n=this.l.get(e.interactionId);if(n||this.u.length<10||e.duration>t.P){if(n?e.duration>n.P?(n.entries=[e],n.P=e.duration):e.duration===n.P&&e.startTime===n.entries[0].startTime&&n.entries.push(e):(n={id:e.interactionId,entries:[e],P:e.duration},this.l.set(n.id,n),this.u.push(n)),this.u.sort(((i,s)=>s.P-i.P)),this.u.length>10){let i=this.u.splice(10);for(let s of i)this.l.delete(s.id)}this.p?.(n)}}},_e=r=>{let e=globalThis.requestIdleCallback||setTimeout;document.visibilityState==="hidden"?r():(r=te(r),addEventListener("visibilitychange",r,{once:!0,capture:!0}),e((()=>{r(),removeEventListener("visibilitychange",r,{capture:!0})})))},Re=[200,500],Le=(r,e={})=>{if(!globalThis.PerformanceEventTiming||!("interactionId"in PerformanceEventTiming.prototype))return;let t=B();O((()=>{Qe();let n,i=k("INP"),s=ee(e,J),a=d=>{_e((()=>{for(let h of d)s.h(h);let u=s.L();u&&u.P!==i.value&&(i.value=u.P,i.entries=u.entries,n())}))},l=L("event",a,{durationThreshold:e.durationThreshold??40});n=v(r,i,Re,e.reportAllChanges),l&&(l.observe({type:"first-input",buffered:!0}),t.onHidden((()=>{a(l.takeRecords()),n(!0)})),I((()=>{s.v(),i=k("INP"),n=v(r,i,Re,e.reportAllChanges)})))}))},X=class{constructor(){R(this,"m")}h(e){this.m?.(e)}},Se=[2500,4e3],Ae=(r,e={})=>{O((()=>{let t=B(),n,i=k("LCP"),s=ee(e,X),a=d=>{e.reportAllChanges||(d=d.slice(-1));for(let u of d)s.h(u),u.startTime{a(l.takeRecords()),l.disconnect(),n(!0)})),u=h=>{h.isTrusted&&(_e(d),removeEventListener(h.type,u,{capture:!0}))};for(let h of["keydown","click","visibilitychange"])addEventListener(h,u,{capture:!0});I((h=>{i=k("LCP"),n=v(r,i,Se,e.reportAllChanges),Y((()=>{i.value=performance.now()-h.timeStamp,n(!0)}))}))}}))},Te=[800,1800],Q=r=>{document.prerendering?O((()=>Q(r))):document.readyState!=="complete"?addEventListener("load",(()=>Q(r)),!0):setTimeout(r)},Fe=(r,e={})=>{let t=k("TTFB"),n=v(r,t,Te,e.reportAllChanges);Q((()=>{let i=Z();i&&(t.value=Math.max(i.responseStart-_(),0),t.entries=[i],n(!0),I((()=>{t=k("TTFB",0),n=v(r,t,Te,e.reportAllChanges),n(!0)})))}))};var N=class{constructor(e){this.data={lcp:null,cls:null,inp:null,fcp:null,ttfb:null};this.sent=!1;this.timeout=null;this.onReadyCallback=null;this.onReadyCallback=e}initialize(){try{Ae(this.collectMetric.bind(this)),Pe(this.collectMetric.bind(this)),Le(this.collectMetric.bind(this)),ne(this.collectMetric.bind(this)),Fe(this.collectMetric.bind(this)),this.timeout=setTimeout(()=>{this.sent||this.sendData()},2e4),window.addEventListener("beforeunload",()=>{this.sent||this.sendData()})}catch(e){console.warn("Error initializing web vitals tracking:",e)}}collectMetric(e){if(this.sent)return;let t=e.name.toLowerCase();this.data[t]=e.value,Object.values(this.data).every(i=>i!==null)&&this.sendData()}sendData(){this.sent||(this.sent=!0,this.timeout&&(clearTimeout(this.timeout),this.timeout=null),this.onReadyCallback&&this.onReadyCallback(this.data))}getData(){return{...this.data}}};var D=class{constructor(e,t){this.tracker=e,this.config=t}initialize(){document.addEventListener("click",this.handleClick.bind(this),!0)}handleClick(e){let t=e.target;this.config.trackButtonClicks&&this.isButton(t)&&this.trackButtonClick(t)}isButton(e){if(e.tagName==="BUTTON"||e.getAttribute("role")==="button")return!0;if(e.tagName==="INPUT"){let i=e.type?.toLowerCase();if(i==="submit"||i==="button")return!0}let t=e.parentElement,n=0;for(;t&&n<3;){if(t.tagName==="BUTTON"||t.getAttribute("role")==="button")return!0;t=t.parentElement,n++}return!1}trackButtonClick(e){let t=this.findButton(e);if(!t||t.hasAttribute("data-rybbit-event"))return;let n={text:this.getElementText(t),...this.extractDataAttributes(t)};this.tracker.trackButtonClick(n)}extractDataAttributes(e){let t={};for(let n of e.attributes)if(n.name.startsWith("data-rybbit-prop-")){let i=n.name.replace("data-rybbit-prop-","");t[i]=n.value}return t}findButton(e){if(e.tagName==="BUTTON"||e.getAttribute("role")==="button")return e;if(e.tagName==="INPUT"){let i=e.type?.toLowerCase();if(i==="submit"||i==="button")return e}let t=e.parentElement,n=0;for(;t&&n<3;){if(t.tagName==="BUTTON"||t.getAttribute("role")==="button")return t;t=t.parentElement,n++}return null}getElementText(e){let t=e.textContent?.trim().substring(0,100);if(t)return t;let n=e.getAttribute("aria-label")?.trim().substring(0,100);if(n)return n;if(e.tagName==="INPUT"){let s=e.value?.trim().substring(0,100);if(s)return s}let i=e.getAttribute("title")?.trim().substring(0,100);if(i)return i}cleanup(){document.removeEventListener("click",this.handleClick.bind(this),!0)}};var U=class{constructor(e){this.tracker=e}initialize(){document.addEventListener("copy",this.handleCopy.bind(this))}handleCopy(){let e=window.getSelection();if(!e||e.isCollapsed)return;let t=e.toString(),n=t.length;if(n===0)return;let i=e.anchorNode,s=i instanceof HTMLElement?i:i?.parentElement;if(!s)return;let a={text:t.substring(0,500),...n>500&&{textLength:n},sourceElement:s.tagName.toLowerCase()};this.tracker.trackCopy(a)}cleanup(){document.removeEventListener("copy",this.handleCopy.bind(this))}};var H=class{constructor(e,t){this.tracker=e,this.config=t,this.boundHandleSubmit=this.handleSubmit.bind(this),this.boundHandleChange=this.handleChange.bind(this)}initialize(){document.addEventListener("submit",this.boundHandleSubmit,!0),document.addEventListener("change",this.boundHandleChange,!0)}cleanup(){document.removeEventListener("submit",this.boundHandleSubmit,!0),document.removeEventListener("change",this.boundHandleChange,!0)}handleSubmit(e){let t=e.target;if(t.tagName!=="FORM")return;let n={formId:t.id||"",formName:t.name||"",formAction:t.action||"",method:(t.method||"get").toUpperCase(),fieldCount:t.elements.length,ariaLabel:t.getAttribute("aria-label")||void 0,...this.extractDataAttributes(t)};this.tracker.trackFormSubmit(n)}handleChange(e){let t=e.target,n=t.tagName.toUpperCase();if(!["INPUT","SELECT","TEXTAREA"].includes(n)||t.disabled||t.readOnly)return;if(n==="INPUT"){let a=t.type?.toLowerCase();if(a==="hidden"||a==="password")return}let i=t.name||t.id||t.getAttribute("aria-label")||t.placeholder||"",s={element:n.toLowerCase(),inputType:n==="INPUT"?t.type?.toLowerCase():void 0,inputName:i,formId:t.form?.id||void 0,formName:t.form?.name||void 0,...this.extractDataAttributes(t)};this.tracker.trackInputChange(s)}extractDataAttributes(e){let t={};for(let n of e.attributes)if(n.name.startsWith("data-rybbit-prop-")){let i=n.name.replace("data-rybbit-prop-","");t[i]=n.value}return t}};(async function(){let r=document.currentScript;if(!r){console.error("Could not find current script tag");return}let e=r.getAttribute("data-namespace")||"rybbit",t=`disable-${e}`;if(window.__RYBBIT_OPTOUT__||localStorage.getItem(t)!==null){window[e]={pageview:()=>{},event:()=>{},error:()=>{},trackOutbound:()=>{},identify:()=>{},setTraits:()=>{},clearUserId:()=>{},getUserId:()=>null,flag:(o,c)=>c,flagPayload:(o,c)=>c,flags:()=>({}),flagPayloads:()=>({}),onReady:()=>{},startSessionReplay:()=>{},stopSessionReplay:()=>{},isSessionReplayActive:()=>!1};return}let n=[],i=o=>(...c)=>{n.push([o,c])};window[e]={pageview:i("pageview"),event:i("event"),error:i("error"),trackOutbound:i("trackOutbound"),identify:i("identify"),setTraits:i("setTraits"),clearUserId:i("clearUserId"),getUserId:()=>null,flag:(o,c)=>c,flagPayload:(o,c)=>c,flags:()=>({}),flagPayloads:()=>({}),onReady:i("onReady"),startSessionReplay:i("startSessionReplay"),stopSessionReplay:i("stopSessionReplay"),isSessionReplayActive:()=>!1};let s=await de(r);if(!s)return;let a=new x(s);s.enableWebVitals&&new N(c=>{a.trackWebVitals(c)}).initialize();let l=null,d=null,u=null;s.trackButtonClicks&&(l=new D(a,s),l.initialize()),s.trackCopy&&(d=new U(a),d.initialize()),s.trackFormInteractions&&(u=new H(a,s),u.initialize()),s.trackErrors&&(window.addEventListener("error",o=>{a.trackError(o.error||new Error(o.message),{filename:o.filename,lineno:o.lineno,colno:o.colno})}),window.addEventListener("unhandledrejection",o=>{let c=o.reason instanceof Error?o.reason:new Error(String(o.reason));a.trackError(c,{type:"unhandledrejection"})}));let h=()=>a.trackPageview(),g=s.debounceDuration>0?ce(h,s.debounceDuration):h;function C(){if(document.addEventListener("click",function(o){let c=o.target;for(;c&&c!==document.documentElement;){if(c.hasAttribute("data-rybbit-event")){let p=c.getAttribute("data-rybbit-event");if(p){let w={};for(let S of c.attributes)if(S.name.startsWith("data-rybbit-prop-")){let T=S.name.replace("data-rybbit-prop-","");w[T]=S.value}a.trackEvent(p,w)}break}c=c.parentElement}if(s.trackOutbound){let p=o.target.closest("a");p?.href&&le(p.href)&&a.trackOutbound(p.href,p.innerText||p.textContent||"",p.target||"_self")}}),s.autoTrackSpa){let o=history.pushState,c=history.replaceState;history.pushState=function(...p){o.apply(this,p),g(),a.onPageChange()},history.replaceState=function(...p){c.apply(this,p),g(),a.onPageChange()},window.addEventListener("popstate",()=>{g(),a.onPageChange()}),window.addEventListener("hashchange",()=>{g(),a.onPageChange()})}}window[s.namespace]={pageview:()=>a.trackPageview(),event:(o,c={})=>a.trackEvent(o,c),error:(o,c={})=>a.trackError(o,c),trackOutbound:(o,c="",p="_self")=>a.trackOutbound(o,c,p),identify:(o,c)=>a.identify(o,c),setTraits:o=>a.setTraits(o),clearUserId:()=>a.clearUserId(),getUserId:()=>a.getUserId(),flag:(o,c)=>a.getFeatureFlag(o,c),flagPayload:(o,c)=>a.getFeatureFlagPayload(o,c),flags:()=>a.getFeatureFlags(),flagPayloads:()=>a.getFeatureFlagPayloads(),onReady:o=>o(window[s.namespace]),startSessionReplay:()=>a.startSessionReplay(),stopSessionReplay:()=>a.stopSessionReplay(),isSessionReplayActive:()=>a.isSessionReplayActive()};let m=window[s.namespace];for(let[o,c]of n)m[o](...c);C(),window.addEventListener("beforeunload",()=>{l?.cleanup(),d?.cleanup(),a.cleanup()}),s.autoTrackPageview&&a.trackPageview()})();})(); +"use strict";(()=>{var De=Object.defineProperty;var Ue=(r,e,t)=>e in r?De(r,e,{enumerable:!0,configurable:!0,writable:!0,value:t}):r[e]=t;var R=(r,e,t)=>Ue(r,typeof e!="symbol"?e+"":e,t);function He(r){if(r.startsWith("re:")){let l=r.slice(3);if(!l)throw new Error("Empty regex pattern");return new RegExp(l)}let t="__DOUBLE_ASTERISK_TOKEN__",n="__SINGLE_ASTERISK_TOKEN__",s=r.replace(/\*\*/g,t).replace(/\*/g,n).replace(/[.+?^${}()|[\]\\]/g,"\\$&");s=s.replace(new RegExp(`/${t}/`,"g"),"/(?:.+/)?"),s=s.replace(new RegExp(t,"g"),".*"),s=s.replace(/\//g,"\\/");let a=s.replace(new RegExp(n,"g"),"[^/]+");return new RegExp("^"+a+"$")}function j(r,e){for(let t of e)try{if(He(t).test(r))return t}catch(n){console.error(`Invalid pattern: ${t}`,n)}return null}function ce(r,e){let t=null;return(...n)=>{t&&clearTimeout(t),t=setTimeout(()=>r(...n),e)}}function le(r){try{let e=window.location.hostname,t=new URL(r).hostname;return t!==e&&t!==""}catch{return!1}}function E(r,e){if(!r)return e;try{let t=JSON.parse(r);return Array.isArray(e)&&!Array.isArray(t)?e:t}catch(t){return console.error("Error parsing JSON:",t),e}}function ue(){try{if(crypto?.randomUUID)return crypto.randomUUID()}catch{}return`${Date.now().toString(36)}-${Math.random().toString(36).slice(2,12)}`}function We(r){let e=`${r}-visitor-id`;try{let t=localStorage.getItem(e);if(t)return t;let n=ue();return localStorage.setItem(e,n),n}catch{return ue()}}function $e(r){try{return localStorage.getItem(`${r}-user-id`)||void 0}catch{return}}function je(r){return r.hash&&r.hash.startsWith("#/")?r.hash.substring(1):r.pathname}async function Ve(r,e,t,n){try{let i=new URL(window.location.href),s=await fetch(`${r}/site/${e}/feature-flags/evaluate`,{method:"POST",headers:{"Content-Type":"application/json"},credentials:"omit",body:JSON.stringify({anonymousId:n,identifiedUserId:$e(t),hostname:i.hostname,pathname:je(i),querystring:i.search,query:Object.fromEntries(i.searchParams.entries()),referrer:document.referrer,language:navigator.language,screenWidth:screen.width,screenHeight:screen.height})});if(!s.ok)return{};let a=await s.json();return a?.flags&&typeof a.flags=="object"?a.flags:{}}catch{return{}}}async function de(r){let e=r.getAttribute("src");if(!e)return console.error("Script src attribute is missing"),null;let t=e.split("/script.js")[0];if(!t)return console.error("Please provide a valid analytics host"),null;let n=r.getAttribute("data-site-id")||r.getAttribute("site-id");if(!n)return console.error("Please provide a valid site ID using the data-site-id attribute"),null;let i=r.getAttribute("data-namespace")||"rybbit",s=We(i),a=E(r.getAttribute("data-skip-patterns"),[]),l=E(r.getAttribute("data-mask-patterns"),[]),d=E(r.getAttribute("data-replay-mask-text-selectors"),[]),u=r.getAttribute("data-debounce")?Math.max(0,parseInt(r.getAttribute("data-debounce"))):500,h=r.getAttribute("data-replay-batch-size")?Math.max(1,parseInt(r.getAttribute("data-replay-batch-size"))):250,g=r.getAttribute("data-replay-batch-interval")?Math.max(1e3,parseInt(r.getAttribute("data-replay-batch-interval"))):5e3,C=r.getAttribute("data-replay-block-class")||void 0,m=r.getAttribute("data-replay-block-selector")||void 0,o=r.getAttribute("data-replay-ignore-class")||void 0,c=r.getAttribute("data-replay-ignore-selector")||void 0,p=r.getAttribute("data-replay-mask-text-class")||void 0,w=r.getAttribute("data-replay-mask-all-inputs"),S=w!==null?w!=="false":void 0,T=r.getAttribute("data-replay-mask-input-options"),A=T?E(T,{password:!0,email:!0}):void 0,re=r.getAttribute("data-replay-collect-fonts"),xe=re!==null?re!=="false":void 0,ie=r.getAttribute("data-replay-sampling"),Me=ie?E(ie,{}):void 0,se=r.getAttribute("data-replay-slim-dom-options"),Be=se?E(se,{}):void 0,ae=r.getAttribute("data-replay-sample-rate"),Oe=ae?Math.min(100,Math.max(0,parseInt(ae,10))):void 0,Ne=r.getAttribute("data-tag")||"",f={namespace:i,analyticsHost:t,siteId:n,visitorId:s,debounceDuration:u,sessionReplayBatchSize:h,sessionReplayBatchInterval:g,sessionReplayMaskTextSelectors:d,skipPatterns:a,maskPatterns:l,autoTrackPageview:!0,autoTrackSpa:!0,trackQuerystring:!0,trackOutbound:!0,enableWebVitals:!1,trackErrors:!1,enableSessionReplay:!1,trackButtonClicks:!1,trackCopy:!1,trackFormInteractions:!1,tag:Ne,featureFlags:{},sessionReplayBlockClass:C,sessionReplayBlockSelector:m,sessionReplayIgnoreClass:o,sessionReplayIgnoreSelector:c,sessionReplayMaskTextClass:p,sessionReplayMaskAllInputs:S,sessionReplayMaskInputOptions:A,sessionReplayCollectFonts:xe,sessionReplaySampling:Me,sessionReplaySlimDOMOptions:Be,sessionReplaySampleRate:Oe},W=f;try{let $=`${t}/site/tracking-config/${n}`,oe=await fetch($,{method:"GET",credentials:"omit"});if(oe.ok){let y=await oe.json();W={...f,autoTrackPageview:y.trackInitialPageView??f.autoTrackPageview,autoTrackSpa:y.trackSpaNavigation??f.autoTrackSpa,trackQuerystring:y.trackUrlParams??f.trackQuerystring,trackOutbound:y.trackOutbound??f.trackOutbound,enableWebVitals:y.webVitals??f.enableWebVitals,trackErrors:y.trackErrors??f.trackErrors,enableSessionReplay:y.sessionReplay??f.enableSessionReplay,trackButtonClicks:y.trackButtonClicks??f.trackButtonClicks,trackCopy:y.trackCopy??f.trackCopy,trackFormInteractions:y.trackFormInteractions??f.trackFormInteractions}}else console.warn("Failed to fetch tracking config from API, using defaults")}catch($){console.warn("Error fetching tracking config:",$)}return W.featureFlags=await Ve(t,n,i,s),W}var pe="rybbit-replay-sampled";function ze(){let r=new Uint8Array(16);if(globalThis.crypto?.getRandomValues)globalThis.crypto.getRandomValues(r);else for(let t=0;tt.toString(16).padStart(2,"0"));return`${e.slice(0,4).join("")}-${e.slice(4,6).join("")}-${e.slice(6,8).join("")}-${e.slice(8,10).join("")}-${e.slice(10).join("")}`}function qe(r){if(r>=100)return!0;if(r<=0)return!1;try{let e=sessionStorage.getItem(pe);if(e!==null)return e==="1";let t=Math.random()*100{let n=document.createElement("script");n.src=`${this.config.analyticsHost}/replay.js`,n.async=!1,n.onload=()=>{e()},n.onerror=()=>t(new Error("Failed to load rrweb")),document.head.appendChild(n)})}startRecording(){if(!(this.isRecording||!window.rrweb||!this.config.enableSessionReplay))try{let e={mousemove:!1,mouseInteraction:{MouseUp:!1,MouseDown:!1,Click:!0,ContextMenu:!1,DblClick:!0,Focus:!0,Blur:!0,TouchStart:!1,TouchEnd:!1},scroll:500,input:"last",media:800},t={script:!1,comment:!0,headFavicon:!0,headWhitespace:!0,headMetaDescKeywords:!0,headMetaSocial:!0,headMetaRobots:!0,headMetaHttpEquiv:!0,headMetaAuthorship:!0,headMetaVerification:!0},n={emit:i=>{this.addEvent({type:i.type,data:i.data,timestamp:i.timestamp||Date.now()})},recordCanvas:!1,checkoutEveryNms:6e4,checkoutEveryNth:500,blockClass:this.config.sessionReplayBlockClass??"rr-block",blockSelector:this.config.sessionReplayBlockSelector??null,ignoreClass:this.config.sessionReplayIgnoreClass??"rr-ignore",ignoreSelector:this.config.sessionReplayIgnoreSelector??null,maskTextClass:this.config.sessionReplayMaskTextClass??"rr-mask",maskAllInputs:this.config.sessionReplayMaskAllInputs??!0,maskInputOptions:this.config.sessionReplayMaskInputOptions??{password:!0,email:!0},collectFonts:this.config.sessionReplayCollectFonts??!0,sampling:this.config.sessionReplaySampling??e,slimDOMOptions:this.config.sessionReplaySlimDOMOptions??t};this.config.sessionReplayMaskTextSelectors&&this.config.sessionReplayMaskTextSelectors.length>0&&(n.maskTextSelector=this.config.sessionReplayMaskTextSelectors.join(", ")),this.stopRecordingFn=window.rrweb.record(n),this.isRecording=!0,this.setupBatchTimer()}catch{}}stopRecording(){this.isRecording&&(this.stopRecordingFn&&this.stopRecordingFn(),this.isRecording=!1,this.clearBatchTimer(),(this.eventBuffer.length>0||this.pendingBatches.length>0)&&this.flushEvents())}isActive(){return this.isRecording}addEvent(e){this.eventBuffer.push({...e,sequence:this.nextSequence++}),this.eventBuffer.length>=this.config.sessionReplayBatchSize&&this.flushEvents()}setupBatchTimer(){this.clearBatchTimer(),this.batchTimer=window.setInterval(()=>{(this.eventBuffer.length>0||this.pendingBatches.length>0)&&this.flushEvents()},this.config.sessionReplayBatchInterval)}clearBatchTimer(){this.batchTimer&&(clearInterval(this.batchTimer),this.batchTimer=void 0)}async flushEvents(){if(this.eventBuffer.length>0){let e={batchId:ze(),userId:this.userId,events:this.eventBuffer,metadata:{pageUrl:window.location.href,viewportWidth:screen.width,viewportHeight:screen.height,language:navigator.language}};this.eventBuffer=[],this.pendingBatches.push(e)}await this.drainPendingBatches()}drainPendingBatches(){return this.sendLoop?this.sendLoop.then(()=>{if(this.pendingBatches.length>0&&this.sendLoop===null)return this.drainPendingBatches()}):(this.sendLoop=(async()=>{for(;this.pendingBatches.length>0;)try{await this.sendBatch(this.pendingBatches[0]),this.pendingBatches.shift()}catch{return}})().finally(()=>{this.sendLoop=null}),this.sendLoop)}updateUserId(e){e!==this.userId&&(this.eventBuffer.length>0&&this.flushEvents(),this.userId=e)}onPageChange(){this.isRecording&&this.flushEvents()}cleanup(){this.stopRecording()}};var b={automationApi:1,webdriver:1,zeroOuterDimensions:2,missingChrome:4,swiftShader:8,emptyPlugins:16,defaultViewport800x600:32,defaultViewport1024x768:64,impossibleDimensions:128,outerDimensionsWeird:256,pluginApiAbsence:512},he=null,Ge=10;function ge(){return me().score}function fe(){return me().mask}function me(){return he??(he=Ke()),he}function Ke(){let r=0,e=0;function t(n,i){(e&n)===0&&(e|=n,r+=i)}try{let n=navigator.userAgent,i=/Chrome\//.test(n)&&!/\bwv\b|; wv\)/.test(n),s=/Windows NT|Macintosh|X11|Linux x86_64/.test(n)&&!/Mobile|Android|iPhone|iPad/.test(n),a=Number(window.screen?.width),l=Number(window.screen?.height),d=Number(window.outerWidth),u=Number(window.outerHeight),h=Number(window.innerWidth),g=Number(window.innerHeight),m=["__webdriver_evaluate","__selenium_evaluate","__webdriver_script_function","__webdriver_script_func","__webdriver_script_fn","__fxdriver_evaluate","__driver_unwrapped","__webdriver_unwrapped","__driver_evaluate","__selenium_unwrapped","__fxdriver_unwrapped","_phantom","callPhantom","__nightmare","domAutomation","domAutomationController"].some(c=>c in window||c in document);(navigator.webdriver===!0||m)&&t(b.automationApi,3),(u===0||d===0)&&t(b.zeroOuterDimensions,2),(!Number.isFinite(a)||!Number.isFinite(l)||a<=0||l<=0||a>1e5||l>1e5)&&t(b.impossibleDimensions,3),s&&a===800&&l===600&&t(b.defaultViewport800x600,3),s&&a===1024&&l===768&&t(b.defaultViewport1024x768,3),Number.isFinite(d)&&Number.isFinite(u)&&Number.isFinite(h)&&Number.isFinite(g)&&d>0&&u>0&&h>0&&g>0&&(d+8this.sendSessionReplayBatch(e)),await this.sessionReplayRecorder.initialize()}catch(e){console.error("Failed to initialize session replay:",e)}}async sendSessionReplayBatch(e){try{let t=await fetch(`${this.config.analyticsHost}/session-replay/record/${this.config.siteId}`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e),mode:"cors",keepalive:!1});if(!t.ok)throw new Error(`Session replay delivery failed: ${t.status} ${t.statusText}`.trim())}catch(t){throw console.error("Failed to send session replay batch:",t),t}}createBasePayload(){let e=new URL(window.location.href),t=e.pathname;if(e.hash&&e.hash.startsWith("#/")&&(t=e.hash.substring(1)),j(t,this.config.skipPatterns))return null;let n=j(t,this.config.maskPatterns);n&&(t=n);let i={site_id:this.config.siteId,hostname:e.hostname,pathname:t,querystring:this.config.trackQuerystring?e.search:"",screenWidth:screen.width,screenHeight:screen.height,language:navigator.language,page_title:document.title,referrer:document.referrer,_bs:ge(),_bsm:fe()};this.customUserId&&(i.user_id=this.customUserId),this.config.tag&&(i.tag=this.config.tag);let s=this.getFeatureFlagEventPayload();return Object.keys(s).length>0&&(i.feature_flags=s),i}sendTrackingData(e){let t=(async()=>{try{await fetch(`${this.config.analyticsHost}/track`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(e),mode:"cors",keepalive:!0})}catch(n){console.error("Failed to send tracking data:",n)}})();return this.pendingTrackingRequests.add(t),t.finally(()=>this.pendingTrackingRequests.delete(t)),t}track(e,t="",n={}){if(e==="custom_event"&&(!t||typeof t!="string")){console.error("Event name is required and must be a string for custom events");return}let i=this.createBasePayload();if(!i)return;let a={...i,type:e,event_name:t,properties:["custom_event","outbound","error","button_click","copy","form_submit","input_change"].includes(e)?JSON.stringify(n):void 0};this.sendTrackingData(a)}trackPageview(){this.track("pageview")}trackEvent(e,t={}){this.track("custom_event",e,t)}getFeatureFlag(e,t){let n=this.config.featureFlags?.[e];if(!n)return t;let i=`${e}:${n.version}:${this.serializeFeatureFlagValue(n.value)}`;return this.exposedFeatureFlags.has(i)||(this.exposedFeatureFlags.add(i),this.trackEvent("feature_flag_exposure",{key:e,value:this.serializeFeatureFlagValue(n.value),version:n.version,reason:n.reason})),n.value}getFeatureFlags(){return Object.fromEntries(Object.entries(this.config.featureFlags||{}).map(([e,t])=>[e,t.value]))}getFeatureFlagPayload(e,t){let n=this.config.featureFlags?.[e];return!n||n.payload===void 0?t:n.payload}getFeatureFlagPayloads(){return Object.fromEntries(Object.entries(this.config.featureFlags||{}).filter(([,e])=>e.payload!==void 0).map(([e,t])=>[e,t.payload]))}trackOutbound(e,t="",n="_self"){this.track("outbound","",{url:e,text:t,target:n})}trackWebVitals(e){let t=this.createBasePayload();if(!t)return;let n={...t,type:"performance",event_name:"web-vitals",...e};this.sendTrackingData(n)}trackError(e,t={}){let n=e?.message||"";if(n.includes("ResizeObserver loop completed with undelivered notifications")||n.includes("ResizeObserver loop limit exceeded"))return;let i=window.location.origin,s=t.filename||"",a=e.stack||"";if(s)try{if(new URL(s).origin!==i)return}catch{}else if(a&&!a.includes(i))return;let d=[e.name||"Error",n,t.filename||"",t.lineno??"",t.colno??""].join("|"),u=Date.now(),h=6e4,g=this.errorDedupeCache.get(d);if(g&&u-gh){for(let[o,c]of this.errorDedupeCache.entries())u-c>C&&this.errorDedupeCache.delete(o);this.errorDedupeLastCleanup=u}let m={message:e.message?.substring(0,500)||"Unknown error",stack:a.substring(0,2e3)||""};if(s&&(m.fileName=s),t.lineno){let o=typeof t.lineno=="string"?parseInt(t.lineno,10):t.lineno;o&&o!==0&&(m.lineNumber=o)}if(t.colno){let o=typeof t.colno=="string"?parseInt(t.colno,10):t.colno;o&&o!==0&&(m.columnNumber=o)}for(let o in t)!["lineno","colno"].includes(o)&&t[o]!==void 0&&(m[o]=t[o]);this.track("error",e.name||"Error",m)}trackButtonClick(e){this.track("button_click","",e)}trackCopy(e){this.track("copy","",e)}trackFormSubmit(e){this.track("form_submit","",e)}trackInputChange(e){this.track("input_change","",e)}identify(e,t){if(typeof e!="string"||e.trim()===""){console.error("User ID must be a non-empty string");return}this.customUserId=e.trim();try{localStorage.setItem(`${this.config.namespace}-user-id`,this.customUserId)}catch{console.warn("Could not persist user ID to localStorage")}let n=this.customUserId,i=[...this.pendingTrackingRequests];Promise.allSettled(i).then(()=>this.sendIdentifyEvent(n,t,!0)).then(()=>this.refreshFeatureFlags()),this.sessionReplayRecorder&&this.sessionReplayRecorder.updateUserId(this.customUserId)}setTraits(e){if(!e||typeof e!="object"){console.error("Traits must be an object");return}let t=this.customUserId;if(!t){console.warn("Cannot set traits without identifying user first. Call identify() first.");return}this.sendIdentifyEvent(t,e,!1).then(()=>this.refreshFeatureFlags())}async sendIdentifyEvent(e,t,n=!0){try{await fetch(`${this.config.analyticsHost}/identify`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({site_id:this.config.siteId,user_id:e,traits:t,is_new_identify:n}),mode:"cors",keepalive:!0})}catch(i){console.error("Failed to send identify event:",i)}}clearUserId(){this.customUserId=null;try{localStorage.removeItem(`${this.config.namespace}-user-id`)}catch{}this.sessionReplayRecorder&&this.sessionReplayRecorder.updateUserId(""),this.refreshFeatureFlags()}getUserId(){return this.customUserId}startSessionReplay(){this.sessionReplayRecorder?this.sessionReplayRecorder.startRecording():console.warn("Session replay not initialized")}stopSessionReplay(){this.sessionReplayRecorder&&this.sessionReplayRecorder.stopRecording()}isSessionReplayActive(){return this.sessionReplayRecorder?.isActive()??!1}onPageChange(){this.refreshFeatureFlags(),this.sessionReplayRecorder&&this.sessionReplayRecorder.onPageChange()}cleanup(){this.sessionReplayRecorder&&this.sessionReplayRecorder.cleanup()}};var Ee=-1,I=r=>{addEventListener("pageshow",(e=>{e.persisted&&(Ee=e.timeStamp,r(e))}),!0)},v=(r,e,t,n)=>{let i,s;return a=>{e.value>=0&&(a||n)&&(s=e.value-(i??0),(s||i===void 0)&&(i=e.value,e.delta=s,e.rating=((l,d)=>l>d[1]?"poor":l>d[0]?"needs-improvement":"good")(e.value,t),r(e)))}},Y=r=>{requestAnimationFrame((()=>requestAnimationFrame((()=>r()))))},Z=()=>{let r=performance.getEntriesByType("navigation")[0];if(r&&r.responseStart>0&&r.responseStartZ()?.activationStart??0,k=(r,e=-1)=>{let t=Z(),n="navigate";return Ee>=0?n="back-forward-cache":t&&(document.prerendering||_()>0?n="prerender":document.wasDiscarded?n="restore":t.type&&(n=t.type.replace(/_/g,"-"))),{name:r,value:e,rating:"good",delta:0,entries:[],id:`v5-${Date.now()}-${Math.floor(8999999999999*Math.random())+1e12}`,navigationType:n}},V=new WeakMap;function ee(r,e){return V.get(r)||V.set(r,new e),V.get(r)}var q=class{constructor(){R(this,"t");R(this,"i",0);R(this,"o",[])}h(e){if(e.hadRecentInput)return;let t=this.o[0],n=this.o.at(-1);this.i&&t&&n&&e.startTime-n.startTime<1e3&&e.startTime-t.startTime<5e3?(this.i+=e.value,this.o.push(e)):(this.i=e.value,this.o=[e]),this.t?.(e)}},L=(r,e,t={})=>{try{if(PerformanceObserver.supportedEntryTypes.includes(r)){let n=new PerformanceObserver((i=>{Promise.resolve().then((()=>{e(i.getEntries())}))}));return n.observe({type:r,buffered:!0,...t}),n}}catch{}},te=r=>{let e=!1;return()=>{e||(r(),e=!0)}},P=-1,Ce=new Set,ye=()=>document.visibilityState!=="hidden"||document.prerendering?1/0:0,G=r=>{if(document.visibilityState==="hidden"){if(r.type==="visibilitychange")for(let e of Ce)e();isFinite(P)||(P=r.type==="visibilitychange"?r.timeStamp:0,removeEventListener("prerenderingchange",G,!0))}},B=()=>{if(P<0){let r=_();P=(document.prerendering?void 0:globalThis.performance.getEntriesByType("visibility-state").filter((t=>t.name==="hidden"&&t.startTime>r))[0]?.startTime)??ye(),addEventListener("visibilitychange",G,!0),addEventListener("prerenderingchange",G,!0),I((()=>{setTimeout((()=>{P=ye()}))}))}return{get firstHiddenTime(){return P},onHidden(r){Ce.add(r)}}},O=r=>{document.prerendering?addEventListener("prerenderingchange",(()=>r()),!0):r()},be=[1800,3e3],ne=(r,e={})=>{O((()=>{let t=B(),n,i=k("FCP"),s=L("paint",(a=>{for(let l of a)l.name==="first-contentful-paint"&&(s.disconnect(),l.startTime{i=k("FCP"),n=v(r,i,be,e.reportAllChanges),Y((()=>{i.value=performance.now()-a.timeStamp,n(!0)}))})))}))},ve=[.1,.25],Pe=(r,e={})=>{let t=B();ne(te((()=>{let n,i=k("CLS",0),s=ee(e,q),a=d=>{for(let u of d)s.h(u);s.i>i.value&&(i.value=s.i,i.entries=s.o,n())},l=L("layout-shift",a);l&&(n=v(r,i,ve,e.reportAllChanges),t.onHidden((()=>{a(l.takeRecords()),n(!0)})),I((()=>{s.i=0,i=k("CLS",0),n=v(r,i,ve,e.reportAllChanges),Y((()=>n()))})),setTimeout(n))})))},Ie=0,z=1/0,M=0,Xe=r=>{for(let e of r)e.interactionId&&(z=Math.min(z,e.interactionId),M=Math.max(M,e.interactionId),Ie=M?(M-z)/7+1:0)},K,ke=()=>K?Ie:performance.interactionCount??0,Qe=()=>{"interactionCount"in performance||K||(K=L("event",Xe,{type:"event",buffered:!0,durationThreshold:0}))},we=0,J=class{constructor(){R(this,"u",[]);R(this,"l",new Map);R(this,"m");R(this,"p")}v(){we=ke(),this.u.length=0,this.l.clear()}L(){let e=Math.min(this.u.length-1,Math.floor((ke()-we)/50));return this.u[e]}h(e){if(this.m?.(e),!e.interactionId&&e.entryType!=="first-input")return;let t=this.u.at(-1),n=this.l.get(e.interactionId);if(n||this.u.length<10||e.duration>t.P){if(n?e.duration>n.P?(n.entries=[e],n.P=e.duration):e.duration===n.P&&e.startTime===n.entries[0].startTime&&n.entries.push(e):(n={id:e.interactionId,entries:[e],P:e.duration},this.l.set(n.id,n),this.u.push(n)),this.u.sort(((i,s)=>s.P-i.P)),this.u.length>10){let i=this.u.splice(10);for(let s of i)this.l.delete(s.id)}this.p?.(n)}}},_e=r=>{let e=globalThis.requestIdleCallback||setTimeout;document.visibilityState==="hidden"?r():(r=te(r),addEventListener("visibilitychange",r,{once:!0,capture:!0}),e((()=>{r(),removeEventListener("visibilitychange",r,{capture:!0})})))},Re=[200,500],Le=(r,e={})=>{if(!globalThis.PerformanceEventTiming||!("interactionId"in PerformanceEventTiming.prototype))return;let t=B();O((()=>{Qe();let n,i=k("INP"),s=ee(e,J),a=d=>{_e((()=>{for(let h of d)s.h(h);let u=s.L();u&&u.P!==i.value&&(i.value=u.P,i.entries=u.entries,n())}))},l=L("event",a,{durationThreshold:e.durationThreshold??40});n=v(r,i,Re,e.reportAllChanges),l&&(l.observe({type:"first-input",buffered:!0}),t.onHidden((()=>{a(l.takeRecords()),n(!0)})),I((()=>{s.v(),i=k("INP"),n=v(r,i,Re,e.reportAllChanges)})))}))},X=class{constructor(){R(this,"m")}h(e){this.m?.(e)}},Se=[2500,4e3],Ae=(r,e={})=>{O((()=>{let t=B(),n,i=k("LCP"),s=ee(e,X),a=d=>{e.reportAllChanges||(d=d.slice(-1));for(let u of d)s.h(u),u.startTime{a(l.takeRecords()),l.disconnect(),n(!0)})),u=h=>{h.isTrusted&&(_e(d),removeEventListener(h.type,u,{capture:!0}))};for(let h of["keydown","click","visibilitychange"])addEventListener(h,u,{capture:!0});I((h=>{i=k("LCP"),n=v(r,i,Se,e.reportAllChanges),Y((()=>{i.value=performance.now()-h.timeStamp,n(!0)}))}))}}))},Te=[800,1800],Q=r=>{document.prerendering?O((()=>Q(r))):document.readyState!=="complete"?addEventListener("load",(()=>Q(r)),!0):setTimeout(r)},Fe=(r,e={})=>{let t=k("TTFB"),n=v(r,t,Te,e.reportAllChanges);Q((()=>{let i=Z();i&&(t.value=Math.max(i.responseStart-_(),0),t.entries=[i],n(!0),I((()=>{t=k("TTFB",0),n=v(r,t,Te,e.reportAllChanges),n(!0)})))}))};var N=class{constructor(e){this.data={lcp:null,cls:null,inp:null,fcp:null,ttfb:null};this.sent=!1;this.timeout=null;this.onReadyCallback=null;this.onReadyCallback=e}initialize(){try{Ae(this.collectMetric.bind(this)),Pe(this.collectMetric.bind(this)),Le(this.collectMetric.bind(this)),ne(this.collectMetric.bind(this)),Fe(this.collectMetric.bind(this)),this.timeout=setTimeout(()=>{this.sent||this.sendData()},2e4),window.addEventListener("beforeunload",()=>{this.sent||this.sendData()})}catch(e){console.warn("Error initializing web vitals tracking:",e)}}collectMetric(e){if(this.sent)return;let t=e.name.toLowerCase();this.data[t]=e.value,Object.values(this.data).every(i=>i!==null)&&this.sendData()}sendData(){this.sent||(this.sent=!0,this.timeout&&(clearTimeout(this.timeout),this.timeout=null),this.onReadyCallback&&this.onReadyCallback(this.data))}getData(){return{...this.data}}};var D=class{constructor(e,t){this.tracker=e,this.config=t}initialize(){document.addEventListener("click",this.handleClick.bind(this),!0)}handleClick(e){let t=e.target;this.config.trackButtonClicks&&this.isButton(t)&&this.trackButtonClick(t)}isButton(e){if(e.tagName==="BUTTON"||e.getAttribute("role")==="button")return!0;if(e.tagName==="INPUT"){let i=e.type?.toLowerCase();if(i==="submit"||i==="button")return!0}let t=e.parentElement,n=0;for(;t&&n<3;){if(t.tagName==="BUTTON"||t.getAttribute("role")==="button")return!0;t=t.parentElement,n++}return!1}trackButtonClick(e){let t=this.findButton(e);if(!t||t.hasAttribute("data-rybbit-event"))return;let n={text:this.getElementText(t),...this.extractDataAttributes(t)};this.tracker.trackButtonClick(n)}extractDataAttributes(e){let t={};for(let n of e.attributes)if(n.name.startsWith("data-rybbit-prop-")){let i=n.name.replace("data-rybbit-prop-","");t[i]=n.value}return t}findButton(e){if(e.tagName==="BUTTON"||e.getAttribute("role")==="button")return e;if(e.tagName==="INPUT"){let i=e.type?.toLowerCase();if(i==="submit"||i==="button")return e}let t=e.parentElement,n=0;for(;t&&n<3;){if(t.tagName==="BUTTON"||t.getAttribute("role")==="button")return t;t=t.parentElement,n++}return null}getElementText(e){let t=e.textContent?.trim().substring(0,100);if(t)return t;let n=e.getAttribute("aria-label")?.trim().substring(0,100);if(n)return n;if(e.tagName==="INPUT"){let s=e.value?.trim().substring(0,100);if(s)return s}let i=e.getAttribute("title")?.trim().substring(0,100);if(i)return i}cleanup(){document.removeEventListener("click",this.handleClick.bind(this),!0)}};var U=class{constructor(e){this.tracker=e}initialize(){document.addEventListener("copy",this.handleCopy.bind(this))}handleCopy(){let e=window.getSelection();if(!e||e.isCollapsed)return;let t=e.toString(),n=t.length;if(n===0)return;let i=e.anchorNode,s=i instanceof HTMLElement?i:i?.parentElement;if(!s)return;let a={text:t.substring(0,500),...n>500&&{textLength:n},sourceElement:s.tagName.toLowerCase()};this.tracker.trackCopy(a)}cleanup(){document.removeEventListener("copy",this.handleCopy.bind(this))}};var H=class{constructor(e,t){this.tracker=e,this.config=t,this.boundHandleSubmit=this.handleSubmit.bind(this),this.boundHandleChange=this.handleChange.bind(this)}initialize(){document.addEventListener("submit",this.boundHandleSubmit,!0),document.addEventListener("change",this.boundHandleChange,!0)}cleanup(){document.removeEventListener("submit",this.boundHandleSubmit,!0),document.removeEventListener("change",this.boundHandleChange,!0)}handleSubmit(e){let t=e.target;if(t.tagName!=="FORM")return;let n={formId:t.id||"",formName:t.name||"",formAction:t.action||"",method:(t.method||"get").toUpperCase(),fieldCount:t.elements.length,ariaLabel:t.getAttribute("aria-label")||void 0,...this.extractDataAttributes(t)};this.tracker.trackFormSubmit(n)}handleChange(e){let t=e.target,n=t.tagName.toUpperCase();if(!["INPUT","SELECT","TEXTAREA"].includes(n)||t.disabled||t.readOnly)return;if(n==="INPUT"){let a=t.type?.toLowerCase();if(a==="hidden"||a==="password")return}let i=t.name||t.id||t.getAttribute("aria-label")||t.placeholder||"",s={element:n.toLowerCase(),inputType:n==="INPUT"?t.type?.toLowerCase():void 0,inputName:i,formId:t.form?.id||void 0,formName:t.form?.name||void 0,...this.extractDataAttributes(t)};this.tracker.trackInputChange(s)}extractDataAttributes(e){let t={};for(let n of e.attributes)if(n.name.startsWith("data-rybbit-prop-")){let i=n.name.replace("data-rybbit-prop-","");t[i]=n.value}return t}};(async function(){let r=document.currentScript;if(!r){console.error("Could not find current script tag");return}let e=r.getAttribute("data-namespace")||"rybbit",t=`disable-${e}`;if(window.__RYBBIT_OPTOUT__||localStorage.getItem(t)!==null){window[e]={pageview:()=>{},event:()=>{},error:()=>{},trackOutbound:()=>{},identify:()=>{},setTraits:()=>{},clearUserId:()=>{},getUserId:()=>null,flag:(o,c)=>c,flagPayload:(o,c)=>c,flags:()=>({}),flagPayloads:()=>({}),onReady:()=>{},startSessionReplay:()=>{},stopSessionReplay:()=>{},isSessionReplayActive:()=>!1};return}let n=[],i=o=>(...c)=>{n.push([o,c])};window[e]={pageview:i("pageview"),event:i("event"),error:i("error"),trackOutbound:i("trackOutbound"),identify:i("identify"),setTraits:i("setTraits"),clearUserId:i("clearUserId"),getUserId:()=>null,flag:(o,c)=>c,flagPayload:(o,c)=>c,flags:()=>({}),flagPayloads:()=>({}),onReady:i("onReady"),startSessionReplay:i("startSessionReplay"),stopSessionReplay:i("stopSessionReplay"),isSessionReplayActive:()=>!1};let s=await de(r);if(!s)return;let a=new x(s);s.enableWebVitals&&new N(c=>{a.trackWebVitals(c)}).initialize();let l=null,d=null,u=null;s.trackButtonClicks&&(l=new D(a,s),l.initialize()),s.trackCopy&&(d=new U(a),d.initialize()),s.trackFormInteractions&&(u=new H(a,s),u.initialize()),s.trackErrors&&(window.addEventListener("error",o=>{a.trackError(o.error||new Error(o.message),{filename:o.filename,lineno:o.lineno,colno:o.colno})}),window.addEventListener("unhandledrejection",o=>{let c=o.reason instanceof Error?o.reason:new Error(String(o.reason));a.trackError(c,{type:"unhandledrejection"})}));let h=()=>a.trackPageview(),g=s.debounceDuration>0?ce(h,s.debounceDuration):h;function C(){if(document.addEventListener("click",function(o){let c=o.target;for(;c&&c!==document.documentElement;){if(c.hasAttribute("data-rybbit-event")){let p=c.getAttribute("data-rybbit-event");if(p){let w={};for(let S of c.attributes)if(S.name.startsWith("data-rybbit-prop-")){let T=S.name.replace("data-rybbit-prop-","");w[T]=S.value}a.trackEvent(p,w)}break}c=c.parentElement}if(s.trackOutbound){let p=o.target.closest("a");p?.href&&le(p.href)&&a.trackOutbound(p.href,p.innerText||p.textContent||"",p.target||"_self")}}),s.autoTrackSpa){let o=history.pushState,c=history.replaceState;history.pushState=function(...p){o.apply(this,p),g(),a.onPageChange()},history.replaceState=function(...p){c.apply(this,p),g(),a.onPageChange()},window.addEventListener("popstate",()=>{g(),a.onPageChange()}),window.addEventListener("hashchange",()=>{g(),a.onPageChange()})}}window[s.namespace]={pageview:()=>a.trackPageview(),event:(o,c={})=>a.trackEvent(o,c),error:(o,c={})=>a.trackError(o,c),trackOutbound:(o,c="",p="_self")=>a.trackOutbound(o,c,p),identify:(o,c)=>a.identify(o,c),setTraits:o=>a.setTraits(o),clearUserId:()=>a.clearUserId(),getUserId:()=>a.getUserId(),flag:(o,c)=>a.getFeatureFlag(o,c),flagPayload:(o,c)=>a.getFeatureFlagPayload(o,c),flags:()=>a.getFeatureFlags(),flagPayloads:()=>a.getFeatureFlagPayloads(),onReady:o=>o(window[s.namespace]),startSessionReplay:()=>a.startSessionReplay(),stopSessionReplay:()=>a.stopSessionReplay(),isSessionReplayActive:()=>a.isSessionReplayActive()};let m=window[s.namespace];for(let[o,c]of n)m[o](...c);C(),window.addEventListener("beforeunload",()=>{l?.cleanup(),d?.cleanup(),a.cleanup()}),s.autoTrackPageview&&a.trackPageview()})();})(); diff --git a/server/src/analytics-script/tracking.test.ts b/server/src/analytics-script/tracking.test.ts index 443b33b0f..d49baaeb7 100644 --- a/server/src/analytics-script/tracking.test.ts +++ b/server/src/analytics-script/tracking.test.ts @@ -324,6 +324,35 @@ describe("Tracker", () => { consoleSpy.mockRestore(); }); + it("rejects replay delivery when the server returns an HTTP error", async () => { + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.mocked(global.fetch).mockResolvedValue({ + ok: false, + status: 503, + statusText: "Service Unavailable", + } as Response); + + const sendSessionReplayBatch = ( + tracker as unknown as { + sendSessionReplayBatch: (batch: { + batchId: string; + userId: string; + events: Array<{ type: number; data: object; timestamp: number }>; + }) => Promise; + } + ).sendSessionReplayBatch.bind(tracker); + + await expect( + sendSessionReplayBatch({ + batchId: "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", + userId: "employee-alice", + events: [{ type: 2, data: {}, timestamp: 1_700_000_000_000 }], + }) + ).rejects.toThrow("503 Service Unavailable"); + + consoleSpy.mockRestore(); + }); + describe("error tracking", () => { it("should track first-party errors", () => { // Enable error tracking for this test diff --git a/server/src/analytics-script/tracking.ts b/server/src/analytics-script/tracking.ts index afc7de418..2a1e0d55b 100644 --- a/server/src/analytics-script/tracking.ts +++ b/server/src/analytics-script/tracking.ts @@ -117,7 +117,7 @@ export class Tracker { private async sendSessionReplayBatch(batch: SessionReplayBatch): Promise { try { - await fetch(`${this.config.analyticsHost}/session-replay/record/${this.config.siteId}`, { + const response = await fetch(`${this.config.analyticsHost}/session-replay/record/${this.config.siteId}`, { method: "POST", headers: { "Content-Type": "application/json", @@ -126,6 +126,9 @@ export class Tracker { mode: "cors", keepalive: false, // Disable keepalive for large session replay requests }); + if (!response.ok) { + throw new Error(`Session replay delivery failed: ${response.status} ${response.statusText}`.trim()); + } } catch (error) { console.error("Failed to send session replay batch:", error); throw error; diff --git a/server/src/api/sites/batchImportEvents.test.ts b/server/src/api/sites/batchImportEvents.test.ts index 98b309092..e4579cb95 100644 --- a/server/src/api/sites/batchImportEvents.test.ts +++ b/server/src/api/sites/batchImportEvents.test.ts @@ -1,13 +1,19 @@ import type { FastifyReply, FastifyRequest } from "fastify"; import { beforeEach, describe, expect, it, vi } from "vitest"; -const mocks = vi.hoisted(() => ({ - completeImport: vi.fn(), - getImportById: vi.fn(), - insert: vi.fn(), - updateImportProgress: vi.fn(), - withOrganizationImportLock: vi.fn(async (_organizationId: string, work: () => Promise) => work()), -})); +const mocks = vi.hoisted(() => { + const transactionExecutor = {}; + return { + completeImport: vi.fn(), + getImportById: vi.fn(), + insert: vi.fn(), + transactionExecutor, + updateImportProgress: vi.fn(), + withOrganizationImportLock: vi.fn(async (_organizationId: string, work: (tx: unknown) => Promise) => + work(transactionExecutor) + ), + }; +}); const dbChain = { from: vi.fn(), @@ -101,4 +107,27 @@ describe("batchImportEvents failure semantics", () => { expect(reply.status).toHaveBeenCalledWith(500); expect(mocks.completeImport).not.toHaveBeenCalled(); }); + + it("keeps import state writes on the transaction holding the organization lock", async () => { + const request = { + params: { siteId: 42, importId: "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4" }, + body: { events: [event], isLastBatch: true }, + } as unknown as FastifyRequest; + const reply = replyStub(); + + await batchImportEvents(request as any, reply); + + expect(mocks.getImportById).toHaveBeenCalledWith("76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", mocks.transactionExecutor); + expect(mocks.updateImportProgress).toHaveBeenCalledWith( + "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", + 1, + 0, + 0, + mocks.transactionExecutor + ); + expect(mocks.completeImport).toHaveBeenCalledWith( + "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", + mocks.transactionExecutor + ); + }); }); diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts index 618b82486..ffdf36210 100644 --- a/server/src/api/sites/batchImportEvents.ts +++ b/server/src/api/sites/batchImportEvents.ts @@ -14,7 +14,7 @@ import { ImportQuotaTracker } from "../../services/import/importQuotaTracker.js" import { db } from "../../db/postgres/postgres.js"; import { organization, sites } from "../../db/postgres/schema.js"; import { eq } from "drizzle-orm"; -import { getBestSubscription } from "../../lib/subscriptionUtils.js"; +import { getBestSubscription, type SubscriptionInfo } from "../../lib/subscriptionUtils.js"; import { IS_CLOUD } from "../../lib/const.js"; const batchImportRequestSchema = z @@ -77,8 +77,9 @@ export async function batchImportEvents(request: FastifyRequest { - const currentImport = await getImportById(importId); + await withOrganizationImportLock(organizationId, async tx => { + const currentImport = await getImportById(importId, tx); if (!currentImport || currentImport.completedAt !== null) { throw new Error("IMPORT_ALREADY_COMPLETED"); } - // Recompute from ClickHouse while holding the cross-worker lock. The - // tracker is discarded on failure, so reservations cannot leak. - const quotaTracker = await ImportQuotaTracker.create(organizationId); const timestamps = transformedEvents.map(event => event.timestamp); + let quotaTracker: ImportQuotaTracker; + + if (IS_CLOUD) { + if (!subscription) throw new Error("Subscription could not be resolved"); + const organizationSites = await tx + .select({ siteId: sites.siteId }) + .from(sites) + .where(eq(sites.organizationId, organizationId)); + quotaTracker = await ImportQuotaTracker.createFromSnapshot({ + siteIds: organizationSites.map(site => site.siteId), + subscription, + timestamps, + }); + } else { + quotaTracker = await ImportQuotaTracker.create(organizationId, { timestamps }); + } + const allowedIndices = quotaTracker.canImportBatch(timestamps); const eventsWithinQuota = allowedIndices.map(index => transformedEvents[index]); const skippedDueToQuota = transformedEvents.length - eventsWithinQuota.length; @@ -122,10 +137,10 @@ export async function batchImportEvents(request: FastifyRequest { // Create events table await execClickhouseInitStep( @@ -140,6 +152,7 @@ export const initializeClickhouse = async () => { ENGINE = MergeTree() PARTITION BY toYYYYMM(timestamp) ORDER BY (site_id, timestamp) + SETTINGS non_replicated_deduplication_window = ${INSERT_DEDUPLICATION_WINDOW} ` ); @@ -187,6 +200,7 @@ export const initializeClickhouse = async () => { PARTITION BY toYYYYMM(timestamp) ORDER BY (site_id, timestamp) TTL timestamp + INTERVAL 3 MONTH + SETTINGS non_replicated_deduplication_window = ${INSERT_DEDUPLICATION_WINDOW} ` ); @@ -213,6 +227,7 @@ export const initializeClickhouse = async () => { PARTITION BY toYYYYMM(timestamp) ORDER BY (site_id, session_id, sequence_number) TTL toDateTime(timestamp) + INTERVAL 30 DAY + SETTINGS non_replicated_deduplication_window = ${INSERT_DEDUPLICATION_WINDOW} `, }); @@ -226,6 +241,14 @@ export const initializeClickhouse = async () => { `, }); + // Retried inserts use stable tokens. Heal existing installations where these + // tables predate the deduplication settings in their CREATE statements. + await Promise.all([ + ensureInsertDeduplication("events"), + ensureInsertDeduplication("bot_events"), + ensureInsertDeduplication("session_replay_events"), + ]); + await clickhouse.exec({ query: ` CREATE TABLE IF NOT EXISTS session_replay_metadata ( diff --git a/server/src/services/import/importQuotaTracker.test.ts b/server/src/services/import/importQuotaTracker.test.ts new file mode 100644 index 000000000..692886c7d --- /dev/null +++ b/server/src/services/import/importQuotaTracker.test.ts @@ -0,0 +1,94 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + query: vi.fn(), +})); + +vi.mock("../../db/clickhouse/clickhouse.js", () => ({ + clickhouse: { query: mocks.query }, +})); +vi.mock("../../lib/const.js", () => ({ IS_CLOUD: true })); + +import { ImportQuotaTracker } from "./importQuotaTracker.js"; + +describe("ImportQuotaTracker batch usage", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-14T12:00:00Z")); + vi.clearAllMocks(); + mocks.query.mockResolvedValue({ json: async () => [] }); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("queries usage only for months represented in the current batch", async () => { + await ImportQuotaTracker.createFromSnapshot({ + siteIds: [42, 43], + subscription: { + source: "stripe", + subscriptionId: "sub_123", + priceId: "price_123", + planName: "pro-100k-monthly", + eventLimit: 100_000, + replayLimit: 1_000, + periodStart: "2026-07-01", + currentPeriodEnd: new Date("2026-08-01T00:00:00Z"), + status: "active", + interval: "month", + cancelAtPeriodEnd: false, + createdAt: new Date("2026-01-01T00:00:00Z"), + }, + timestamps: ["2026-06-01 12:00:00", "2026-07-01 12:00:00", "2026-07-02 12:00:00"], + }); + + expect(mocks.query).toHaveBeenCalledTimes(1); + expect(mocks.query).toHaveBeenCalledWith( + expect.objectContaining({ + query: expect.stringContaining("toYYYYMM(timestamp) IN {months:Array(UInt32)}"), + query_params: expect.objectContaining({ months: [202606, 202607] }), + }) + ); + }); + + it("does not query ClickHouse for an empty final batch", async () => { + await ImportQuotaTracker.createFromSnapshot({ + siteIds: [42], + subscription: { + source: "free", + eventLimit: 10_000, + periodStart: "2026-07-01", + planName: "free", + status: "free", + }, + timestamps: [], + }); + + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it("applies numeric ClickHouse month keys to the matching import month", async () => { + mocks.query.mockResolvedValue({ json: async () => [{ month: 202607, count: "99999" }] }); + const tracker = await ImportQuotaTracker.createFromSnapshot({ + siteIds: [42], + subscription: { + source: "stripe", + subscriptionId: "sub_123", + priceId: "price_123", + planName: "pro-100k-monthly", + eventLimit: 100_000, + replayLimit: 1_000, + periodStart: "2026-07-01", + currentPeriodEnd: new Date("2026-08-01T00:00:00Z"), + status: "active", + interval: "month", + cancelAtPeriodEnd: false, + createdAt: new Date("2026-01-01T00:00:00Z"), + }, + timestamps: ["2026-07-01 12:00:00", "2026-07-02 12:00:00"], + }); + + expect(tracker.canImportBatch(["2026-07-01 12:00:00", "2026-07-02 12:00:00"])).toEqual([0]); + }); +}); diff --git a/server/src/services/import/importQuotaTracker.ts b/server/src/services/import/importQuotaTracker.ts index ea715c9ba..0ff514ab6 100644 --- a/server/src/services/import/importQuotaTracker.ts +++ b/server/src/services/import/importQuotaTracker.ts @@ -40,7 +40,7 @@ export class ImportQuotaTracker { this.oldestAllowedMonth = oldestAllowedMonth; } - static async create(organizationId: string): Promise { + static async create(organizationId: string, options: { timestamps?: string[] } = {}): Promise { if (!IS_CLOUD) { return new ImportQuotaTracker(new Map(), Infinity, "190001"); } @@ -57,32 +57,68 @@ export class ImportQuotaTracker { const subscription = await getBestSubscription(organizationId, org.stripeCustomerId); - const monthlyLimit = subscription.eventLimit; - const historicalWindowMonths = getHistoricalWindowMonths(subscription); - - const oldestAllowedDate = DateTime.utc().minus({ months: historicalWindowMonths }).startOf("month"); - const oldestAllowedMonth = oldestAllowedDate.toFormat("yyyyMM"); - const siteRecords = await db .select({ siteId: sites.siteId }) .from(sites) .where(eq(sites.organizationId, organizationId)); - const siteIds = siteRecords.map(s => s.siteId); + return ImportQuotaTracker.createFromSnapshot({ + siteIds: siteRecords.map(site => site.siteId), + subscription, + timestamps: options.timestamps, + }); + } - if (siteIds.length === 0) { - return new ImportQuotaTracker(new Map(), monthlyLimit, oldestAllowedMonth); + static async createFromSnapshot({ + siteIds, + subscription, + timestamps, + }: { + siteIds: number[]; + subscription: SubscriptionInfo; + timestamps?: string[]; + }): Promise { + if (!IS_CLOUD) { + return new ImportQuotaTracker(new Map(), Infinity, "190001"); } - const monthlyUsage = await ImportQuotaTracker.queryMonthlyUsage(siteIds, oldestAllowedDate.toFormat("yyyy-MM-dd")); + const monthlyLimit = subscription.eventLimit; + const historicalWindowMonths = getHistoricalWindowMonths(subscription); + const oldestAllowedDate = DateTime.utc().minus({ months: historicalWindowMonths }).startOf("month"); + const oldestAllowedMonth = oldestAllowedDate.toFormat("yyyyMM"); + const now = DateTime.utc(); + + const relevantMonths = + timestamps === undefined + ? undefined + : [ + ...new Set( + timestamps.flatMap(timestamp => { + const date = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" }); + if (!date.isValid || date > now) return []; + const month = date.toFormat("yyyyMM"); + return month >= oldestAllowedMonth ? [Number(month)] : []; + }) + ), + ].sort((left, right) => left - right); + + const monthlyUsage = await ImportQuotaTracker.queryMonthlyUsage( + siteIds, + oldestAllowedDate.toFormat("yyyy-MM-dd"), + relevantMonths + ); return new ImportQuotaTracker(monthlyUsage, monthlyLimit, oldestAllowedMonth); } - private static async queryMonthlyUsage(siteIds: number[], startDate: string): Promise> { + private static async queryMonthlyUsage( + siteIds: number[], + startDate: string, + months?: number[] + ): Promise> { const monthlyUsage = new Map(); - if (siteIds.length === 0) { + if (siteIds.length === 0 || (months !== undefined && months.length === 0)) { return monthlyUsage; } @@ -109,6 +145,7 @@ export class ImportQuotaTracker { WHERE site_id IN {grandfatheredSites:Array(Int32)} AND type = 'pageview' AND timestamp >= toDate({startDate:String}) + ${months !== undefined ? "AND toYYYYMM(timestamp) IN {months:Array(UInt32)}" : ""} ` : "" } @@ -123,6 +160,7 @@ export class ImportQuotaTracker { WHERE site_id IN {newSites:Array(Int32)} AND type IN ('pageview', 'custom_event', 'performance') AND timestamp >= toDate({startDate:String}) + ${months !== undefined ? "AND toYYYYMM(timestamp) IN {months:Array(UInt32)}" : ""} ` : "" } @@ -134,13 +172,14 @@ export class ImportQuotaTracker { ...(grandfatheredSites.length > 0 && { grandfatheredSites }), ...(newSites.length > 0 && { newSites }), startDate: startDate, + ...(months !== undefined && { months }), }, format: "JSONEachRow", }); - const rows = await processResults<{ month: string; count: number }>(result); + const rows = await processResults<{ month: number; count: number }>(result); for (const row of rows) { - monthlyUsage.set(row.month, row.count); + monthlyUsage.set(String(row.month), row.count); } return monthlyUsage; diff --git a/server/src/services/import/importStatusManager.test.ts b/server/src/services/import/importStatusManager.test.ts index ddc01854f..17e1c0011 100644 --- a/server/src/services/import/importStatusManager.test.ts +++ b/server/src/services/import/importStatusManager.test.ts @@ -6,6 +6,7 @@ const mocks = vi.hoisted(() => { const insertReturning = vi.fn(); const transaction = vi.fn(); const updateWhere = vi.fn(); + const findActiveWithLimit = vi.fn(); const lockChain = { from: vi.fn(), where: vi.fn(), for: forUpdate }; lockChain.from.mockReturnValue(lockChain); @@ -17,6 +18,16 @@ const mocks = vi.hoisted(() => { const insertChain = { values: vi.fn(), returning: insertReturning }; insertChain.values.mockReturnValue(insertChain); + const activeImportChain = { from: vi.fn(), where: vi.fn(), limit: findActiveWithLimit }; + activeImportChain.from.mockReturnValue(activeImportChain); + activeImportChain.where.mockReturnValue(activeImportChain); + + const tx = { + insert: vi.fn(() => insertChain), + select: vi.fn((selection?: unknown) => (selection ? lockChain : activeImportChain)), + update: vi.fn(() => updateChain), + }; + const db = { insert: vi.fn(() => insertChain), query: { importStatus: { findFirst: findActive } }, @@ -27,11 +38,13 @@ const mocks = vi.hoisted(() => { return { db, findActive, + findActiveWithLimit, forUpdate, insertChain, insertReturning, lockChain, transaction, + tx, updateChain, updateWhere, }; @@ -50,11 +63,12 @@ describe("cluster-safe import creation", () => { mocks.insertChain.values.mockReturnValue(mocks.insertChain); mocks.forUpdate.mockResolvedValue([{ id: "org-1" }]); mocks.updateWhere.mockResolvedValue(undefined); - mocks.transaction.mockImplementation(async callback => callback({ select: () => mocks.lockChain })); + mocks.transaction.mockImplementation(async callback => callback(mocks.tx)); }); it("locks the organization row and rejects a second active import", async () => { mocks.findActive.mockResolvedValue({ importId: "existing" }); + mocks.findActiveWithLimit.mockResolvedValue([{ importId: "existing" }]); const result = await createImport({ siteId: 42, @@ -65,11 +79,12 @@ describe("cluster-safe import creation", () => { expect(result).toBeNull(); expect(mocks.forUpdate).toHaveBeenCalledWith("update"); - expect(mocks.db.insert).not.toHaveBeenCalled(); + expect(mocks.tx.insert).not.toHaveBeenCalled(); }); it("creates the import while holding the same organization lock", async () => { mocks.findActive.mockResolvedValue(undefined); + mocks.findActiveWithLimit.mockResolvedValue([]); mocks.insertReturning.mockResolvedValue([{ importId: "new-import" }]); const result = await createImport({ @@ -81,6 +96,7 @@ describe("cluster-safe import creation", () => { expect(result).toEqual({ importId: "new-import" }); expect(mocks.forUpdate).toHaveBeenCalledWith("update"); - expect(mocks.db.insert).toHaveBeenCalledTimes(1); + expect(mocks.tx.insert).toHaveBeenCalledTimes(1); + expect(mocks.db.insert).not.toHaveBeenCalled(); }); }); diff --git a/server/src/services/import/importStatusManager.ts b/server/src/services/import/importStatusManager.ts index d0d8f24b0..d3c44769b 100644 --- a/server/src/services/import/importStatusManager.ts +++ b/server/src/services/import/importStatusManager.ts @@ -4,11 +4,15 @@ import { importPlatforms, importStatus, organization } from "../../db/postgres/s import { DateTime } from "luxon"; export type SelectImportStatus = typeof importStatus.$inferSelect; +export type ImportStatusExecutor = Pick; const IMPORT_TIMEOUT_HOURS = 2; /** Serialize import state changes across every worker using the organization row. */ -export async function withOrganizationImportLock(organizationId: string, work: () => Promise): Promise { +export async function withOrganizationImportLock( + organizationId: string, + work: (tx: ImportStatusExecutor) => Promise +): Promise { return db.transaction(async tx => { const [lockedOrganization] = await tx .select({ id: organization.id }) @@ -20,7 +24,7 @@ export async function withOrganizationImportLock(organizationId: string, work throw new Error(`Organization ${organizationId} not found`); } - return work(); + return work(tx); }); } @@ -30,8 +34,8 @@ export async function createImport(data: { platform: (typeof importPlatforms)[number]; enforceSingleActive?: boolean; }): Promise<{ importId: string } | null> { - const insertImport = async () => { - const [result] = await db + const insertImport = async (executor: ImportStatusExecutor = db) => { + const [result] = await executor .insert(importStatus) .values({ siteId: data.siteId, @@ -45,12 +49,12 @@ export async function createImport(data: { if (!data.enforceSingleActive) return insertImport(); - return withOrganizationImportLock(data.organizationId, async () => { + return withOrganizationImportLock(data.organizationId, async tx => { const now = DateTime.utc(); const staleCutoff = now.minus({ hours: IMPORT_TIMEOUT_HOURS }).toSQL({ includeOffset: false }); if (!staleCutoff) throw new Error("Failed to calculate import timeout"); - await db + await tx .update(importStatus) .set({ completedAt: now.toISO() }) .where( @@ -61,12 +65,14 @@ export async function createImport(data: { ) ); - const activeImport = await db.query.importStatus.findFirst({ - where: and(eq(importStatus.organizationId, data.organizationId), isNull(importStatus.completedAt)), - }); + const [activeImport] = await tx + .select() + .from(importStatus) + .where(and(eq(importStatus.organizationId, data.organizationId), isNull(importStatus.completedAt))) + .limit(1); if (activeImport) return null; - return insertImport(); + return insertImport(tx); }); } @@ -74,9 +80,10 @@ export async function updateImportProgress( importId: string, importedEvents: number, skippedEvents: number, - invalidEvents: number + invalidEvents: number, + executor: ImportStatusExecutor = db ): Promise { - await db + await executor .update(importStatus) .set({ importedEvents: sql`${importStatus.importedEvents} + ${importedEvents}`, @@ -86,8 +93,8 @@ export async function updateImportProgress( .where(eq(importStatus.importId, importId)); } -export async function completeImport(importId: string): Promise { - await db +export async function completeImport(importId: string, executor: ImportStatusExecutor = db): Promise { + await executor .update(importStatus) .set({ completedAt: DateTime.utc().toISO(), @@ -107,8 +114,10 @@ export async function deleteImport(importId: string): Promise { await db.delete(importStatus).where(eq(importStatus.importId, importId)); } -export async function getImportById(importId: string): Promise { - return await db.query.importStatus.findFirst({ - where: eq(importStatus.importId, importId), - }); +export async function getImportById( + importId: string, + executor: ImportStatusExecutor = db +): Promise { + const [result] = await executor.select().from(importStatus).where(eq(importStatus.importId, importId)).limit(1); + return result; } diff --git a/server/src/services/replay/sessionReplayIngestService.test.ts b/server/src/services/replay/sessionReplayIngestService.test.ts index 028a0e87f..e5bf949b5 100644 --- a/server/src/services/replay/sessionReplayIngestService.test.ts +++ b/server/src/services/replay/sessionReplayIngestService.test.ts @@ -123,6 +123,9 @@ describe("SessionReplayIngestService identity", () => { expect(mocks.insert).toHaveBeenCalledWith( expect.objectContaining({ + clickhouse_settings: { + insert_deduplication_token: expect.stringMatching(/^[a-f0-9]{64}$/), + }, values: [ expect.objectContaining({ batch_id: "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", @@ -135,7 +138,28 @@ describe("SessionReplayIngestService identity", () => { }); it("does not insert a replay batch whose indices are already stored", async () => { - mocks.query.mockResolvedValue({ json: async () => [{ batch_index: 0 }] }); + mocks.query.mockResolvedValue({ json: async () => [{ session_id: "original-session", batch_index: 0 }] }); + const service = new SessionReplayIngestService(); + + await service.recordEvents( + 42, + { + batchId: "76c8fb17-e7b5-41f7-b4f9-a21a4efca1d4", + userId: "employee-alice", + events: [{ type: 3, data: { ordered: true }, timestamp: 1_700_000_000_000, sequence: 73 }], + }, + requestMeta + ); + + expect(mocks.insert).not.toHaveBeenCalled(); + }); + + it("deduplicates a retry even when session resolution would return a different session", async () => { + mocks.updateSession.mockResolvedValue({ sessionId: "new-session" }); + mocks.query.mockImplementation(async options => ({ + json: async () => + "sessionId" in options.query_params ? [] : [{ session_id: "original-session", batch_index: 0 }], + })); const service = new SessionReplayIngestService(); await service.recordEvents( diff --git a/server/src/services/replay/sessionReplayIngestService.ts b/server/src/services/replay/sessionReplayIngestService.ts index 32ac73a9d..5cec3dca8 100644 --- a/server/src/services/replay/sessionReplayIngestService.ts +++ b/server/src/services/replay/sessionReplayIngestService.ts @@ -50,25 +50,36 @@ export class SessionReplayIngestService { // scopes session assignment so stored user identity semantics stay unchanged. const userId = deviceFingerprint; - // Get or create a session ID from the sessions service - const { sessionId } = await sessionsService.updateSession({ - userId, - identifiedUserId, - siteId, - }); + type ExistingBatchRow = { + session_id: string; + batch_index: number | null; + }; + const findExistingBatch = async () => { + const result = await clickhouse.query({ + query: ` + SELECT session_id, batch_index + FROM session_replay_events + WHERE site_id = {siteId:UInt16} + AND batch_id = {batchId:String} + `, + query_params: { siteId, batchId }, + format: "JSONEachRow", + }); + return processResults(result); + }; + + let existingBatchRows = await findExistingBatch(); + let sessionId = existingBatchRows[0]?.session_id; + + if (!sessionId) { + const session = await sessionsService.updateSession({ + userId, + identifiedUserId, + siteId, + }); + sessionId = session.sessionId; + } - const existingBatchResult = await clickhouse.query({ - query: ` - SELECT batch_index - FROM session_replay_events - WHERE site_id = {siteId:UInt16} - AND session_id = {sessionId:String} - AND batch_id = {batchId:String} - `, - query_params: { siteId, sessionId, batchId }, - format: "JSONEachRow", - }); - const existingBatchRows = await processResults<{ batch_index: number | null }>(existingBatchResult); const storedBatchIndices = new Set( existingBatchRows.flatMap(row => (row.batch_index === null ? [] : [row.batch_index])) ); @@ -140,11 +151,24 @@ export class SessionReplayIngestService { // Batch insert events if (eventsToInsert.length > 0) { + const insertDeduplicationToken = crypto + .createHash("sha256") + .update(JSON.stringify([siteId, batchId, missingEvents.map(({ index }) => index)])) + .digest("hex"); + await clickhouse.insert({ table: "session_replay_events", values: eventsToInsert, format: "JSONEachRow", + clickhouse_settings: { + insert_deduplication_token: insertDeduplicationToken, + }, }); + + // A concurrent retry may have won the deduplicated insert with a different + // newly-resolved session. Always use the session that actually owns the rows. + existingBatchRows = await findExistingBatch(); + sessionId = existingBatchRows[0]?.session_id ?? sessionId; } // Update or insert metadata diff --git a/server/src/services/tracker/botBlocking/botEventQueue.ts b/server/src/services/tracker/botBlocking/botEventQueue.ts index f8275db3c..59c262a3f 100644 --- a/server/src/services/tracker/botBlocking/botEventQueue.ts +++ b/server/src/services/tracker/botBlocking/botEventQueue.ts @@ -2,7 +2,7 @@ import { DateTime } from "luxon"; import { clickhouse } from "../../../db/clickhouse/clickhouse.js"; import { getLocation } from "../../../db/geolocation/geolocation.js"; import { getDeviceType } from "../../../utils.js"; -import { ReliableBatchQueue } from "../reliableBatchQueue.js"; +import { ReliableBatchContext, ReliableBatchQueue } from "../reliableBatchQueue.js"; import { clearSelfReferrer, type TotalTrackingPayload } from "../utils.js"; import type { BotEventProperties } from "./index.js"; @@ -14,7 +14,7 @@ type BotEventPayload = TotalTrackingPayload & const BOT_EVENT_BATCH_SIZE = 5000; const BOT_EVENT_FLUSH_INTERVAL_MS = 250; -async function processBotEventBatch(batch: BotEventPayload[]): Promise { +async function processBotEventBatch(batch: BotEventPayload[], context: ReliableBatchContext): Promise { const ips = [...new Set(batch.map(event => event.ipAddress))]; const geoData = await getLocation(ips); @@ -63,6 +63,9 @@ async function processBotEventBatch(batch: BotEventPayload[]): Promise { table: "bot_events", values: processedBotEvents, format: "JSONEachRow", + clickhouse_settings: { + insert_deduplication_token: context.batchId, + }, }); } diff --git a/server/src/services/tracker/pageviewQueue.test.ts b/server/src/services/tracker/pageviewQueue.test.ts index d60394099..421e67fcb 100644 --- a/server/src/services/tracker/pageviewQueue.test.ts +++ b/server/src/services/tracker/pageviewQueue.test.ts @@ -28,8 +28,12 @@ describe("PageviewQueue delivery", () => { mocks.getLocation.mockResolvedValue({}); mocks.insert.mockResolvedValue(undefined); (pageviewQueue as any).queue = []; + (pageviewQueue as any).retryBatch = null; + (pageviewQueue as any).activeBatch = null; + (pageviewQueue as any).activeProcess = null; (pageviewQueue as any).processing = false; (pageviewQueue as any).closing = false; + (pageviewQueue as any).nextAttemptAt = 0; }); it("does not acknowledge an event until ClickHouse stores its batch", async () => { @@ -44,6 +48,13 @@ describe("PageviewQueue delivery", () => { await (pageviewQueue as any).processQueue(); await delivery; expect(acknowledged).toBe(true); + expect(mocks.insert).toHaveBeenCalledWith( + expect.objectContaining({ + clickhouse_settings: { + insert_deduplication_token: expect.any(String), + }, + }) + ); }); it("requeues a transiently failed batch instead of dropping it", async () => { @@ -51,7 +62,7 @@ describe("PageviewQueue delivery", () => { const delivery = pageviewQueue.add(payload); await (pageviewQueue as any).processQueue(); - expect((pageviewQueue as any).queue).toHaveLength(1); + expect((pageviewQueue as any).retryBatch.entries).toHaveLength(1); await (pageviewQueue as any).processQueue({ ignoreBackoff: true }); await expect(delivery).resolves.toBeUndefined(); @@ -64,7 +75,7 @@ describe("PageviewQueue delivery", () => { await expect((pageviewQueue as any).processQueue()).resolves.toBeUndefined(); expect((pageviewQueue as any).processing).toBe(false); - expect((pageviewQueue as any).queue).toHaveLength(1); + expect((pageviewQueue as any).retryBatch.entries).toHaveLength(1); await (pageviewQueue as any).processQueue({ ignoreBackoff: true }); await expect(delivery).resolves.toBeUndefined(); diff --git a/server/src/services/tracker/pageviewQueue.ts b/server/src/services/tracker/pageviewQueue.ts index 49805cc80..249c24759 100644 --- a/server/src/services/tracker/pageviewQueue.ts +++ b/server/src/services/tracker/pageviewQueue.ts @@ -4,7 +4,7 @@ import { getLocation } from "../../db/geolocation/geolocation.js"; import { getDeviceType } from "../../utils.js"; import { getChannel } from "./getChannel.js"; import { clearSelfReferrer, getAllUrlParams, TotalTrackingPayload } from "./utils.js"; -import { ReliableBatchQueue } from "./reliableBatchQueue.js"; +import { ReliableBatchContext, ReliableBatchQueue } from "./reliableBatchQueue.js"; type TotalPayload = TotalTrackingPayload & { sessionId: string; @@ -21,7 +21,7 @@ const getParsedProperties = (properties: string | undefined) => { } }; -async function processPageviewBatch(batch: TotalPayload[]): Promise { +async function processPageviewBatch(batch: TotalPayload[], context: ReliableBatchContext): Promise { const ips = [...new Set(batch.map(pv => pv.ipAddress))]; const geoData = await getLocation(ips); @@ -99,6 +99,9 @@ async function processPageviewBatch(batch: TotalPayload[]): Promise { table: "events", values: processedPageviews, format: "JSONEachRow", + clickhouse_settings: { + insert_deduplication_token: context.batchId, + }, }); } diff --git a/server/src/services/tracker/reliableBatchQueue.test.ts b/server/src/services/tracker/reliableBatchQueue.test.ts index 81e03a17a..1fff2b7ab 100644 --- a/server/src/services/tracker/reliableBatchQueue.test.ts +++ b/server/src/services/tracker/reliableBatchQueue.test.ts @@ -35,7 +35,7 @@ describe("ReliableBatchQueue", () => { await queue.close(); await expect(delivery).resolves.toBeUndefined(); - expect(processBatch).toHaveBeenCalledWith([1]); + expect(processBatch).toHaveBeenCalledWith([1], { batchId: expect.any(String) }); }); it("applies bounded backpressure", async () => { @@ -47,4 +47,32 @@ describe("ReliableBatchQueue", () => { await queue.close(); await expect(firstDelivery).resolves.toBeUndefined(); }); + + it("retries the exact same batch with a stable insertion id", async () => { + const calls: Array<{ batch: number[]; batchId: string | undefined }> = []; + const processBatch = vi + .fn() + .mockImplementationOnce(async (batch: number[], context?: { batchId: string }) => { + calls.push({ batch, batchId: context?.batchId }); + throw new Error("ambiguous insert result"); + }) + .mockImplementation(async (batch: number[], context?: { batchId: string }) => { + calls.push({ batch, batchId: context?.batchId }); + }); + const queue = createQueue(processBatch); + const firstDelivery = queue.add(1); + + await queue.processQueue({ ignoreBackoff: true }); + const secondDelivery = queue.add(2); + await queue.processQueue({ ignoreBackoff: true }); + + await expect(firstDelivery).resolves.toBeUndefined(); + expect(calls.slice(0, 2).map(call => call.batch)).toEqual([[1], [1]]); + expect(calls[0].batchId).toBeTruthy(); + expect(calls[1].batchId).toBe(calls[0].batchId); + + await queue.processQueue({ ignoreBackoff: true }); + await expect(secondDelivery).resolves.toBeUndefined(); + await queue.close(); + }); }); diff --git a/server/src/services/tracker/reliableBatchQueue.ts b/server/src/services/tracker/reliableBatchQueue.ts index 8b4935d9f..4935bfab6 100644 --- a/server/src/services/tracker/reliableBatchQueue.ts +++ b/server/src/services/tracker/reliableBatchQueue.ts @@ -1,12 +1,18 @@ +import { randomUUID } from "node:crypto"; import { createServiceLogger } from "../../lib/logger/logger.js"; type QueueEntry = { value: T; - attempts: number; resolve: () => void; reject: (error: Error) => void; }; +type PendingBatch = { + batchId: string; + entries: QueueEntry[]; + attempts: number; +}; + type ProcessQueueOptions = { ignoreBackoff?: boolean; }; @@ -17,7 +23,11 @@ type ReliableBatchQueueOptions = { maxAttempts?: number; maxQueueSize?: number; name: string; - processBatch: (batch: T[]) => Promise; + processBatch: (batch: T[], context: ReliableBatchContext) => Promise; +}; + +export type ReliableBatchContext = { + batchId: string; }; const DEFAULT_MAX_ATTEMPTS = 3; @@ -43,6 +53,8 @@ export class QueueFullError extends Error { */ export class ReliableBatchQueue { private queue: QueueEntry[] = []; + private retryBatch: PendingBatch | null = null; + private activeBatch: PendingBatch | null = null; private processing = false; private closing = false; private nextAttemptAt = 0; @@ -63,12 +75,12 @@ export class ReliableBatchQueue { if (this.closing) { return Promise.reject(new QueueClosedError(this.options.name)); } - if (this.queue.length >= this.maxQueueSize) { + if (this.pendingSize() >= this.maxQueueSize) { return Promise.reject(new QueueFullError(this.options.name, this.maxQueueSize)); } const delivery = new Promise((resolve, reject) => { - this.queue.push({ value, attempts: 0, resolve, reject }); + this.queue.push({ value, resolve, reject }); }); if (this.queue.length >= this.options.batchSize) { @@ -83,7 +95,7 @@ export class ReliableBatchQueue { await this.activeProcess; return; } - if (this.queue.length === 0) return; + if (!this.hasPendingBatch()) return; if (!processOptions.ignoreBackoff && Date.now() < this.nextAttemptAt) return; this.activeProcess = this.runBatch(); @@ -96,44 +108,58 @@ export class ReliableBatchQueue { private async runBatch(): Promise { this.processing = true; - const entries = this.queue.splice(0, this.options.batchSize); + const batch = + this.retryBatch ?? + ({ + batchId: randomUUID(), + entries: this.queue.splice(0, this.options.batchSize), + attempts: 0, + } satisfies PendingBatch); + this.retryBatch = null; + this.activeBatch = batch; try { - await this.options.processBatch(entries.map(entry => entry.value)); + await this.options.processBatch( + batch.entries.map(entry => entry.value), + { batchId: batch.batchId } + ); this.nextAttemptAt = 0; - for (const entry of entries) entry.resolve(); + for (const entry of batch.entries) entry.resolve(); } catch (cause) { const error = cause instanceof Error ? cause : new Error(String(cause)); - const retryEntries: QueueEntry[] = []; - - for (const entry of entries) { - entry.attempts += 1; - if (entry.attempts < this.maxAttempts) { - retryEntries.push(entry); - } else { - entry.reject(error); - } - } - - if (retryEntries.length > 0) { - this.queue.unshift(...retryEntries); - const attempt = Math.max(...retryEntries.map(entry => entry.attempts)); - this.nextAttemptAt = Date.now() + Math.min(100 * 2 ** (attempt - 1), MAX_RETRY_DELAY_MS); + batch.attempts += 1; + + if (batch.attempts < this.maxAttempts) { + this.retryBatch = batch; + this.nextAttemptAt = Date.now() + Math.min(100 * 2 ** (batch.attempts - 1), MAX_RETRY_DELAY_MS); + } else { + this.nextAttemptAt = 0; + for (const entry of batch.entries) entry.reject(error); } this.logger.error( { err: error, - batchSize: entries.length, - requeued: retryEntries.length, + batchId: batch.batchId, + batchSize: batch.entries.length, + requeued: this.retryBatch ? batch.entries.length : 0, }, "Failed to persist ingestion batch" ); } finally { + this.activeBatch = null; this.processing = false; } } + private hasPendingBatch(): boolean { + return this.retryBatch !== null || this.queue.length > 0; + } + + private pendingSize(): number { + return this.queue.length + (this.retryBatch?.entries.length ?? 0) + (this.activeBatch?.entries.length ?? 0); + } + /** Stop the timer and synchronously exhaust pending batches before shutdown. */ async close(): Promise { if (this.closing) { @@ -145,7 +171,7 @@ export class ReliableBatchQueue { clearInterval(this.intervalHandle); if (this.activeProcess) await this.activeProcess; - while (this.queue.length > 0) { + while (this.hasPendingBatch()) { await this.processQueue({ ignoreBackoff: true }); } } From 19432eeb99a0fbc7186055a44fc89d48b77f741c Mon Sep 17 00:00:00 2001 From: Bill Yang Date: Tue, 14 Jul 2026 00:51:38 -0700 Subject: [PATCH 3/3] Index session replay batch lookups The replay dedup query filters by (site_id, batch_id), which the sort key (site_id, session_id, sequence_number) cannot serve, so every ingested batch scanned all of a site's replay rows. Add a bloom filter skip index on batch_id (healed into existing installations on boot) and bound the lookup to the last two days so partition pruning covers parts that predate the index. Verified against ClickHouse: the lookup reads 4/62 granules instead of 62/62 on a 500k-row site. Co-Authored-By: Claude Fable 5 --- server/src/db/clickhouse/clickhouse.ts | 15 ++++++++++++++- .../services/replay/sessionReplayIngestService.ts | 5 +++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.ts index 172691e92..694ce3261 100644 --- a/server/src/db/clickhouse/clickhouse.ts +++ b/server/src/db/clickhouse/clickhouse.ts @@ -221,7 +221,8 @@ export const initializeClickhouse = async () => { event_size_bytes UInt32, viewport_width Nullable(UInt16), viewport_height Nullable(UInt16), - is_complete UInt8 DEFAULT 0 + is_complete UInt8 DEFAULT 0, + INDEX idx_batch_id batch_id TYPE bloom_filter GRANULARITY 4 ) ENGINE = MergeTree() PARTITION BY toYYYYMM(timestamp) @@ -249,6 +250,18 @@ export const initializeClickhouse = async () => { ensureInsertDeduplication("session_replay_events"), ]); + // Replay ingestion looks batches up by (site_id, batch_id), which the sort key + // cannot serve. Parts written before the index exists are left unmaterialized; + // they age out through the 30-day TTL. + await execClickhouseInitStep( + "add session replay batch id index", + ` + ALTER TABLE session_replay_events + ADD INDEX IF NOT EXISTS idx_batch_id batch_id TYPE bloom_filter GRANULARITY 4 + `, + { lockAcquireTimeoutSeconds: 15 } + ); + await clickhouse.exec({ query: ` CREATE TABLE IF NOT EXISTS session_replay_metadata ( diff --git a/server/src/services/replay/sessionReplayIngestService.ts b/server/src/services/replay/sessionReplayIngestService.ts index 5cec3dca8..d11fa5f28 100644 --- a/server/src/services/replay/sessionReplayIngestService.ts +++ b/server/src/services/replay/sessionReplayIngestService.ts @@ -55,12 +55,17 @@ export class SessionReplayIngestService { batch_index: number | null; }; const findExistingBatch = async () => { + // Retried batches never leave the recording page, so their event timestamps + // are at most hours old — the bound prunes partitions that predate the + // batch_id skip index. A clock-skewed client can fall outside it, but the + // insert deduplication token still prevents duplicate rows. const result = await clickhouse.query({ query: ` SELECT session_id, batch_index FROM session_replay_events WHERE site_id = {siteId:UInt16} AND batch_id = {batchId:String} + AND timestamp >= now() - INTERVAL 2 DAY `, query_params: { siteId, batchId }, format: "JSONEachRow",