From 2ff432e5eaca19f098aa94ab252932c183baa5b8 Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Tue, 5 May 2026 16:29:45 -0400 Subject: [PATCH 01/10] Add Sentinel integration and SENTINEL_RARE_CLASS_AFFINITY signal type enums --- client/src/graphql/generated.ts | 2 ++ client/src/models/signal.ts | 2 ++ server/graphql/generated.ts | 2 ++ server/graphql/modules/integration.ts | 1 + server/graphql/modules/signal.ts | 1 + .../services/signalsService/types/Integration.ts | 1 + .../signalsService/types/SignalArgsByType.ts | 15 +++++++++++++++ .../services/signalsService/types/SignalType.ts | 3 +++ 8 files changed, 27 insertions(+) diff --git a/client/src/graphql/generated.ts b/client/src/graphql/generated.ts index 5937b816..f92d7928 100644 --- a/client/src/graphql/generated.ts +++ b/client/src/graphql/generated.ts @@ -1249,6 +1249,7 @@ export const GQLIntegration = { MicrosoftAzureContentModerator: 'MICROSOFT_AZURE_CONTENT_MODERATOR', Oopspam: 'OOPSPAM', OpenAi: 'OPEN_AI', + Sentinel: 'SENTINEL', SightEngine: 'SIGHT_ENGINE', TwoHat: 'TWO_HAT', Zentropi: 'ZENTROPI', @@ -3991,6 +3992,7 @@ export const GQLSignalType = { OpenAiSexualTextModel: 'OPEN_AI_SEXUAL_TEXT_MODEL', OpenAiViolenceTextModel: 'OPEN_AI_VIOLENCE_TEXT_MODEL', OpenAiWhisperTranscription: 'OPEN_AI_WHISPER_TRANSCRIPTION', + SentinelRareClassAffinity: 'SENTINEL_RARE_CLASS_AFFINITY', TextMatchingContainsRegex: 'TEXT_MATCHING_CONTAINS_REGEX', TextMatchingContainsText: 'TEXT_MATCHING_CONTAINS_TEXT', TextMatchingContainsVariant: 'TEXT_MATCHING_CONTAINS_VARIANT', diff --git a/client/src/models/signal.ts b/client/src/models/signal.ts index 3cb180a2..e0209f7f 100644 --- a/client/src/models/signal.ts +++ b/client/src/models/signal.ts @@ -60,6 +60,8 @@ export function integrationForSignalType(type: GQLSignalType) { case 'OPEN_AI_VIOLENCE_TEXT_MODEL': case 'OPEN_AI_WHISPER_TRANSCRIPTION': return GQLIntegration.OpenAi; + case 'SENTINEL_RARE_CLASS_AFFINITY': + return GQLIntegration.Sentinel; case 'ZENTROPI_LABELER': return GQLIntegration.Zentropi; case 'AGGREGATION': diff --git a/server/graphql/generated.ts b/server/graphql/generated.ts index 3bced1e1..82c37f38 100644 --- a/server/graphql/generated.ts +++ b/server/graphql/generated.ts @@ -1318,6 +1318,7 @@ export const GQLIntegration = { MicrosoftAzureContentModerator: 'MICROSOFT_AZURE_CONTENT_MODERATOR', Oopspam: 'OOPSPAM', OpenAi: 'OPEN_AI', + Sentinel: 'SENTINEL', SightEngine: 'SIGHT_ENGINE', TwoHat: 'TWO_HAT', Zentropi: 'ZENTROPI', @@ -4060,6 +4061,7 @@ export const GQLSignalType = { OpenAiSexualTextModel: 'OPEN_AI_SEXUAL_TEXT_MODEL', OpenAiViolenceTextModel: 'OPEN_AI_VIOLENCE_TEXT_MODEL', OpenAiWhisperTranscription: 'OPEN_AI_WHISPER_TRANSCRIPTION', + SentinelRareClassAffinity: 'SENTINEL_RARE_CLASS_AFFINITY', TextMatchingContainsRegex: 'TEXT_MATCHING_CONTAINS_REGEX', TextMatchingContainsText: 'TEXT_MATCHING_CONTAINS_TEXT', TextMatchingContainsVariant: 'TEXT_MATCHING_CONTAINS_VARIANT', diff --git a/server/graphql/modules/integration.ts b/server/graphql/modules/integration.ts index ed8e0153..fe370104 100644 --- a/server/graphql/modules/integration.ts +++ b/server/graphql/modules/integration.ts @@ -23,6 +23,7 @@ const typeDefs = /* GraphQL */ ` MICROSOFT_AZURE_CONTENT_MODERATOR OOPSPAM OPEN_AI + SENTINEL SIGHT_ENGINE TWO_HAT ZENTROPI diff --git a/server/graphql/modules/signal.ts b/server/graphql/modules/signal.ts index ff023091..b96f093c 100644 --- a/server/graphql/modules/signal.ts +++ b/server/graphql/modules/signal.ts @@ -96,6 +96,7 @@ const typeDefs = /* GraphQL */ ` OPEN_AI_SEXUAL_TEXT_MODEL OPEN_AI_VIOLENCE_TEXT_MODEL OPEN_AI_WHISPER_TRANSCRIPTION + SENTINEL_RARE_CLASS_AFFINITY ZENTROPI_LABELER GEO_CONTAINED_WITHIN USER_SCORE diff --git a/server/services/signalsService/types/Integration.ts b/server/services/signalsService/types/Integration.ts index 1c1e2f5b..dd8e53bb 100644 --- a/server/services/signalsService/types/Integration.ts +++ b/server/services/signalsService/types/Integration.ts @@ -7,6 +7,7 @@ import { makeEnumLike } from '@roostorg/types'; export const Integration = makeEnumLike([ 'GOOGLE_CONTENT_SAFETY_API', 'OPEN_AI', + 'SENTINEL', 'ZENTROPI', ]); diff --git a/server/services/signalsService/types/SignalArgsByType.ts b/server/services/signalsService/types/SignalArgsByType.ts index 8fdcf992..8d4d7d19 100644 --- a/server/services/signalsService/types/SignalArgsByType.ts +++ b/server/services/signalsService/types/SignalArgsByType.ts @@ -1,3 +1,5 @@ +import { type ItemIdentifier } from '@roostorg/types'; + import { type Satisfies } from '../../../utils/typescript-types.js'; import type { AggregationClause, @@ -32,6 +34,7 @@ export type SignalArgsByType = Satisfies< [SignalType.OPEN_AI_SEXUAL_MINORS_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_SEXUAL_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_VIOLENCE_TEXT_MODEL]: undefined; + [SignalType.SENTINEL_RARE_CLASS_AFFINITY]: undefined; [SignalType.ZENTROPI_LABELER]: undefined; [SignalType.CUSTOM]: undefined; }, @@ -67,6 +70,18 @@ export type RuntimeSignalArgsByType = Satisfies< [SignalType.OPEN_AI_SEXUAL_MINORS_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_SEXUAL_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_VIOLENCE_TEXT_MODEL]: undefined; + [SignalType.SENTINEL_RARE_CLASS_AFFINITY]: { + /** + * The identifier of the thread/context item that this content belongs to. + * Used by the Sentinel signal to include thread-level text in scoring. + */ + threadIdentifier?: ItemIdentifier; + /** + * The name of the content field whose text value should be scored. + * Derived from the condition's input field name. + */ + contentTextFieldName?: string; + }; [SignalType.ZENTROPI_LABELER]: undefined; [SignalType.CUSTOM]: undefined; }, diff --git a/server/services/signalsService/types/SignalType.ts b/server/services/signalsService/types/SignalType.ts index 8abd311e..db99648f 100644 --- a/server/services/signalsService/types/SignalType.ts +++ b/server/services/signalsService/types/SignalType.ts @@ -48,6 +48,7 @@ export const BuiltInThirdPartySignalType = makeEnumLike([ 'OPEN_AI_SEXUAL_MINORS_TEXT_MODEL', 'OPEN_AI_SEXUAL_TEXT_MODEL', 'OPEN_AI_VIOLENCE_TEXT_MODEL', + 'SENTINEL_RARE_CLASS_AFFINITY', 'ZENTROPI_LABELER', ]); @@ -99,6 +100,8 @@ export function integrationForSignalType(type: SignalType) { case 'OPEN_AI_VIOLENCE_TEXT_MODEL': case 'OPEN_AI_WHISPER_TRANSCRIPTION': return Integration.OPEN_AI; + case 'SENTINEL_RARE_CLASS_AFFINITY': + return Integration.SENTINEL; case 'ZENTROPI_LABELER': return Integration.ZENTROPI; case 'AGGREGATION': From eb354a1bd8ed4897b6c72025ef6ee05af5bcadb5 Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Tue, 5 May 2026 16:30:44 -0400 Subject: [PATCH 02/10] Add Sentinel HTTP service client with graceful-disabled handling and tests --- server/.env.example | 6 +- server/decs.d.ts | 1 + server/services/sentinelService/index.ts | 10 + .../sentinelService/sentinelService.test.ts | 188 ++++++++++++++++++ .../sentinelService/sentinelService.ts | 160 +++++++++++++++ 5 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 server/services/sentinelService/index.ts create mode 100644 server/services/sentinelService/sentinelService.test.ts create mode 100644 server/services/sentinelService/sentinelService.ts diff --git a/server/.env.example b/server/.env.example index 9369d209..930d135d 100644 --- a/server/.env.example +++ b/server/.env.example @@ -32,7 +32,11 @@ SNOWFLAKE_PASSWORD= SNOWFLAKE_DB_NAME=MY_DEV_DB SNOWFLAKE_ACCOUNT= -HMA_SERVICE_URL=http://localhost:5000 +HMA_SERVICE_URL=http://localhost:5001 + +# Sentinel API URL (local Sentinel service for rare class affinity signal) +# Start Sentinel locally: docker compose -f docker-compose.yaml -f docker-compose.sentinel.yaml up -d sentinel +SENTINEL_API_URL=http://localhost:8000 # Kafka authentication info. KAFKA_BROKER_HOST=localhost:29092 diff --git a/server/decs.d.ts b/server/decs.d.ts index 5ec3e96d..cd6f224a 100644 --- a/server/decs.d.ts +++ b/server/decs.d.ts @@ -282,5 +282,6 @@ namespace NodeJS { OPEN_AI_API_KEY?: string; SLACK_APP_BEARER_TOKEN?: string; GRAPHQL_OPAQUE_SCALAR_SECRET?: string; + SENTINEL_API_URL?: string; } } diff --git a/server/services/sentinelService/index.ts b/server/services/sentinelService/index.ts new file mode 100644 index 00000000..fc63acd0 --- /dev/null +++ b/server/services/sentinelService/index.ts @@ -0,0 +1,10 @@ +export { + makeSentinelService, + SentinelServiceError, + type SentinelBanksStatus, + type SentinelHealthResponse, + type SentinelLoadBanksRequest, + type SentinelScoreRequest, + type SentinelScoreResponse, + type SentinelService, +} from './sentinelService.js'; diff --git a/server/services/sentinelService/sentinelService.test.ts b/server/services/sentinelService/sentinelService.test.ts new file mode 100644 index 00000000..4a89ebaf --- /dev/null +++ b/server/services/sentinelService/sentinelService.test.ts @@ -0,0 +1,188 @@ +import { + makeSentinelService, + SentinelServiceError, + type SentinelHealthResponse, + type SentinelScoreResponse, + type SentinelBanksStatus, +} from './sentinelService.js'; + +function makeOkResponse(body: T): Response { + return { + ok: true, + status: 200, + json: () => Promise.resolve(body), + text: () => Promise.resolve(JSON.stringify(body)), + } as unknown as Response; +} + +function makeErrorResponse(status: number, text: string): Response { + return { + ok: false, + status, + text: () => Promise.resolve(text), + json: () => Promise.reject(new Error('Not JSON')), + } as unknown as Response; +} + +describe('makeSentinelService', () => { + const BASE_URL = 'http://localhost:8000'; + + beforeEach(() => { + jest.resetAllMocks(); + global.fetch = jest.fn(); + }); + + describe('healthCheck', () => { + it('calls /health endpoint and returns response', async () => { + const mockResponse: SentinelHealthResponse = { status: 'ok', banks_loaded: true }; + (global.fetch as jest.Mock).mockResolvedValueOnce(makeOkResponse(mockResponse)); + + const service = makeSentinelService(BASE_URL); + const result = await service.healthCheck(); + + expect(global.fetch).toHaveBeenCalledWith( + `${BASE_URL}/health`, + expect.objectContaining({ method: 'GET' }), + ); + expect(result).toEqual(mockResponse); + }); + + it('throws SentinelServiceError on non-ok response', async () => { + (global.fetch as jest.Mock).mockResolvedValueOnce( + makeErrorResponse(503, 'Service Unavailable'), + ); + + const service = makeSentinelService(BASE_URL); + await expect(service.healthCheck()).rejects.toBeInstanceOf(SentinelServiceError); + }); + + it('wraps fetch errors in SentinelServiceError', async () => { + (global.fetch as jest.Mock).mockRejectedValueOnce( + new Error('ECONNREFUSED'), + ); + + const service = makeSentinelService(BASE_URL); + const error = await service.healthCheck().catch((e) => e); + expect(error).toBeInstanceOf(SentinelServiceError); + expect(error.message).toContain('ECONNREFUSED'); + }); + }); + + describe('getBanksStatus', () => { + it('calls /banks/status and returns loaded status', async () => { + const mockStatus: SentinelBanksStatus = { + loaded: true, + model_name: 'all-MiniLM-L6-v2', + positive_count: 100, + negative_count: 200, + }; + (global.fetch as jest.Mock).mockResolvedValueOnce(makeOkResponse(mockStatus)); + + const service = makeSentinelService(BASE_URL); + const result = await service.getBanksStatus(); + + expect(global.fetch).toHaveBeenCalledWith( + `${BASE_URL}/banks/status`, + expect.objectContaining({ method: 'GET' }), + ); + expect(result).toEqual(mockStatus); + }); + }); + + describe('scoreTexts', () => { + it('calls /score with the provided texts and returns scores', async () => { + const mockScore: SentinelScoreResponse = { + rare_class_affinity_score: 0.82, + observation_scores: { 'hello world': 0.82 }, + num_observations: 1, + }; + (global.fetch as jest.Mock).mockResolvedValueOnce(makeOkResponse(mockScore)); + + const service = makeSentinelService(BASE_URL); + const result = await service.scoreTexts({ texts: ['hello world'] }); + + expect(global.fetch).toHaveBeenCalledWith( + `${BASE_URL}/score`, + expect.objectContaining({ + method: 'POST', + body: expect.stringContaining('hello world'), + }), + ); + expect(result.rare_class_affinity_score).toBe(0.82); + }); + + it('includes default top_k and min_score_to_consider when not specified', async () => { + const mockScore: SentinelScoreResponse = { + rare_class_affinity_score: 0.5, + observation_scores: { text: 0.5 }, + num_observations: 1, + }; + (global.fetch as jest.Mock).mockResolvedValueOnce(makeOkResponse(mockScore)); + + const service = makeSentinelService(BASE_URL); + await service.scoreTexts({ texts: ['text'] }); + + const body = JSON.parse( + (global.fetch as jest.Mock).mock.calls[0][1].body, + ); + expect(body.top_k).toBe(5); + expect(body.min_score_to_consider).toBe(0.0); + }); + }); + + describe('scoreSingleText', () => { + it('returns the score for the single text from observation_scores', async () => { + const mockScore: SentinelScoreResponse = { + rare_class_affinity_score: 0.75, + observation_scores: { 'test text': 0.75 }, + num_observations: 1, + }; + (global.fetch as jest.Mock).mockResolvedValueOnce(makeOkResponse(mockScore)); + + const service = makeSentinelService(BASE_URL); + const score = await service.scoreSingleText('test text'); + expect(score).toBe(0.75); + }); + + it('returns 0 when observation_scores is empty', async () => { + const mockScore: SentinelScoreResponse = { + rare_class_affinity_score: 0, + observation_scores: {}, + num_observations: 0, + }; + (global.fetch as jest.Mock).mockResolvedValueOnce(makeOkResponse(mockScore)); + + const service = makeSentinelService(BASE_URL); + const score = await service.scoreSingleText('test text'); + expect(score).toBe(0); + }); + }); + + describe('default URL', () => { + it('uses localhost:8000 when no URL is provided', async () => { + const mockResponse: SentinelHealthResponse = { status: 'ok', banks_loaded: true }; + (global.fetch as jest.Mock).mockResolvedValueOnce(makeOkResponse(mockResponse)); + + const service = makeSentinelService(); + await service.healthCheck(); + + expect(global.fetch).toHaveBeenCalledWith( + 'http://localhost:8000/health', + expect.anything(), + ); + }); + + it('strips trailing slash from base URL', async () => { + const mockResponse: SentinelHealthResponse = { status: 'ok', banks_loaded: true }; + (global.fetch as jest.Mock).mockResolvedValueOnce(makeOkResponse(mockResponse)); + + const service = makeSentinelService('http://sentinel.example.com/'); + await service.healthCheck(); + + expect(global.fetch).toHaveBeenCalledWith( + 'http://sentinel.example.com/health', + expect.anything(), + ); + }); + }); +}); diff --git a/server/services/sentinelService/sentinelService.ts b/server/services/sentinelService/sentinelService.ts new file mode 100644 index 00000000..33621bf4 --- /dev/null +++ b/server/services/sentinelService/sentinelService.ts @@ -0,0 +1,160 @@ +/** + * Sentinel Service — HTTP client for the Sentinel API. + * + * Sentinel is a contrastive-learning system for detecting rare text patterns + * (e.g. grooming, hate speech) by comparing text against positive (rare/harmful) + * and negative (common/normal) example banks. + * + * The factory does NOT read process.env directly — the caller passes in the + * URL, keeping the module testable and DI-friendly. + */ + +export type SentinelScoreRequest = { + texts: string[]; + top_k?: number; + min_score_to_consider?: number; +}; + +export type SentinelScoreResponse = { + rare_class_affinity_score: number; + observation_scores: { [text: string]: number }; + num_observations: number; +}; + +export type SentinelBanksStatus = { + loaded: boolean; + model_name?: string; + positive_count?: number; + negative_count?: number; + model_card?: Record; +}; + +export type SentinelHealthResponse = { + status: string; + banks_loaded: boolean; + version?: string; +}; + +export type SentinelLoadBanksRequest = { + path: string; + negative_to_positive_ratio?: number; +}; + +export class SentinelServiceError extends Error { + public readonly statusCode?: number; + + constructor(message: string, statusCode?: number, cause?: Error) { + super(message, { cause }); + this.name = 'SentinelServiceError'; + this.statusCode = statusCode; + } +} + +export interface SentinelService { + /** Score an array of texts for rare class affinity. */ + scoreTexts(request: SentinelScoreRequest): Promise; + + /** + * Score a single text for rare class affinity. + * Returns the individual observation score for that text. + */ + scoreSingleText(text: string): Promise; + + /** Get the current banks status. */ + getBanksStatus(): Promise; + + /** Check if the Sentinel service is healthy. */ + healthCheck(): Promise; + + /** Load banks from a given path. */ + loadBanks(request: SentinelLoadBanksRequest): Promise; + + /** Unload current banks from memory. */ + unloadBanks(): Promise; +} + +const DEFAULT_BASE_URL = 'http://localhost:8000'; + +export function makeSentinelService(apiUrl?: string): SentinelService { + const baseUrl = (apiUrl ?? DEFAULT_BASE_URL).replace(/\/$/, ''); + + async function fetchWithError( + endpoint: string, + options: RequestInit = {}, + ): Promise { + try { + const response = await fetch(`${baseUrl}${endpoint}`, { + ...options, + headers: { + 'Content-Type': 'application/json', + ...options.headers, + }, + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new SentinelServiceError( + `Sentinel API error: ${errorText}`, + response.status, + ); + } + + return response.json() as Promise; + } catch (error) { + if (error instanceof SentinelServiceError) { + throw error; + } + throw new SentinelServiceError( + `Failed to connect to Sentinel service: ${error instanceof Error ? error.message : 'Unknown error'}`, + undefined, + error instanceof Error ? error : undefined, + ); + } + } + + return { + async scoreTexts(request: SentinelScoreRequest): Promise { + return fetchWithError('/score', { + method: 'POST', + body: JSON.stringify({ + texts: request.texts, + top_k: request.top_k ?? 5, + min_score_to_consider: request.min_score_to_consider ?? 0.0, + }), + }); + }, + + async scoreSingleText(text: string): Promise { + const response = await this.scoreTexts({ texts: [text] }); + const scores = Object.values(response.observation_scores); + return scores.length > 0 ? (scores[0] ?? 0) : 0; + }, + + async getBanksStatus(): Promise { + return fetchWithError('/banks/status', { + method: 'GET', + }); + }, + + async healthCheck(): Promise { + return fetchWithError('/health', { + method: 'GET', + }); + }, + + async loadBanks(request: SentinelLoadBanksRequest): Promise { + await fetchWithError('/banks/load', { + method: 'POST', + body: JSON.stringify(request), + }); + }, + + async unloadBanks(): Promise { + await fetchWithError('/banks/unload', { + method: 'POST', + }); + }, + }; +} + +export default makeSentinelService; From 465c469928a3bb22d656ec6725fc3c9ea58362c3 Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Tue, 5 May 2026 16:32:25 -0400 Subject: [PATCH 03/10] Add SentinelRareClassAffinitySignal with thread-aware scoring and tests --- .../SentinelRareClassAffinitySignal.test.ts | 271 ++++++++++++++++++ .../SentinelRareClassAffinitySignal.ts | 217 ++++++++++++++ 2 files changed, 488 insertions(+) create mode 100644 server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts create mode 100644 server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts diff --git a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts new file mode 100644 index 00000000..cf0d6289 --- /dev/null +++ b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts @@ -0,0 +1,271 @@ +import { ScalarTypes } from '@roostorg/types'; + +import { Integration } from '../../../types/Integration.js'; +import { SignalPricingStructure } from '../../../types/SignalPricingStructure.js'; +import { SignalType } from '../../../types/SignalType.js'; +import { type SignalInput } from '../../SignalBase.js'; +import { + type SentinelService, + SentinelServiceError, +} from '../../../../sentinelService/sentinelService.js'; +import SentinelRareClassAffinitySignal from './SentinelRareClassAffinitySignal.js'; + +type SentinelSignalInput = SignalInput< + ScalarTypes['STRING'], + false, + false, + string, + 'SENTINEL_RARE_CLASS_AFFINITY' +>; + +function makeInput( + overrides: Partial = {}, +): SentinelSignalInput { + return { + value: { type: 'STRING', value: 'test content' }, + matchingValues: undefined, + actionPenalties: undefined, + orgId: 'org-1', + ...overrides, + } as unknown as SentinelSignalInput; +} + +function makeSentinelService( + overrides: Partial = {}, +): SentinelService { + return { + healthCheck: jest.fn().mockResolvedValue({ status: 'ok', banks_loaded: true }), + getBanksStatus: jest.fn().mockResolvedValue({ loaded: true }), + scoreTexts: jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.5, + observation_scores: { 'test content': 0.5 }, + num_observations: 1, + }), + scoreSingleText: jest.fn().mockResolvedValue(0.5), + loadBanks: jest.fn().mockResolvedValue(undefined), + unloadBanks: jest.fn().mockResolvedValue(undefined), + ...overrides, + }; +} + +type MockItemInvestigationService = { + insertItem: jest.Mock; + getThreadSubmissionsByPosition: jest.Mock; + getThreadSubmissionsByTime: jest.Mock; + getItemByIdentifier: jest.Mock; + getItemByTypeAgnosticIdentifier: jest.Mock; + getAncestorItems: jest.Mock; + getItemSubmissionsByCreator: jest.Mock; + getItemActionHistory: jest.Mock; +}; + +function makeItemInvestigationService( + overrides: Partial = {}, +): MockItemInvestigationService { + return { + insertItem: jest.fn().mockResolvedValue(undefined), + getThreadSubmissionsByPosition: jest.fn(), + getThreadSubmissionsByTime: jest.fn().mockReturnValue( + (async function* () { + // Empty async iterable by default + })(), + ), + getItemByIdentifier: jest.fn(), + getItemByTypeAgnosticIdentifier: jest.fn(), + getAncestorItems: jest.fn(), + getItemSubmissionsByCreator: jest.fn(), + getItemActionHistory: jest.fn(), + ...overrides, + }; +} + +function makeSignal( + sentinelService?: Partial, + iisOverrides?: Partial, +) { + return new SentinelRareClassAffinitySignal( + makeSentinelService(sentinelService), + makeItemInvestigationService(iisOverrides) as any, + ); +} + +describe('SentinelRareClassAffinitySignal', () => { + describe('signal metadata', () => { + it('returns correct id', () => { + expect(makeSignal().id).toEqual({ type: SignalType.SENTINEL_RARE_CLASS_AFFINITY }); + }); + + it('returns correct integration', () => { + expect(makeSignal().integration).toBe(Integration.SENTINEL); + }); + + it('returns STRING as eligible input type', () => { + expect(makeSignal().eligibleInputs).toEqual([ScalarTypes.STRING]); + }); + + it('returns NUMBER as output type', () => { + expect(makeSignal().outputType).toEqual({ scalarType: ScalarTypes.NUMBER }); + }); + + it('returns FREE pricing structure', () => { + expect(makeSignal().pricingStructure).toBe(SignalPricingStructure.FREE); + }); + + it('is allowed in automated rules', () => { + expect(makeSignal().allowedInAutomatedRules).toBe(true); + }); + + it('does not need matching values', () => { + expect(makeSignal().needsMatchingValues).toBe(false); + }); + + it('does not need action penalties', () => { + expect(makeSignal().needsActionPenalties).toBe(false); + }); + + it('has no eligible subcategories', () => { + expect(makeSignal().eligibleSubcategories).toEqual([]); + }); + + it('returns ALL as supported languages', () => { + expect(makeSignal().supportedLanguages).toBe('ALL'); + }); + }); + + describe('getDisabledInfo', () => { + it('returns disabled=false when service is healthy and banks are loaded', async () => { + const info = await makeSignal().getDisabledInfo('org-1'); + expect(info.disabled).toBe(false); + }); + + it('returns disabled=true when health check fails', async () => { + const signal = makeSignal({ + healthCheck: jest.fn().mockRejectedValue(new Error('Connection refused')), + }); + const info = await signal.getDisabledInfo('org-1'); + expect(info.disabled).toBe(true); + expect(info.disabledMessage).toContain('unavailable'); + }); + + it('returns disabled=true when health status is not ok', async () => { + const signal = makeSignal({ + healthCheck: jest.fn().mockResolvedValue({ status: 'error', banks_loaded: false }), + }); + const info = await signal.getDisabledInfo('org-1'); + expect(info.disabled).toBe(true); + expect(info.disabledMessage).toContain('not healthy'); + }); + + it('returns disabled=true when banks are not loaded', async () => { + const signal = makeSignal({ + getBanksStatus: jest.fn().mockResolvedValue({ loaded: false }), + }); + const info = await signal.getDisabledInfo('org-1'); + expect(info.disabled).toBe(true); + expect(info.disabledMessage).toContain('banks are not loaded'); + }); + }); + + describe('run', () => { + it('scores the primary text and returns rare_class_affinity_score', async () => { + const scoreTexts = jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.72, + observation_scores: { 'test content': 0.72 }, + num_observations: 1, + }); + const signal = makeSignal({ scoreTexts }); + + const result = await signal.run(makeInput()); + + expect(scoreTexts).toHaveBeenCalledWith( + expect.objectContaining({ texts: expect.arrayContaining(['test content']) }), + ); + expect(result).toMatchObject({ + outputType: { scalarType: ScalarTypes.NUMBER }, + score: 0.72, + }); + }); + + it('includes thread context texts when threadIdentifier is provided', async () => { + const scoreTexts = jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.85, + observation_scores: {}, + num_observations: 2, + }); + + // Async iterable that yields one prior thread item + const threadItem = { + latestSubmission: { + data: { text: 'prior thread message' }, + itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, + }, + priorSubmissions: undefined, + parents: (async function* () {})(), + }; + const getThreadSubmissionsByTime = jest + .fn() + .mockReturnValue( + (async function* () { + yield threadItem; + })(), + ); + + const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + + await signal.run( + makeInput({ + runtimeArgs: { + threadIdentifier: { id: 'thread-1', typeId: 'content-type-1' }, + contentTextFieldName: 'text', + }, + }), + ); + + const texts = scoreTexts.mock.calls[0][0].texts; + expect(texts).toContain('test content'); + expect(texts).toContain('prior thread message'); + }); + + it('still returns a score when thread fetch fails', async () => { + const scoreTexts = jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.5, + observation_scores: { 'test content': 0.5 }, + num_observations: 1, + }); + const getThreadSubmissionsByTime = jest.fn().mockImplementation(() => { + throw new Error('Scylla unavailable'); + }); + + const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + + const result = await signal.run( + makeInput({ + runtimeArgs: { + threadIdentifier: { id: 'thread-1', typeId: 'content-type-1' }, + }, + }), + ); + + // Should still score with just the primary text + expect(result).toMatchObject({ score: 0.5 }); + expect(scoreTexts).toHaveBeenCalledWith( + expect.objectContaining({ texts: ['test content'] }), + ); + }); + + it('returns an error result when Sentinel service throws SentinelServiceError', async () => { + const signal = makeSignal({ + scoreTexts: jest.fn().mockRejectedValue(new SentinelServiceError('Banks not loaded', 503)), + }); + const result = await signal.run(makeInput()); + expect(result.type).toBe('ERROR'); + }); + + it('re-throws non-SentinelServiceError exceptions', async () => { + const signal = makeSignal({ + scoreTexts: jest.fn().mockRejectedValue(new Error('Unexpected error')), + }); + await expect(signal.run(makeInput())).rejects.toThrow('Unexpected error'); + }); + }); +}); diff --git a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts new file mode 100644 index 00000000..d2929b6f --- /dev/null +++ b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts @@ -0,0 +1,217 @@ +import { ScalarTypes } from '@roostorg/types'; + +import { type ItemInvestigationService } from '../../../../itemInvestigationService/index.js'; +import { type ItemSubmission } from '../../../../itemProcessingService/index.js'; +import { SignalType } from '../../../types/SignalType.js'; +import { SignalPricingStructure } from '../../../types/SignalPricingStructure.js'; +import SignalBase, { type SignalInput } from '../../SignalBase.js'; +import { + type SentinelService, + SentinelServiceError, +} from '../../../../sentinelService/index.js'; +import { makeSignalPermanentError } from '../../../../../utils/errors.js'; + +const SENTINEL_DOCS_URL = 'https://github.com/UMass-Rescue/Sentinel'; + +/** + * How many prior thread items to include in Sentinel scoring. + * More context improves recall for pattern detection but adds latency. + */ +const DEFAULT_THREAD_CONTEXT_LIMIT = 10; + +export default class SentinelRareClassAffinitySignal extends SignalBase< + ScalarTypes['STRING'], + { scalarType: ScalarTypes['NUMBER'] } +> { + constructor( + private readonly sentinelService: SentinelService, + private readonly itemInvestigationService: ItemInvestigationService, + ) { + super(); + } + + override get id() { + return { type: SignalType.SENTINEL_RARE_CLASS_AFFINITY }; + } + + override get displayName() { + return 'Sentinel Rare Class Affinity'; + } + + override get description() { + return `Sentinel is a contrastive-learning system that detects rare harmful text patterns (e.g., grooming) by measuring semantic affinity to curated example banks. + +It compares submitted content against labeled positive (rare/harmful) and negative (common/normal) examples to produce a score between 0 and 1. Higher scores indicate stronger affinity to the rare class. Thread context is included when available, improving detection of patterns that span multiple messages.`; + } + + override get docsUrl() { + return SENTINEL_DOCS_URL; + } + + override get recommendedThresholds() { + return null; + } + + override get supportedLanguages(): 'ALL' { + return 'ALL'; + } + + override get pricingStructure(): SignalPricingStructure { + return SignalPricingStructure.FREE; + } + + override get eligibleInputs() { + return [ScalarTypes.STRING]; + } + + override get outputType() { + return { scalarType: ScalarTypes.NUMBER }; + } + + override getCost() { + return 50; + } + + override get needsMatchingValues() { + return false; + } + + override get eligibleSubcategories(): [] { + return []; + } + + override get needsActionPenalties() { + return false; + } + + override get integration() { + return 'SENTINEL' as const; + } + + override get allowedInAutomatedRules() { + return true; + } + + override async getDisabledInfo(_orgId: string) { + try { + const health = await this.sentinelService.healthCheck(); + if (health.status !== 'ok' && health.status !== 'healthy') { + return { + disabled: true as const, + disabledMessage: + 'Sentinel service is not healthy. Please ensure the Sentinel server is running and banks are loaded.', + }; + } + + const banksStatus = await this.sentinelService.getBanksStatus(); + if (!banksStatus.loaded) { + return { + disabled: true as const, + disabledMessage: + 'Sentinel banks are not loaded. Please load the required model banks before using this signal.', + }; + } + + return { disabled: false as const }; + } catch { + return { + disabled: true as const, + disabledMessage: + 'Sentinel service is unavailable. Please ensure the Sentinel server is running and reachable.', + }; + } + } + + async run( + input: SignalInput< + ScalarTypes['STRING'], + false, + false, + string, + 'SENTINEL_RARE_CLASS_AFFINITY' + >, + ) { + const { value, orgId, runtimeArgs } = input; + + // The primary text to score is always the signal input value. + const primaryText = String(value.value); + const texts: string[] = [primaryText]; + + // If thread context is available, fetch prior messages in the thread and + // include their text in the scoring request. This provides contextual + // signals that help Sentinel detect patterns spanning multiple messages. + if (runtimeArgs?.threadIdentifier) { + try { + const threadItems = this.itemInvestigationService.getThreadSubmissionsByTime({ + orgId, + threadId: runtimeArgs.threadIdentifier, + limit: DEFAULT_THREAD_CONTEXT_LIMIT, + }); + + for await (const { latestSubmission } of threadItems) { + const itemText = extractTextFromSubmission( + latestSubmission, + runtimeArgs.contentTextFieldName, + ); + if (itemText) { + texts.push(itemText); + } + } + } catch { + // Thread context is best-effort. If it fails, we continue with just + // the primary text rather than failing the entire signal execution. + } + } + + try { + const response = await this.sentinelService.scoreTexts({ texts }); + return { + outputType: { scalarType: ScalarTypes.NUMBER }, + score: response.rare_class_affinity_score, + }; + } catch (error) { + if (error instanceof SentinelServiceError) { + return { + type: 'ERROR' as const, + score: makeSignalPermanentError('Sentinel scoring failed', { + detail: error.message, + shouldErrorSpan: false, + }), + }; + } + throw error; + } + } +} + +/** + * Extract the text to score from an ItemSubmission. + * + * If a specific `contentTextFieldName` is given (derived from the condition's + * input field), we look up that field by name. Otherwise, we collect the first + * string-valued field we can find. Returns `undefined` if no suitable text is + * found. + */ +function extractTextFromSubmission( + submission: ItemSubmission, + contentTextFieldName?: string, +): string | undefined { + const data = submission.data as Record; + + if (contentTextFieldName != null) { + const rawValue = data[contentTextFieldName]; + if (typeof rawValue === 'string' && rawValue.trim()) { + return rawValue; + } + return undefined; + } + + // Fall back: return the first string field value in the item's data. + for (const value of Object.values(data)) { + if (typeof value === 'string' && value.trim()) { + return value; + } + } + + return undefined; +} From 518ada2c4b953b88edf2d2284a2d53151506d70a Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Tue, 5 May 2026 16:33:05 -0400 Subject: [PATCH 04/10] Register Sentinel signal in instantiateBuiltInSignals and leafCondition runtime args --- server/condition_evaluator/leafCondition.ts | 26 ++++++++++++++++++- .../services/signalsService/SignalsService.ts | 3 +++ .../helpers/instantiateBuiltInSignals.ts | 8 ++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/server/condition_evaluator/leafCondition.ts b/server/condition_evaluator/leafCondition.ts index eb3cf44a..9b3585d8 100644 --- a/server/condition_evaluator/leafCondition.ts +++ b/server/condition_evaluator/leafCondition.ts @@ -2,6 +2,7 @@ import { getScalarType, ScalarTypes, + type ItemIdentifier, type ScalarType, type TaggedScalar, } from '@roostorg/types'; @@ -205,7 +206,7 @@ export async function runLeafCondition( : undefined, args: signal.args, runtimeArgs: isFullSubmission(ruleInput) - ? await getSignalRuntimeArgs(evaluationContext, ruleInput, signal) + ? await getSignalRuntimeArgs(evaluationContext, ruleInput, signal, conditionInput) : undefined, }); }), @@ -558,6 +559,7 @@ async function getSignalRuntimeArgs( evaluationContext: RuleEvaluationContext, itemSubmission: ItemSubmission, conditionSignalInfo: ReadonlyDeep, + conditionInput: ReadonlyDeep, ) { if (conditionSignalInfo.type === 'AGGREGATION') { return evaluateAggregationRuntimeArgsForItem( @@ -566,6 +568,28 @@ async function getSignalRuntimeArgs( conditionSignalInfo.args.aggregationClause, ); } + + if (conditionSignalInfo.type === 'SENTINEL_RARE_CLASS_AFFINITY') { + // Extract the thread item identifier from the content schema's threadId + // role — this lets Sentinel score the content in its conversational context. + const threadIdentifier: ItemIdentifier | undefined = + itemSubmission.itemType.kind === 'CONTENT' + ? (getFieldValueForRole( + itemSubmission.itemType.schema, + itemSubmission.itemType.schemaFieldRoles, + 'threadId', + itemSubmission.data, + ) ?? undefined) + : undefined; + + // The field name the condition is evaluating is the most relevant text to + // score. If the condition input is a CONTENT_FIELD, use that field name. + const contentTextFieldName: string | undefined = + conditionInput.type === 'CONTENT_FIELD' ? conditionInput.name : undefined; + + return { threadIdentifier, contentTextFieldName }; + } + return undefined; } diff --git a/server/services/signalsService/SignalsService.ts b/server/services/signalsService/SignalsService.ts index 963d06f0..7bbe02c1 100644 --- a/server/services/signalsService/SignalsService.ts +++ b/server/services/signalsService/SignalsService.ts @@ -115,6 +115,7 @@ export class SignalsService { signalAuthService: Dependencies['SignalAuthService'], getUserScoreEventuallyConsistent: Dependencies['getUserScoreEventuallyConsistent'], fetchHTTP: Dependencies['fetchHTTP'], + itemInvestigationService: Dependencies['ItemInvestigationService'], ) { const cachedCredentialGetters = makeCachedCredentialGetters(signalAuthService); @@ -132,6 +133,7 @@ export class SignalsService { this.userStrikeService, this.getPoliciesByIdEventuallyConsistent, this.hmaService, + itemInvestigationService, ); this.close = async function () { @@ -342,6 +344,7 @@ export default inject( 'SignalAuthService', 'getUserScoreEventuallyConsistent', 'fetchHTTP', + 'ItemInvestigationService', ], SignalsService, ); diff --git a/server/services/signalsService/helpers/instantiateBuiltInSignals.ts b/server/services/signalsService/helpers/instantiateBuiltInSignals.ts index 466dace4..9215fd0e 100644 --- a/server/services/signalsService/helpers/instantiateBuiltInSignals.ts +++ b/server/services/signalsService/helpers/instantiateBuiltInSignals.ts @@ -3,6 +3,7 @@ import { type ItemIdentifier } from '@roostorg/types'; import type { AggregationsService } from '../../aggregationsService/index.js'; import type { HmaService } from '../../hmaService/index.js'; +import type { ItemInvestigationService } from '../../itemInvestigationService/index.js'; import type { GetPoliciesByIdEventuallyConsistent } from '../../manualReviewToolService/manualReviewToolQueries.js'; import { type UserScore } from '../../userStatisticsService/userStatisticsService.js'; import { type UserStrikeService } from '../../userStrikeService/index.js'; @@ -33,12 +34,14 @@ import OpenAiSexualMinorsTextSignal from '../signals/third_party_signals/open_ai import OpenAiSexualTextSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiSexualTextSignal.js'; import OpenAiViolenceTextSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiViolenceTextSignal.js'; import OpenAiWhisperTranscriptionSignal from '../signals/third_party_signals/open_ai/whisper/OpenAiWhisperTranscriptionSignal.js'; +import SentinelRareClassAffinitySignal from '../signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.js'; import ZentropiLabelerSignal from '../signals/third_party_signals/zentropi/ZentropiLabelerSignal.js'; import UserScoreSignal from '../signals/UserScoreSignal.js'; import UserStrikesSignal from '../signals/UserStrikesSignal.js'; import { SignalType, type BuiltInSignalType } from '../types/SignalType.js'; import { type CredentialGetters } from './makeCachedCredentialsGetters.js'; import { type CachedFetchers } from './makeCachedFetchers.js'; +import { makeSentinelService } from '../../sentinelService/index.js'; export function instantiateBuiltInSignals( credentialGetters: CredentialGetters, @@ -51,6 +54,7 @@ export function instantiateBuiltInSignals( userStrikeService: UserStrikeService, _getPoliciesByIdEventuallyConsistent: GetPoliciesByIdEventuallyConsistent, hmaService: HmaService, + itemInvestigationService: ItemInvestigationService, ) { const { googleContentSafetyFetcher: getGoogleContentSafetyScores, @@ -128,6 +132,10 @@ export function instantiateBuiltInSignals( new GoogleCloudTranslationAPISignal(), [SignalType.BENIGN_MODEL]: new CoopRiskModelSignal(), [SignalType.AGGREGATION]: new AggregationSignal(aggregationsService), + [SignalType.SENTINEL_RARE_CLASS_AFFINITY]: new SentinelRareClassAffinitySignal( + makeSentinelService(process.env.SENTINEL_API_URL), + itemInvestigationService, + ), [SignalType.ZENTROPI_LABELER]: new ZentropiLabelerSignal( credentialGetters.ZENTROPI, getZentropiScores, From 465237d8c17abc1dc0148e1766d893866e22fe61 Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Tue, 5 May 2026 16:33:27 -0400 Subject: [PATCH 05/10] Pre-write submitted content to thread store so thread-aware signals can read history --- server/routes/content/submitContent.ts | 51 +++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/server/routes/content/submitContent.ts b/server/routes/content/submitContent.ts index 7cd3032c..16125219 100644 --- a/server/routes/content/submitContent.ts +++ b/server/routes/content/submitContent.ts @@ -31,7 +31,7 @@ import { sanitizeError, type SerializableError, } from '../../utils/errors.js'; -import { safeGet, safePick, sleep } from '../../utils/misc.js'; +import { safeGet, safePick, sleep, withRetries } from '../../utils/misc.js'; import { type RequestHandlerWithBodies } from '../../utils/route-helpers.js'; import { instantiateOpaqueType } from '../../utils/typescript-types.js'; import { hasOrgId } from '../../utils/apiKeyMiddleware.js'; @@ -46,6 +46,7 @@ export default function submitContent({ Tracer, ModerationConfigService, Meter, + ItemInvestigationService, }: // @ts-ignore Dependencies): RequestHandlerWithBodies< EvaluateContentInputCamelCase, @@ -193,6 +194,37 @@ Dependencies): RequestHandlerWithBodies< res.status(202).end(); } + // Write to Scylla before running rules so that thread-aware signals + // (e.g. Sentinel) can read prior messages in the thread when scoring. + // This is best-effort: failures are swallowed so they never block rule execution. + const insertWithRetries = Tracer.traced( + { + resource: 'SubmitContent', + operation: 'ItemInvestigationService.insertItem', + }, + withRetries( + { + maxRetries: 1, + initialTimeMsBetweenRetries: 50, + maxTimeMsBetweenRetries: 200, + }, + ItemInvestigationService.insertItem.bind(ItemInvestigationService), + ), + ); + try { + await insertWithRetries({ + requestId, + orgId, + itemSubmission: { + ...itemSubmissionOrError.itemSubmission, + submissionTime: + itemSubmissionOrError.itemSubmission.submissionTime ?? new Date(), + }, + }); + } catch { + // Swallow insert errors; rules can still run without thread context. + } + // Run rules try { const results = await RuleEngine.runEnabledRules( @@ -328,11 +360,26 @@ function rawContentSubmissionToItemSubmission( | { itemSubmission: ItemSubmission; error?: undefined } { const submissionTime = new Date(); + // Coerce a string threadId to { id, typeId } when the content type has a + // threadId schema field role. This allows callers to pass a plain string ID + // for the parent thread without knowing the internal identifier format. + const raw = rawContentSubmission.content; + const threadIdField = + 'threadId' in itemType.schemaFieldRoles + ? (itemType.schemaFieldRoles as { threadId?: string }).threadId + : undefined; + const content: RawItemData = + threadIdField && + typeof raw[threadIdField] === 'string' && + (raw[threadIdField] as string).length > 0 + ? { ...raw, [threadIdField]: { id: raw[threadIdField] as string, typeId: itemType.id } } + : raw; + // Validate content JSON const normalizedDataOrValidationErrors = toNormalizedItemDataOrErrors( legalItemTypeIds, itemType, - rawContentSubmission.content, + content, ); return Array.isArray(normalizedDataOrValidationErrors) From 476a75ce6473bf9e61fe187c0f31f1c0ac658575 Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Tue, 5 May 2026 16:33:50 -0400 Subject: [PATCH 06/10] Add non-configurable Sentinel tile to integrations dashboard --- .../dashboard/integrations/integrationConfigs.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/client/src/webpages/dashboard/integrations/integrationConfigs.ts b/client/src/webpages/dashboard/integrations/integrationConfigs.ts index 159fc3d9..2538644a 100644 --- a/client/src/webpages/dashboard/integrations/integrationConfigs.ts +++ b/client/src/webpages/dashboard/integrations/integrationConfigs.ts @@ -3,6 +3,7 @@ import GoogleLogo from '../../../images/GoogleLogo.png'; import GoogleLogoWithBackground from '../../../images/GoogleLogoWithBackground.png'; import OpenAILogo from '../../../images/OpenAILogo.png'; import OpenAILogoWithBackground from '../../../images/OpenAILogoWithBackground.png'; +import RobloxLogo from '../../../images/RobloxLogo.png'; import ZentropiLogo from '../../../images/ZentropiLogo.png'; import { IntegrationConfig } from './IntegrationsDashboard'; @@ -23,6 +24,14 @@ export const INTEGRATION_CONFIGS: IntegrationConfig[] = [ url: 'https://openai.com/', requiresInfo: true, }, + { + name: GQLIntegration.Sentinel, + title: 'Sentinel', + logo: RobloxLogo, + logoWithBackground: RobloxLogo, + url: 'https://github.com/UMass-Rescue/Sentinel', + requiresInfo: false, + }, { name: GQLIntegration.Zentropi, title: 'Zentropi', From 24a028c499f2b98a155fc1d40acb6c2a456dc698 Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Tue, 5 May 2026 16:34:12 -0400 Subject: [PATCH 07/10] Document Sentinel local development setup in DEVELOPMENT.md --- docs/DEVELOPMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 2107ac8f..71275ace 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -169,7 +169,7 @@ Schema changes trigger recompilation of both client and server. If you experienc ## HMA Development HMA is not started automatically with `npm run up`. Start it separately if you're doing hash matching: `docker compose up --build -d hma` -HMA is pre-configured in `server/.env` with `HMA_SERVICE_URL=http://localhost:5000`. No additional environment setup is needed for local development. +HMA is pre-configured in `server/.env` with `HMA_SERVICE_URL=http://localhost:5001`. No additional environment setup is needed for local development. ### Image URL Accessibility When submitting items to Coop, image URLs must be reachable by the HMA Docker container and not just your browser or the Node.js server. From 9903ad7727709959ce21196c0072c2c1a21e038f Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Tue, 5 May 2026 16:34:34 -0400 Subject: [PATCH 08/10] Fix itemType create/update mutations returning null data on success --- server/graphql/modules/itemType.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/server/graphql/modules/itemType.ts b/server/graphql/modules/itemType.ts index 94f82004..61ac28ba 100644 --- a/server/graphql/modules/itemType.ts +++ b/server/graphql/modules/itemType.ts @@ -769,7 +769,7 @@ const Mutation: GQLMutationResolvers = { } return gqlSuccessResult( - contentItemType, + { data: contentItemType }, 'MutateContentTypeSuccessResponse', ); }, @@ -815,7 +815,7 @@ const Mutation: GQLMutationResolvers = { } return gqlSuccessResult( - contentItemType, + { data: contentItemType }, 'MutateContentTypeSuccessResponse', ); }, @@ -853,7 +853,10 @@ const Mutation: GQLMutationResolvers = { ); } - return gqlSuccessResult(threadItemType, 'MutateThreadTypeSuccessResponse'); + return gqlSuccessResult( + { data: threadItemType }, + 'MutateThreadTypeSuccessResponse', + ); }, async updateThreadItemType(_, params, context) { const user = context.getUser(); @@ -897,7 +900,10 @@ const Mutation: GQLMutationResolvers = { ); } - return gqlSuccessResult(threadItemType, 'MutateThreadTypeSuccessResponse'); + return gqlSuccessResult( + { data: threadItemType }, + 'MutateThreadTypeSuccessResponse', + ); }, async createUserItemType(__, params, context) { const user = context.getUser(); @@ -936,7 +942,10 @@ const Mutation: GQLMutationResolvers = { ); } - return gqlSuccessResult(userItemType, 'MutateUserTypeSuccessResponse'); + return gqlSuccessResult( + { data: userItemType }, + 'MutateUserTypeSuccessResponse', + ); }, async updateUserItemType(_, params, context) { const user = context.getUser(); @@ -978,7 +987,10 @@ const Mutation: GQLMutationResolvers = { ); } - return gqlSuccessResult(contentItemType, 'MutateUserTypeSuccessResponse'); + return gqlSuccessResult( + { data: contentItemType }, + 'MutateUserTypeSuccessResponse', + ); }, async deleteItemType(_, params, context) { const user = context.getUser(); From 9102c53709aabe3f0a3e1f39a61f33b117fb49f6 Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Mon, 22 Jun 2026 22:16:09 -0400 Subject: [PATCH 09/10] resolve merge conflict with main --- server/.env.example | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/server/.env.example b/server/.env.example index b5a19910..44b8d079 100644 --- a/server/.env.example +++ b/server/.env.example @@ -26,48 +26,12 @@ CLICKHOUSE_PASSWORD=clickhouse CLICKHOUSE_DATABASE=analytics CLICKHOUSE_PROTOCOL=http -<<<<<<< HEAD -# Snowflake Personal Account Credentials -SNOWFLAKE_USERNAME= -SNOWFLAKE_PASSWORD= -SNOWFLAKE_DB_NAME=MY_DEV_DB -SNOWFLAKE_ACCOUNT= - -HMA_SERVICE_URL=http://localhost:5001 +HMA_SERVICE_URL=http://localhost:9876 # Sentinel API URL (local Sentinel service for rare class affinity signal) # Start Sentinel locally: docker compose -f docker-compose.yaml -f docker-compose.sentinel.yaml up -d sentinel SENTINEL_API_URL=http://localhost:8000 -# Kafka authentication info. -KAFKA_BROKER_HOST=localhost:29092 -KAFKA_BROKER_USERNAME= -KAFKA_BROKER_PASSWORD= -KAFKA_SCHEMA_REGISTRY_HOST=http://localhost:8081 -KAFKA_SCHEMA_REGISTRY_USERNAME= -KAFKA_SCHEMA_REGISTRY_PASSWORD= -KAFKAJS_NO_PARTITIONER_WARNING=1 - -# NB: these schema ids are different on prod + staging clusters; may be -# different in our future local dev setup. Eventually, we'll likely want -# a more sophisticated approach than env vars for determining these values, -# but we need to figure out our Kafka schema migration system first. -KAFKA_TOPIC_KEY_SCHEMA_ID_SNOWFLAKE_INGEST_EVENTS= -KAFKA_TOPIC_KEY_SCHEMA_ID_DATA_WAREHOUSE_INGEST_EVENTS= -KAFKA_TOPIC_VALUE_SCHEMA_ID_SNOWFLAKE_INGEST_EVENTS= -KAFKA_TOPIC_VALUE_SCHEMA_ID_DATA_WAREHOUSE_INGEST_EVENTS= -KAFKA_TOPIC_KEY_SCHEMA_ID_ITEM_SUBMISSION_EVENTS= -KAFKA_TOPIC_VALUE_SCHEMA_ID_ITEM_SUBMISSION_EVENTS= -KAFKA_TOPIC_KEY_SCHEMA_ID_ITEM_SUBMISSION_EVENTS_RETRY_0= -KAFKA_TOPIC_VALUE_SCHEMA_ID_ITEM_SUBMISSION_EVENTS_RETRY_0= - -# Bucket info for moving pg data in bulk into Snowflake -SNOWFLAKE_S3_BUCKET_NAME= -SNOWFLAKE_S3_BUCKET_REGION=us-east-2 -======= -HMA_SERVICE_URL=http://localhost:9876 ->>>>>>> main - # Scylla Cluster Details SCYLLA_USERNAME=cassandra SCYLLA_PASSWORD=cassandra From 0dcd5c14b8c4183d400d083d84b298fa823bc85f Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Thu, 2 Jul 2026 17:49:14 -0400 Subject: [PATCH 10/10] Fix double-counting in Sentinel rare-class-affinity thread context * SentinelRareClassAffinitySignal.ts: skip the first thread-context item that matches the primary text - submitContent.ts writes the current submission to Scylla before rules run, so getThreadSubmissionsByTime (time-bounded, not identity-bounded) returns the triggering submission itself alongside genuine prior messages, skewing the aggregate score Sentinel computes * SentinelRareClassAffinitySignal.test.ts: add regression tests - covers the de-dup path when the triggering submission is echoed back - covers that only one occurrence is dropped if a genuine duplicate message exists in the thread --- .../SentinelRareClassAffinitySignal.test.ts | 86 +++++++++++++++++++ .../SentinelRareClassAffinitySignal.ts | 18 +++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts index 3e00cda8..6f12aa44 100644 --- a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts +++ b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts @@ -236,6 +236,92 @@ describe('SentinelRareClassAffinitySignal', () => { expect(texts).toContain('prior thread message'); }); + it('does not double-count the triggering submission when it is echoed back as thread context', async () => { + // submitContent.ts writes the current submission to Scylla before + // running rules, so getThreadSubmissionsByTime (which is time-bounded, + // not identity-bounded) can return the very submission that triggered + // this signal run alongside genuine prior messages. + const scoreTexts = jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.5, + observation_scores: {}, + num_observations: 2, + }); + + const priorItem = { + latestSubmission: { + data: { text: 'prior thread message' }, + itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, + }, + priorSubmissions: undefined, + parents: (async function* () {})(), + }; + const selfItem = { + latestSubmission: { + data: { text: 'test content' }, + itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, + }, + priorSubmissions: undefined, + parents: (async function* () {})(), + }; + const getThreadSubmissionsByTime = jest.fn().mockReturnValue( + (async function* () { + yield selfItem; + yield priorItem; + })(), + ); + + const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + + await signal.run( + makeInput({ + runtimeArgs: { + threadIdentifier: { id: 'thread-1', typeId: 'content-type-1' }, + contentTextFieldName: 'text', + }, + }), + ); + + const texts = scoreTexts.mock.calls[0][0].texts; + expect(texts).toEqual(['test content', 'prior thread message']); + }); + + it('only drops one occurrence of duplicate text, in case a genuine duplicate message exists', async () => { + const scoreTexts = jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.5, + observation_scores: {}, + num_observations: 3, + }); + + const makeItem = (text: string) => ({ + latestSubmission: { + data: { text }, + itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, + }, + priorSubmissions: undefined, + parents: (async function* () {})(), + }); + const getThreadSubmissionsByTime = jest.fn().mockReturnValue( + (async function* () { + yield makeItem('test content'); + yield makeItem('test content'); + })(), + ); + + const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + + await signal.run( + makeInput({ + runtimeArgs: { + threadIdentifier: { id: 'thread-1', typeId: 'content-type-1' }, + contentTextFieldName: 'text', + }, + }), + ); + + const texts = scoreTexts.mock.calls[0][0].texts; + expect(texts).toEqual(['test content', 'test content']); + }); + it('still returns a score when thread fetch fails', async () => { const scoreTexts = jest.fn().mockResolvedValue({ rare_class_affinity_score: 0.5, diff --git a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts index de635d62..070879ba 100644 --- a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts +++ b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts @@ -149,14 +149,28 @@ It compares submitted content against labeled positive (rare/harmful) and negati limit: DEFAULT_THREAD_CONTEXT_LIMIT, }); + // `submitContent.ts` writes the current submission to the thread's + // history *before* rules run (so thread-aware signals have context + // available), and `getThreadSubmissionsByTime` is bounded by time, + // not by item identity. That means the submission which triggered + // this very signal run is indistinguishable from real "prior" + // messages and comes back in `threadItems` too. Drop (at most) one + // occurrence matching `primaryText` so it isn't double-counted, + // which would otherwise skew the aggregate score Sentinel computes. + let skippedSelf = false; for await (const { latestSubmission } of threadItems) { const itemText = extractTextFromSubmission( latestSubmission, runtimeArgs.contentTextFieldName, ); - if (itemText) { - texts.push(itemText); + if (itemText == null) { + continue; } + if (!skippedSelf && itemText === primaryText) { + skippedSelf = true; + continue; + } + texts.push(itemText); } } catch { // Thread context is best-effort. If it fails, we continue with just