diff --git a/client/src/graphql/generated.ts b/client/src/graphql/generated.ts index afc2273b..51682af6 100644 --- a/client/src/graphql/generated.ts +++ b/client/src/graphql/generated.ts @@ -1403,6 +1403,7 @@ export type GQLIgnoreDecisionComponent = export const GQLIntegration = { GoogleContentSafetyApi: 'GOOGLE_CONTENT_SAFETY_API', OpenAi: 'OPEN_AI', + Sentinel: 'SENTINEL', Zentropi: 'ZENTROPI', } as const; @@ -4480,6 +4481,7 @@ export const GQLSignalType = { OpenAiViolenceImageModel: 'OPEN_AI_VIOLENCE_IMAGE_MODEL', OpenAiViolenceTextModel: 'OPEN_AI_VIOLENCE_TEXT_MODEL', OpenAiWhisperTranscription: 'OPEN_AI_WHISPER_TRANSCRIPTION', + SentinelRareClassAffinity: 'SENTINEL_RARE_CLASS_AFFINITY', TextMatchingContainsRegex: 'TEXT_MATCHING_CONTAINS_REGEX', TextMatchingContainsText: 'TEXT_MATCHING_CONTAINS_TEXT', TextMatchingContainsVariant: 'TEXT_MATCHING_CONTAINS_VARIANT', diff --git a/client/src/images/RobloxLogo.png b/client/src/images/RobloxLogo.png new file mode 100644 index 00000000..1c651ca7 Binary files /dev/null and b/client/src/images/RobloxLogo.png differ diff --git a/client/src/models/signal.ts b/client/src/models/signal.ts index 24f06f66..2f8cb2e0 100644 --- a/client/src/models/signal.ts +++ b/client/src/models/signal.ts @@ -37,6 +37,8 @@ export function integrationForSignalType(type: string) { case 'OPEN_AI_VIOLENCE_TEXT_MODEL': case 'OPEN_AI_WHISPER_TRANSCRIPTION': return GQLIntegration.OpenAi; + case 'SENTINEL_RARE_CLASS_AFFINITY': + return GQLIntegration.Sentinel; case 'ZENTROPI_LABELER': return GQLIntegration.Zentropi; case 'AGGREGATION': diff --git a/client/src/webpages/dashboard/integrations/integrationConfigs.ts b/client/src/webpages/dashboard/integrations/integrationConfigs.ts index f9781f79..b297674d 100644 --- a/client/src/webpages/dashboard/integrations/integrationConfigs.ts +++ b/client/src/webpages/dashboard/integrations/integrationConfigs.ts @@ -8,6 +8,7 @@ import GoogleLogo from '../../../images/GoogleLogo.png'; import GoogleLogoWithBackground from '../../../images/GoogleLogoWithBackground.png'; import OpenAILogo from '../../../images/OpenAILogo.png'; import OpenAILogoWithBackground from '../../../images/OpenAILogoWithBackground.png'; +import RobloxLogo from '../../../images/RobloxLogo.png'; import ZentropiLogo from '../../../images/ZentropiLogo.png'; export type IntegrationConfig = { @@ -36,6 +37,14 @@ export const INTEGRATION_CONFIGS: IntegrationConfig[] = [ url: 'https://openai.com/', requiresInfo: true, }, + { + name: 'SENTINEL', + title: 'Sentinel', + logo: RobloxLogo, + logoWithBackground: RobloxLogo, + url: 'https://github.com/UMass-Rescue/Sentinel', + requiresInfo: false, + }, { name: 'ZENTROPI', title: 'Zentropi', diff --git a/server/.env.example b/server/.env.example index 774142aa..74f83e3e 100644 --- a/server/.env.example +++ b/server/.env.example @@ -75,6 +75,10 @@ CLICKHOUSE_PROTOCOL=http HMA_SERVICE_URL=http://localhost:9876 +# Sentinel API URL (local Sentinel service for rare class affinity signal) +# Start Sentinel locally: docker compose -f docker-compose.yaml -f docker-compose.sentinel.yaml up -d sentinel +SENTINEL_API_URL=http://localhost:8000 + # Scylla Cluster Details SCYLLA_USERNAME=cassandra SCYLLA_PASSWORD=cassandra diff --git a/server/condition_evaluator/leafCondition.ts b/server/condition_evaluator/leafCondition.ts index e4721bfc..97b31c97 100644 --- a/server/condition_evaluator/leafCondition.ts +++ b/server/condition_evaluator/leafCondition.ts @@ -2,6 +2,7 @@ import { getScalarType, ScalarTypes, + type ItemIdentifier, type ScalarType, type TaggedScalar, } from '@roostorg/coop-types'; @@ -198,7 +199,12 @@ export async function runLeafCondition( : undefined, args: signal.args, runtimeArgs: isFullSubmission(ruleInput) - ? await getSignalRuntimeArgs(evaluationContext, ruleInput, signal) + ? await getSignalRuntimeArgs( + evaluationContext, + ruleInput, + signal, + conditionInput, + ) : undefined, }); }), @@ -551,6 +557,7 @@ async function getSignalRuntimeArgs( evaluationContext: RuleEvaluationContext, itemSubmission: ItemSubmission, conditionSignalInfo: ReadonlyDeep, + conditionInput: ReadonlyDeep, ) { if (conditionSignalInfo.type === 'AGGREGATION') { const args = conditionSignalInfo.args; @@ -562,6 +569,28 @@ async function getSignalRuntimeArgs( ); } } + + if (conditionSignalInfo.type === 'SENTINEL_RARE_CLASS_AFFINITY') { + // Extract the thread item identifier from the content schema's threadId + // role — this lets Sentinel score the content in its conversational context. + const threadIdentifier: ItemIdentifier | undefined = + itemSubmission.itemType.kind === 'CONTENT' + ? (getFieldValueForRole( + itemSubmission.itemType.schema, + itemSubmission.itemType.schemaFieldRoles, + 'threadId', + itemSubmission.data, + ) ?? undefined) + : undefined; + + // The field name the condition is evaluating is the most relevant text to + // score. If the condition input is a CONTENT_FIELD, use that field name. + const contentTextFieldName: string | undefined = + conditionInput.type === 'CONTENT_FIELD' ? conditionInput.name : undefined; + + return { threadIdentifier, contentTextFieldName }; + } + return undefined; } diff --git a/server/decs.d.ts b/server/decs.d.ts index f85d230c..dfdf0abd 100644 --- a/server/decs.d.ts +++ b/server/decs.d.ts @@ -265,5 +265,6 @@ namespace NodeJS { OPEN_AI_API_KEY?: string; SLACK_APP_BEARER_TOKEN?: string; GRAPHQL_OPAQUE_SCALAR_SECRET?: string; + SENTINEL_API_URL?: string; } } diff --git a/server/graphql/generated.ts b/server/graphql/generated.ts index 4920888e..6a16c1e5 100644 --- a/server/graphql/generated.ts +++ b/server/graphql/generated.ts @@ -1471,6 +1471,7 @@ export type GQLIgnoreDecisionComponent = export const GQLIntegration = { GoogleContentSafetyApi: 'GOOGLE_CONTENT_SAFETY_API', OpenAi: 'OPEN_AI', + Sentinel: 'SENTINEL', Zentropi: 'ZENTROPI', } as const; @@ -4548,6 +4549,7 @@ export const GQLSignalType = { OpenAiViolenceImageModel: 'OPEN_AI_VIOLENCE_IMAGE_MODEL', OpenAiViolenceTextModel: 'OPEN_AI_VIOLENCE_TEXT_MODEL', OpenAiWhisperTranscription: 'OPEN_AI_WHISPER_TRANSCRIPTION', + SentinelRareClassAffinity: 'SENTINEL_RARE_CLASS_AFFINITY', TextMatchingContainsRegex: 'TEXT_MATCHING_CONTAINS_REGEX', TextMatchingContainsText: 'TEXT_MATCHING_CONTAINS_TEXT', TextMatchingContainsVariant: 'TEXT_MATCHING_CONTAINS_VARIANT', diff --git a/server/graphql/modules/integration.ts b/server/graphql/modules/integration.ts index 844d0fa9..0d49e0b8 100644 --- a/server/graphql/modules/integration.ts +++ b/server/graphql/modules/integration.ts @@ -19,6 +19,7 @@ const typeDefs = /* GraphQL */ ` enum Integration { GOOGLE_CONTENT_SAFETY_API OPEN_AI + SENTINEL ZENTROPI } diff --git a/server/graphql/modules/itemType.ts b/server/graphql/modules/itemType.ts index 021c52df..40fea0be 100644 --- a/server/graphql/modules/itemType.ts +++ b/server/graphql/modules/itemType.ts @@ -799,7 +799,7 @@ const Mutation: GQLMutationResolvers = { } return gqlSuccessResult( - contentItemType, + { data: contentItemType }, 'MutateContentTypeSuccessResponse', ); }, @@ -845,7 +845,7 @@ const Mutation: GQLMutationResolvers = { } return gqlSuccessResult( - contentItemType, + { data: contentItemType }, 'MutateContentTypeSuccessResponse', ); }, @@ -883,7 +883,10 @@ const Mutation: GQLMutationResolvers = { ); } - return gqlSuccessResult(threadItemType, 'MutateThreadTypeSuccessResponse'); + return gqlSuccessResult( + { data: threadItemType }, + 'MutateThreadTypeSuccessResponse', + ); }, async updateThreadItemType(_, params, context) { const user = context.getUser(); @@ -927,7 +930,10 @@ const Mutation: GQLMutationResolvers = { ); } - return gqlSuccessResult(threadItemType, 'MutateThreadTypeSuccessResponse'); + return gqlSuccessResult( + { data: threadItemType }, + 'MutateThreadTypeSuccessResponse', + ); }, async createUserItemType(__, params, context) { const user = context.getUser(); @@ -966,7 +972,10 @@ const Mutation: GQLMutationResolvers = { ); } - return gqlSuccessResult(userItemType, 'MutateUserTypeSuccessResponse'); + return gqlSuccessResult( + { data: userItemType }, + 'MutateUserTypeSuccessResponse', + ); }, async updateUserItemType(_, params, context) { const user = context.getUser(); @@ -1008,7 +1017,10 @@ const Mutation: GQLMutationResolvers = { ); } - return gqlSuccessResult(contentItemType, 'MutateUserTypeSuccessResponse'); + return gqlSuccessResult( + { data: contentItemType }, + 'MutateUserTypeSuccessResponse', + ); }, async deleteItemType(_, params, context) { const user = context.getUser(); diff --git a/server/graphql/modules/signal.ts b/server/graphql/modules/signal.ts index 060d0c35..5d4a0fb2 100644 --- a/server/graphql/modules/signal.ts +++ b/server/graphql/modules/signal.ts @@ -111,6 +111,7 @@ const typeDefs = /* GraphQL */ ` OPEN_AI_VIOLENCE_IMAGE_MODEL OPEN_AI_VIOLENCE_TEXT_MODEL OPEN_AI_WHISPER_TRANSCRIPTION + SENTINEL_RARE_CLASS_AFFINITY ZENTROPI_LABELER GEO_CONTAINED_WITHIN USER_SCORE diff --git a/server/package-lock.json b/server/package-lock.json index 77d2fac8..bea35df9 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -26,6 +26,7 @@ "@opentelemetry/semantic-conventions": "^1.22.0", "@roostorg/coop-integration-example": "^2.0.0", "@roostorg/coop-types": "^2.4.0", + "@roostorg/types": "^2.1.0", "@sendgrid/mail": "^8.1.6", "@stdlib/stats-binomial-test": "^0.0.7", "@total-typescript/ts-reset": "^0.3.7", @@ -4241,7 +4242,6 @@ "resolved": "https://registry.npmjs.org/@roostorg/types/-/types-2.1.0.tgz", "integrity": "sha512-nxPXjtAdf7ut/X/NYgbDl39r2wDKiO5vEeLPIHjV8MjDQ1/O5CbgJDyi1jvchMnIJW/AKskdask/Yu0J2JuICQ==", "license": "ISC", - "peer": true, "dependencies": { "date-fns": "^2.29.3", "type-fest": "^4.3.2" diff --git a/server/package.json b/server/package.json index cd0d3303..c0c1ce03 100644 --- a/server/package.json +++ b/server/package.json @@ -45,6 +45,7 @@ "@opentelemetry/semantic-conventions": "^1.22.0", "@roostorg/coop-integration-example": "^2.0.0", "@roostorg/coop-types": "^2.4.0", + "@roostorg/types": "^2.1.0", "@sendgrid/mail": "^8.1.6", "@stdlib/stats-binomial-test": "^0.0.7", "@total-typescript/ts-reset": "^0.3.7", diff --git a/server/routes/content/submitContent.ts b/server/routes/content/submitContent.ts index 3017e4ac..013860ab 100644 --- a/server/routes/content/submitContent.ts +++ b/server/routes/content/submitContent.ts @@ -32,7 +32,7 @@ import { sanitizeError, type SerializableError, } from '../../utils/errors.js'; -import { safeGet, safePick, sleep } from '../../utils/misc.js'; +import { safeGet, safePick, sleep, withRetries } from '../../utils/misc.js'; import { type RequestHandlerWithBodies } from '../../utils/route-helpers.js'; import { instantiateOpaqueType } from '../../utils/typescript-types.js'; import { @@ -46,6 +46,7 @@ export default function submitContent({ Tracer, ModerationConfigService, Meter, + ItemInvestigationService, }: // @ts-ignore Dependencies): RequestHandlerWithBodies< EvaluateContentInputCamelCase, @@ -193,6 +194,37 @@ Dependencies): RequestHandlerWithBodies< res.status(202).end(); } + // Write to Scylla before running rules so that thread-aware signals + // (e.g. Sentinel) can read prior messages in the thread when scoring. + // This is best-effort: failures are swallowed so they never block rule execution. + const insertWithRetries = Tracer.traced( + { + resource: 'SubmitContent', + operation: 'ItemInvestigationService.insertItem', + }, + withRetries( + { + maxRetries: 1, + initialTimeMsBetweenRetries: 50, + maxTimeMsBetweenRetries: 200, + }, + ItemInvestigationService.insertItem.bind(ItemInvestigationService), + ), + ); + try { + await insertWithRetries({ + requestId, + orgId, + itemSubmission: { + ...itemSubmissionOrError.itemSubmission, + submissionTime: + itemSubmissionOrError.itemSubmission.submissionTime ?? new Date(), + }, + }); + } catch { + // Swallow insert errors; rules can still run without thread context. + } + // Run rules try { const results = await RuleEngine.runEnabledRules( @@ -328,11 +360,32 @@ function rawContentSubmissionToItemSubmission( | { itemSubmission: ItemSubmission; error?: undefined } { const submissionTime = new Date(); + // Coerce a string threadId to { id, typeId } when the content type has a + // threadId schema field role. This allows callers to pass a plain string ID + // for the parent thread without knowing the internal identifier format. + const raw = rawContentSubmission.content; + const threadIdField = + 'threadId' in itemType.schemaFieldRoles + ? (itemType.schemaFieldRoles as { threadId?: string }).threadId + : undefined; + const content: RawItemData = + threadIdField && + typeof raw[threadIdField] === 'string' && + (raw[threadIdField] as string).length > 0 + ? { + ...raw, + [threadIdField]: { + id: raw[threadIdField] as string, + typeId: itemType.id, + }, + } + : raw; + // Validate content JSON const normalizedDataOrValidationErrors = toNormalizedItemDataOrErrors( legalItemTypeIds, itemType, - rawContentSubmission.content, + content, ); return Array.isArray(normalizedDataOrValidationErrors) diff --git a/server/services/sentinelService/index.ts b/server/services/sentinelService/index.ts new file mode 100644 index 00000000..fc63acd0 --- /dev/null +++ b/server/services/sentinelService/index.ts @@ -0,0 +1,10 @@ +export { + makeSentinelService, + SentinelServiceError, + type SentinelBanksStatus, + type SentinelHealthResponse, + type SentinelLoadBanksRequest, + type SentinelScoreRequest, + type SentinelScoreResponse, + type SentinelService, +} from './sentinelService.js'; diff --git a/server/services/sentinelService/sentinelService.test.ts b/server/services/sentinelService/sentinelService.test.ts new file mode 100644 index 00000000..5f653b3a --- /dev/null +++ b/server/services/sentinelService/sentinelService.test.ts @@ -0,0 +1,208 @@ +import { type FetchHTTP } from '../networkingService/index.js'; +import { + makeSentinelService, + SentinelServiceError, + type SentinelBanksStatus, + type SentinelHealthResponse, + type SentinelScoreResponse, +} from './sentinelService.js'; + +function makeOkResponse(body: T) { + return { + ok: true, + status: 200, + headers: new Headers(), + body, + }; +} + +function makeErrorResponse(status: number, body: unknown = undefined) { + return { + ok: false, + status, + headers: new Headers(), + body, + }; +} + +function makeFetchHTTPMock() { + const mock = jest.fn(); + return { fetchHTTP: mock as unknown as FetchHTTP, mock }; +} + +describe('makeSentinelService', () => { + const BASE_URL = 'http://localhost:8000'; + + describe('healthCheck', () => { + it('calls /health endpoint and returns response', async () => { + const { fetchHTTP, mock } = makeFetchHTTPMock(); + const mockResponse: SentinelHealthResponse = { + status: 'ok', + banks_loaded: true, + }; + mock.mockResolvedValueOnce(makeOkResponse(mockResponse)); + + const service = makeSentinelService(fetchHTTP, BASE_URL); + const result = await service.healthCheck(); + + expect(mock).toHaveBeenCalledWith( + expect.objectContaining({ url: `${BASE_URL}/health`, method: 'get' }), + ); + expect(result).toEqual(mockResponse); + }); + + it('throws SentinelServiceError on non-ok response', async () => { + const { fetchHTTP, mock } = makeFetchHTTPMock(); + mock.mockResolvedValueOnce(makeErrorResponse(503)); + + const service = makeSentinelService(fetchHTTP, BASE_URL); + await expect(service.healthCheck()).rejects.toBeInstanceOf( + SentinelServiceError, + ); + }); + + it('wraps fetch errors in SentinelServiceError', async () => { + const { fetchHTTP, mock } = makeFetchHTTPMock(); + mock.mockRejectedValueOnce(new Error('ECONNREFUSED')); + + const service = makeSentinelService(fetchHTTP, BASE_URL); + const error = await service.healthCheck().catch((e) => e); + expect(error).toBeInstanceOf(SentinelServiceError); + expect(error.message).toContain('ECONNREFUSED'); + }); + }); + + describe('getBanksStatus', () => { + it('calls /banks/status and returns loaded status', async () => { + const { fetchHTTP, mock } = makeFetchHTTPMock(); + const mockStatus: SentinelBanksStatus = { + loaded: true, + model_name: 'all-MiniLM-L6-v2', + positive_count: 100, + negative_count: 200, + }; + mock.mockResolvedValueOnce(makeOkResponse(mockStatus)); + + const service = makeSentinelService(fetchHTTP, BASE_URL); + const result = await service.getBanksStatus(); + + expect(mock).toHaveBeenCalledWith( + expect.objectContaining({ + url: `${BASE_URL}/banks/status`, + method: 'get', + }), + ); + expect(result).toEqual(mockStatus); + }); + }); + + describe('scoreTexts', () => { + it('calls /score with the provided texts and returns scores', async () => { + const { fetchHTTP, mock } = makeFetchHTTPMock(); + const mockScore: SentinelScoreResponse = { + rare_class_affinity_score: 0.82, + observation_scores: { 'hello world': 0.82 }, + num_observations: 1, + }; + mock.mockResolvedValueOnce(makeOkResponse(mockScore)); + + const service = makeSentinelService(fetchHTTP, BASE_URL); + const result = await service.scoreTexts({ texts: ['hello world'] }); + + expect(mock).toHaveBeenCalledWith( + expect.objectContaining({ + url: `${BASE_URL}/score`, + method: 'post', + body: expect.stringContaining('hello world'), + }), + ); + expect(result.rare_class_affinity_score).toBe(0.82); + }); + + it('includes default top_k and min_score_to_consider when not specified', async () => { + const { fetchHTTP, mock } = makeFetchHTTPMock(); + const mockScore: SentinelScoreResponse = { + rare_class_affinity_score: 0.5, + observation_scores: { text: 0.5 }, + num_observations: 1, + }; + mock.mockResolvedValueOnce(makeOkResponse(mockScore)); + + const service = makeSentinelService(fetchHTTP, BASE_URL); + await service.scoreTexts({ texts: ['text'] }); + + const [call] = mock.mock.calls[0] as [{ body: string }]; + expect(call.body).toContain('"top_k":5'); + expect(call.body).toContain('"min_score_to_consider":0'); + }); + }); + + describe('scoreSingleText', () => { + it('returns the score for the single text from observation_scores', async () => { + const { fetchHTTP, mock } = makeFetchHTTPMock(); + const mockScore: SentinelScoreResponse = { + rare_class_affinity_score: 0.75, + observation_scores: { 'test text': 0.75 }, + num_observations: 1, + }; + mock.mockResolvedValueOnce(makeOkResponse(mockScore)); + + const service = makeSentinelService(fetchHTTP, BASE_URL); + const score = await service.scoreSingleText('test text'); + expect(score).toBe(0.75); + }); + + it('returns 0 when observation_scores is empty', async () => { + const { fetchHTTP, mock } = makeFetchHTTPMock(); + const mockScore: SentinelScoreResponse = { + rare_class_affinity_score: 0, + observation_scores: {}, + num_observations: 0, + }; + mock.mockResolvedValueOnce(makeOkResponse(mockScore)); + + const service = makeSentinelService(fetchHTTP, BASE_URL); + const score = await service.scoreSingleText('test text'); + expect(score).toBe(0); + }); + }); + + describe('default URL', () => { + it('uses localhost:8000 when no URL is provided', async () => { + const { fetchHTTP, mock } = makeFetchHTTPMock(); + const mockResponse: SentinelHealthResponse = { + status: 'ok', + banks_loaded: true, + }; + mock.mockResolvedValueOnce(makeOkResponse(mockResponse)); + + const service = makeSentinelService(fetchHTTP); + await service.healthCheck(); + + expect(mock).toHaveBeenCalledWith( + expect.objectContaining({ url: 'http://localhost:8000/health' }), + ); + }); + + it('strips trailing slash from base URL', async () => { + const { fetchHTTP, mock } = makeFetchHTTPMock(); + const mockResponse: SentinelHealthResponse = { + status: 'ok', + banks_loaded: true, + }; + mock.mockResolvedValueOnce(makeOkResponse(mockResponse)); + + const service = makeSentinelService( + fetchHTTP, + 'http://sentinel.example.com/', + ); + await service.healthCheck(); + + expect(mock).toHaveBeenCalledWith( + expect.objectContaining({ + url: 'http://sentinel.example.com/health', + }), + ); + }); + }); +}); diff --git a/server/services/sentinelService/sentinelService.ts b/server/services/sentinelService/sentinelService.ts new file mode 100644 index 00000000..de58d5b4 --- /dev/null +++ b/server/services/sentinelService/sentinelService.ts @@ -0,0 +1,167 @@ +/** + * Sentinel Service — HTTP client for the Sentinel API. + * + * Sentinel is a contrastive-learning system for detecting rare text patterns + * (e.g. grooming, hate speech) by comparing text against positive (rare/harmful) + * and negative (common/normal) example banks. + * + * The factory does NOT read process.env directly — the caller passes in the + * URL, keeping the module testable and DI-friendly. + */ +import { jsonStringify } from '../../utils/encoding.js'; +import { type FetchHTTP } from '../networkingService/index.js'; + +export type SentinelScoreRequest = { + texts: string[]; + top_k?: number; + min_score_to_consider?: number; +}; + +export type SentinelScoreResponse = { + rare_class_affinity_score: number; + observation_scores: { [text: string]: number }; + num_observations: number; +}; + +export type SentinelBanksStatus = { + loaded: boolean; + model_name?: string; + positive_count?: number; + negative_count?: number; + model_card?: Record; +}; + +export type SentinelHealthResponse = { + status: string; + banks_loaded: boolean; + version?: string; +}; + +export type SentinelLoadBanksRequest = { + path: string; + negative_to_positive_ratio?: number; +}; + +export class SentinelServiceError extends Error { + public readonly statusCode?: number; + + constructor(message: string, statusCode?: number, cause?: Error) { + super(message, { cause }); + this.name = 'SentinelServiceError'; + this.statusCode = statusCode; + } +} + +export interface SentinelService { + /** Score an array of texts for rare class affinity. */ + scoreTexts(request: SentinelScoreRequest): Promise; + + /** + * Score a single text for rare class affinity. + * Returns the individual observation score for that text. + */ + scoreSingleText(text: string): Promise; + + /** Get the current banks status. */ + getBanksStatus(): Promise; + + /** Check if the Sentinel service is healthy. */ + healthCheck(): Promise; + + /** Load banks from a given path. */ + loadBanks(request: SentinelLoadBanksRequest): Promise; + + /** Unload current banks from memory. */ + unloadBanks(): Promise; +} + +const DEFAULT_BASE_URL = 'http://localhost:8000'; + +export function makeSentinelService( + fetchHTTP: FetchHTTP, + apiUrl?: string, +): SentinelService { + const baseUrl = (apiUrl ?? DEFAULT_BASE_URL).replace(/\/$/, ''); + + async function fetchWithError( + endpoint: string, + options: { method: 'get' | 'post'; body?: string } = { method: 'get' }, + ): Promise { + try { + const response = await fetchHTTP({ + url: `${baseUrl}${endpoint}`, + method: options.method, + headers: { 'Content-Type': 'application/json' }, + body: options.body, + handleResponseBody: 'as-json', + timeoutMs: 10_000, + }); + + if (!response.ok) { + throw new SentinelServiceError( + `Sentinel API error: ${jsonStringify(response.body)}`, + response.status, + ); + } + + return response.body as T; + } catch (error) { + if (error instanceof SentinelServiceError) { + throw error; + } + throw new SentinelServiceError( + `Failed to connect to Sentinel service: ${error instanceof Error ? error.message : 'Unknown error'}`, + undefined, + error instanceof Error ? error : undefined, + ); + } + } + + return { + async scoreTexts( + request: SentinelScoreRequest, + ): Promise { + return fetchWithError('/score', { + method: 'post', + body: jsonStringify({ + texts: request.texts, + top_k: request.top_k ?? 5, + min_score_to_consider: request.min_score_to_consider ?? 0.0, + }), + }); + }, + + async scoreSingleText(text: string): Promise { + const response = await this.scoreTexts({ texts: [text] }); + const scores = Object.values(response.observation_scores); + return scores.length > 0 ? (scores[0] ?? 0) : 0; + }, + + async getBanksStatus(): Promise { + return fetchWithError('/banks/status', { + method: 'get', + }); + }, + + async healthCheck(): Promise { + return fetchWithError('/health', { + method: 'get', + }); + }, + + async loadBanks(request: SentinelLoadBanksRequest): Promise { + await fetchWithError('/banks/load', { + method: 'post', + body: jsonStringify(request), + }); + }, + + async unloadBanks(): Promise { + await fetchWithError('/banks/unload', { + method: 'post', + }); + }, + }; +} + +export default makeSentinelService; diff --git a/server/services/signalsService/SignalsService.ts b/server/services/signalsService/SignalsService.ts index cdc5348b..af021a01 100644 --- a/server/services/signalsService/SignalsService.ts +++ b/server/services/signalsService/SignalsService.ts @@ -128,6 +128,7 @@ export class SignalsService { signalAuthService: Dependencies['SignalAuthService'], getUserScoreEventuallyConsistent: Dependencies['getUserScoreEventuallyConsistent'], fetchHTTP: Dependencies['fetchHTTP'], + itemInvestigationService: Dependencies['ItemInvestigationService'], ) { const cachedCredentialGetters = makeCachedCredentialGetters(signalAuthService); @@ -142,6 +143,8 @@ export class SignalsService { this.userStrikeService, this.getPoliciesByIdEventuallyConsistent, this.hmaService, + itemInvestigationService, + fetchHTTP, ); const pluginEntries = getIntegrationRegistry().getPluginEntries(); @@ -371,6 +374,7 @@ export default inject( 'SignalAuthService', 'getUserScoreEventuallyConsistent', 'fetchHTTP', + 'ItemInvestigationService', ], SignalsService, ); diff --git a/server/services/signalsService/helpers/instantiateBuiltInSignals.ts b/server/services/signalsService/helpers/instantiateBuiltInSignals.ts index d7a67280..8143f962 100644 --- a/server/services/signalsService/helpers/instantiateBuiltInSignals.ts +++ b/server/services/signalsService/helpers/instantiateBuiltInSignals.ts @@ -2,7 +2,10 @@ import { type ItemIdentifier } from '@roostorg/coop-types'; import type { AggregationsService } from '../../aggregationsService/index.js'; import type { HmaService } from '../../hmaService/index.js'; +import type { ItemInvestigationService } from '../../itemInvestigationService/index.js'; import type { GetPoliciesByIdEventuallyConsistent } from '../../manualReviewToolService/manualReviewToolQueries.js'; +import { type FetchHTTP } from '../../networkingService/index.js'; +import { makeSentinelService } from '../../sentinelService/index.js'; import { type UserScore } from '../../userStatisticsService/userStatisticsService.js'; import { type UserStrikeService } from '../../userStrikeService/index.js'; import AggregationSignal from '../signals/aggregation/AggregationSignal.js'; @@ -35,6 +38,7 @@ import OpenAiSexualTextSignal from '../signals/third_party_signals/open_ai/moder import OpenAiViolenceImageSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiViolenceImageSignal.js'; import OpenAiViolenceTextSignal from '../signals/third_party_signals/open_ai/moderation/OpenAiViolenceTextSignal.js'; import OpenAiWhisperTranscriptionSignal from '../signals/third_party_signals/open_ai/whisper/OpenAiWhisperTranscriptionSignal.js'; +import SentinelRareClassAffinitySignal from '../signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.js'; import ZentropiLabelerSignal from '../signals/third_party_signals/zentropi/ZentropiLabelerSignal.js'; import UserScoreSignal from '../signals/UserScoreSignal.js'; import UserStrikesSignal from '../signals/UserStrikesSignal.js'; @@ -53,6 +57,8 @@ export function instantiateBuiltInSignals( userStrikeService: UserStrikeService, _getPoliciesByIdEventuallyConsistent: GetPoliciesByIdEventuallyConsistent, hmaService: HmaService, + itemInvestigationService: ItemInvestigationService, + fetchHTTP: FetchHTTP, ) { const { googleContentSafetyFetcher: getGoogleContentSafetyScores, @@ -146,6 +152,11 @@ export function instantiateBuiltInSignals( [SignalType.GOOGLE_CLOUD_TRANSLATE_MODEL]: new GoogleCloudTranslationAPISignal(), [SignalType.AGGREGATION]: new AggregationSignal(aggregationsService), + [SignalType.SENTINEL_RARE_CLASS_AFFINITY]: + new SentinelRareClassAffinitySignal( + makeSentinelService(fetchHTTP, process.env.SENTINEL_API_URL), + itemInvestigationService, + ), [SignalType.ZENTROPI_LABELER]: new ZentropiLabelerSignal( credentialGetters.ZENTROPI, getZentropiScores, diff --git a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts new file mode 100644 index 00000000..6f12aa44 --- /dev/null +++ b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts @@ -0,0 +1,369 @@ +import { ScalarTypes } from '@roostorg/types'; + +import { + SentinelServiceError, + type SentinelService, +} from '../../../../sentinelService/sentinelService.js'; +import { Integration } from '../../../types/Integration.js'; +import { SignalPricingStructure } from '../../../types/SignalPricingStructure.js'; +import { SignalType } from '../../../types/SignalType.js'; +import { type SignalInput } from '../../SignalBase.js'; +import SentinelRareClassAffinitySignal from './SentinelRareClassAffinitySignal.js'; + +type SentinelSignalInput = SignalInput< + ScalarTypes['STRING'], + false, + false, + string, + 'SENTINEL_RARE_CLASS_AFFINITY' +>; + +function makeInput( + overrides: Partial = {}, +): SentinelSignalInput { + return { + value: { type: 'STRING', value: 'test content' }, + matchingValues: undefined, + actionPenalties: undefined, + orgId: 'org-1', + ...overrides, + } as unknown as SentinelSignalInput; +} + +function makeSentinelService( + overrides: Partial = {}, +): SentinelService { + return { + healthCheck: jest + .fn() + .mockResolvedValue({ status: 'ok', banks_loaded: true }), + getBanksStatus: jest.fn().mockResolvedValue({ loaded: true }), + scoreTexts: jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.5, + observation_scores: { 'test content': 0.5 }, + num_observations: 1, + }), + scoreSingleText: jest.fn().mockResolvedValue(0.5), + loadBanks: jest.fn().mockResolvedValue(undefined), + unloadBanks: jest.fn().mockResolvedValue(undefined), + ...overrides, + }; +} + +type MockItemInvestigationService = { + insertItem: jest.Mock; + getThreadSubmissionsByPosition: jest.Mock; + getThreadSubmissionsByTime: jest.Mock; + getItemByIdentifier: jest.Mock; + getItemByTypeAgnosticIdentifier: jest.Mock; + getAncestorItems: jest.Mock; + getItemSubmissionsByCreator: jest.Mock; + getItemActionHistory: jest.Mock; +}; + +function makeItemInvestigationService( + overrides: Partial = {}, +): MockItemInvestigationService { + return { + insertItem: jest.fn().mockResolvedValue(undefined), + getThreadSubmissionsByPosition: jest.fn(), + getThreadSubmissionsByTime: jest.fn().mockReturnValue( + (async function* () { + // Empty async iterable by default + })(), + ), + getItemByIdentifier: jest.fn(), + getItemByTypeAgnosticIdentifier: jest.fn(), + getAncestorItems: jest.fn(), + getItemSubmissionsByCreator: jest.fn(), + getItemActionHistory: jest.fn(), + ...overrides, + }; +} + +function makeSignal( + sentinelService?: Partial, + iisOverrides?: Partial, +) { + return new SentinelRareClassAffinitySignal( + makeSentinelService(sentinelService), + makeItemInvestigationService(iisOverrides) as any, + ); +} + +describe('SentinelRareClassAffinitySignal', () => { + describe('signal metadata', () => { + it('returns correct id', () => { + expect(makeSignal().id).toEqual({ + type: SignalType.SENTINEL_RARE_CLASS_AFFINITY, + }); + }); + + it('returns correct integration', () => { + expect(makeSignal().integration).toBe(Integration.SENTINEL); + }); + + it('returns STRING as eligible input type', () => { + expect(makeSignal().eligibleInputs).toEqual([ScalarTypes.STRING]); + }); + + it('returns NUMBER as output type', () => { + expect(makeSignal().outputType).toEqual({ + scalarType: ScalarTypes.NUMBER, + }); + }); + + it('returns FREE pricing structure', () => { + expect(makeSignal().pricingStructure).toBe(SignalPricingStructure.FREE); + }); + + it('is allowed in automated rules', () => { + expect(makeSignal().allowedInAutomatedRules).toBe(true); + }); + + it('does not need matching values', () => { + expect(makeSignal().needsMatchingValues).toBe(false); + }); + + it('does not need action penalties', () => { + expect(makeSignal().needsActionPenalties).toBe(false); + }); + + it('has no eligible subcategories', () => { + expect(makeSignal().eligibleSubcategories).toEqual([]); + }); + + it('returns ALL as supported languages', () => { + expect(makeSignal().supportedLanguages).toBe('ALL'); + }); + }); + + describe('getDisabledInfo', () => { + it('returns disabled=false when service is healthy and banks are loaded', async () => { + const info = await makeSignal().getDisabledInfo('org-1'); + expect(info.disabled).toBe(false); + }); + + it('returns disabled=true when health check fails', async () => { + const signal = makeSignal({ + healthCheck: jest + .fn() + .mockRejectedValue(new Error('Connection refused')), + }); + const info = await signal.getDisabledInfo('org-1'); + expect(info.disabled).toBe(true); + expect(info.disabledMessage).toContain('unavailable'); + }); + + it('returns disabled=true when health status is not ok', async () => { + const signal = makeSignal({ + healthCheck: jest + .fn() + .mockResolvedValue({ status: 'error', banks_loaded: false }), + }); + const info = await signal.getDisabledInfo('org-1'); + expect(info.disabled).toBe(true); + expect(info.disabledMessage).toContain('not healthy'); + }); + + it('returns disabled=true when banks are not loaded', async () => { + const signal = makeSignal({ + getBanksStatus: jest.fn().mockResolvedValue({ loaded: false }), + }); + const info = await signal.getDisabledInfo('org-1'); + expect(info.disabled).toBe(true); + expect(info.disabledMessage).toContain('banks are not loaded'); + }); + }); + + describe('run', () => { + it('scores the primary text and returns rare_class_affinity_score', async () => { + const scoreTexts = jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.72, + observation_scores: { 'test content': 0.72 }, + num_observations: 1, + }); + const signal = makeSignal({ scoreTexts }); + + const result = await signal.run(makeInput()); + + expect(scoreTexts).toHaveBeenCalledWith( + expect.objectContaining({ + texts: expect.arrayContaining(['test content']), + }), + ); + expect(result).toMatchObject({ + outputType: { scalarType: ScalarTypes.NUMBER }, + score: 0.72, + }); + }); + + it('includes thread context texts when threadIdentifier is provided', async () => { + const scoreTexts = jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.85, + observation_scores: {}, + num_observations: 2, + }); + + // Async iterable that yields one prior thread item + const threadItem = { + latestSubmission: { + data: { text: 'prior thread message' }, + itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, + }, + priorSubmissions: undefined, + parents: (async function* () {})(), + }; + const getThreadSubmissionsByTime = jest.fn().mockReturnValue( + (async function* () { + yield threadItem; + })(), + ); + + const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + + await signal.run( + makeInput({ + runtimeArgs: { + threadIdentifier: { id: 'thread-1', typeId: 'content-type-1' }, + contentTextFieldName: 'text', + }, + }), + ); + + const texts = scoreTexts.mock.calls[0][0].texts; + expect(texts).toContain('test content'); + expect(texts).toContain('prior thread message'); + }); + + it('does not double-count the triggering submission when it is echoed back as thread context', async () => { + // submitContent.ts writes the current submission to Scylla before + // running rules, so getThreadSubmissionsByTime (which is time-bounded, + // not identity-bounded) can return the very submission that triggered + // this signal run alongside genuine prior messages. + const scoreTexts = jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.5, + observation_scores: {}, + num_observations: 2, + }); + + const priorItem = { + latestSubmission: { + data: { text: 'prior thread message' }, + itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, + }, + priorSubmissions: undefined, + parents: (async function* () {})(), + }; + const selfItem = { + latestSubmission: { + data: { text: 'test content' }, + itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, + }, + priorSubmissions: undefined, + parents: (async function* () {})(), + }; + const getThreadSubmissionsByTime = jest.fn().mockReturnValue( + (async function* () { + yield selfItem; + yield priorItem; + })(), + ); + + const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + + await signal.run( + makeInput({ + runtimeArgs: { + threadIdentifier: { id: 'thread-1', typeId: 'content-type-1' }, + contentTextFieldName: 'text', + }, + }), + ); + + const texts = scoreTexts.mock.calls[0][0].texts; + expect(texts).toEqual(['test content', 'prior thread message']); + }); + + it('only drops one occurrence of duplicate text, in case a genuine duplicate message exists', async () => { + const scoreTexts = jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.5, + observation_scores: {}, + num_observations: 3, + }); + + const makeItem = (text: string) => ({ + latestSubmission: { + data: { text }, + itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, + }, + priorSubmissions: undefined, + parents: (async function* () {})(), + }); + const getThreadSubmissionsByTime = jest.fn().mockReturnValue( + (async function* () { + yield makeItem('test content'); + yield makeItem('test content'); + })(), + ); + + const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + + await signal.run( + makeInput({ + runtimeArgs: { + threadIdentifier: { id: 'thread-1', typeId: 'content-type-1' }, + contentTextFieldName: 'text', + }, + }), + ); + + const texts = scoreTexts.mock.calls[0][0].texts; + expect(texts).toEqual(['test content', 'test content']); + }); + + it('still returns a score when thread fetch fails', async () => { + const scoreTexts = jest.fn().mockResolvedValue({ + rare_class_affinity_score: 0.5, + observation_scores: { 'test content': 0.5 }, + num_observations: 1, + }); + const getThreadSubmissionsByTime = jest.fn().mockImplementation(() => { + throw new Error('Scylla unavailable'); + }); + + const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + + const result = await signal.run( + makeInput({ + runtimeArgs: { + threadIdentifier: { id: 'thread-1', typeId: 'content-type-1' }, + }, + }), + ); + + // Should still score with just the primary text + expect(result).toMatchObject({ score: 0.5 }); + expect(scoreTexts).toHaveBeenCalledWith( + expect.objectContaining({ texts: ['test content'] }), + ); + }); + + it('returns an error result when Sentinel service throws SentinelServiceError', async () => { + const signal = makeSignal({ + scoreTexts: jest + .fn() + .mockRejectedValue(new SentinelServiceError('Banks not loaded', 503)), + }); + const result = await signal.run(makeInput()); + expect(result.type).toBe('ERROR'); + }); + + it('re-throws non-SentinelServiceError exceptions', async () => { + const signal = makeSignal({ + scoreTexts: jest.fn().mockRejectedValue(new Error('Unexpected error')), + }); + await expect(signal.run(makeInput())).rejects.toThrow('Unexpected error'); + }); + }); +}); diff --git a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts new file mode 100644 index 00000000..070879ba --- /dev/null +++ b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts @@ -0,0 +1,232 @@ +import { ScalarTypes } from '@roostorg/types'; + +import { makeSignalPermanentError } from '../../../../../utils/errors.js'; +import { type ItemInvestigationService } from '../../../../itemInvestigationService/index.js'; +import { type ItemSubmission } from '../../../../itemProcessingService/index.js'; +import { + SentinelServiceError, + type SentinelService, +} from '../../../../sentinelService/index.js'; +import { SignalPricingStructure } from '../../../types/SignalPricingStructure.js'; +import { SignalType } from '../../../types/SignalType.js'; +import SignalBase, { type SignalInput } from '../../SignalBase.js'; + +const SENTINEL_DOCS_URL = 'https://github.com/UMass-Rescue/Sentinel'; + +/** + * How many prior thread items to include in Sentinel scoring. + * More context improves recall for pattern detection but adds latency. + */ +const DEFAULT_THREAD_CONTEXT_LIMIT = 10; + +export default class SentinelRareClassAffinitySignal extends SignalBase< + ScalarTypes['STRING'], + { scalarType: ScalarTypes['NUMBER'] } +> { + constructor( + private readonly sentinelService: SentinelService, + private readonly itemInvestigationService: ItemInvestigationService, + ) { + super(); + } + + override get id() { + return { type: SignalType.SENTINEL_RARE_CLASS_AFFINITY }; + } + + override get displayName() { + return 'Sentinel Rare Class Affinity'; + } + + override get description() { + return `Sentinel is a contrastive-learning system that detects rare harmful text patterns (e.g., grooming) by measuring semantic affinity to curated example banks. + +It compares submitted content against labeled positive (rare/harmful) and negative (common/normal) examples to produce a score between 0 and 1. Higher scores indicate stronger affinity to the rare class. Thread context is included when available, improving detection of patterns that span multiple messages.`; + } + + override get docsUrl() { + return SENTINEL_DOCS_URL; + } + + override get recommendedThresholds() { + return null; + } + + override get supportedLanguages(): 'ALL' { + return 'ALL'; + } + + override get pricingStructure(): SignalPricingStructure { + return SignalPricingStructure.FREE; + } + + override get eligibleInputs() { + return [ScalarTypes.STRING]; + } + + override get outputType() { + return { scalarType: ScalarTypes.NUMBER }; + } + + override getCost() { + return 50; + } + + override get needsMatchingValues() { + return false; + } + + override get eligibleSubcategories(): [] { + return []; + } + + override get needsActionPenalties() { + return false; + } + + override get integration() { + return 'SENTINEL' as const; + } + + override get allowedInAutomatedRules() { + return true; + } + + override async getDisabledInfo(_orgId: string) { + try { + const health = await this.sentinelService.healthCheck(); + if (health.status !== 'ok' && health.status !== 'healthy') { + return { + disabled: true as const, + disabledMessage: + 'Sentinel service is not healthy. Please ensure the Sentinel server is running and banks are loaded.', + }; + } + + const banksStatus = await this.sentinelService.getBanksStatus(); + if (!banksStatus.loaded) { + return { + disabled: true as const, + disabledMessage: + 'Sentinel banks are not loaded. Please load the required model banks before using this signal.', + }; + } + + return { disabled: false as const }; + } catch { + return { + disabled: true as const, + disabledMessage: + 'Sentinel service is unavailable. Please ensure the Sentinel server is running and reachable.', + }; + } + } + + async run( + input: SignalInput< + ScalarTypes['STRING'], + false, + false, + string, + 'SENTINEL_RARE_CLASS_AFFINITY' + >, + ) { + const { value, orgId, runtimeArgs } = input; + + // The primary text to score is always the signal input value. + const primaryText = String(value.value); + const texts: string[] = [primaryText]; + + // If thread context is available, fetch prior messages in the thread and + // include their text in the scoring request. This provides contextual + // signals that help Sentinel detect patterns spanning multiple messages. + if (runtimeArgs?.threadIdentifier) { + try { + const threadItems = + this.itemInvestigationService.getThreadSubmissionsByTime({ + orgId, + threadId: runtimeArgs.threadIdentifier, + limit: DEFAULT_THREAD_CONTEXT_LIMIT, + }); + + // `submitContent.ts` writes the current submission to the thread's + // history *before* rules run (so thread-aware signals have context + // available), and `getThreadSubmissionsByTime` is bounded by time, + // not by item identity. That means the submission which triggered + // this very signal run is indistinguishable from real "prior" + // messages and comes back in `threadItems` too. Drop (at most) one + // occurrence matching `primaryText` so it isn't double-counted, + // which would otherwise skew the aggregate score Sentinel computes. + let skippedSelf = false; + for await (const { latestSubmission } of threadItems) { + const itemText = extractTextFromSubmission( + latestSubmission, + runtimeArgs.contentTextFieldName, + ); + if (itemText == null) { + continue; + } + if (!skippedSelf && itemText === primaryText) { + skippedSelf = true; + continue; + } + texts.push(itemText); + } + } catch { + // Thread context is best-effort. If it fails, we continue with just + // the primary text rather than failing the entire signal execution. + } + } + + try { + const response = await this.sentinelService.scoreTexts({ texts }); + return { + outputType: { scalarType: ScalarTypes.NUMBER }, + score: response.rare_class_affinity_score, + }; + } catch (error) { + if (error instanceof SentinelServiceError) { + return { + type: 'ERROR' as const, + score: makeSignalPermanentError('Sentinel scoring failed', { + detail: error.message, + shouldErrorSpan: false, + }), + }; + } + throw error; + } + } +} + +/** + * Extract the text to score from an ItemSubmission. + * + * If a specific `contentTextFieldName` is given (derived from the condition's + * input field), we look up that field by name. Otherwise, we collect the first + * string-valued field we can find. Returns `undefined` if no suitable text is + * found. + */ +function extractTextFromSubmission( + submission: ItemSubmission, + contentTextFieldName?: string, +): string | undefined { + const data = submission.data as Record; + + if (contentTextFieldName != null) { + const rawValue = data[contentTextFieldName]; + if (typeof rawValue === 'string' && rawValue.trim()) { + return rawValue; + } + return undefined; + } + + // Fall back: return the first string field value in the item's data. + for (const value of Object.values(data)) { + if (typeof value === 'string' && value.trim()) { + return value; + } + } + + return undefined; +} diff --git a/server/services/signalsService/types/Integration.ts b/server/services/signalsService/types/Integration.ts index 77e9a158..58dfb5ca 100644 --- a/server/services/signalsService/types/Integration.ts +++ b/server/services/signalsService/types/Integration.ts @@ -7,6 +7,7 @@ import { makeEnumLike } from '@roostorg/coop-types'; export const Integration = makeEnumLike([ 'GOOGLE_CONTENT_SAFETY_API', 'OPEN_AI', + 'SENTINEL', 'ZENTROPI', ]); diff --git a/server/services/signalsService/types/SignalArgsByType.ts b/server/services/signalsService/types/SignalArgsByType.ts index 44c0b0d0..88b530c6 100644 --- a/server/services/signalsService/types/SignalArgsByType.ts +++ b/server/services/signalsService/types/SignalArgsByType.ts @@ -1,3 +1,5 @@ +import { type ItemIdentifier } from '@roostorg/types'; + import { type Satisfies } from '../../../utils/typescript-types.js'; import type { AggregationClause, @@ -35,6 +37,7 @@ export type SignalArgsByType = Satisfies< [SignalType.OPEN_AI_SEXUAL_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_VIOLENCE_IMAGE_MODEL]: undefined; [SignalType.OPEN_AI_VIOLENCE_TEXT_MODEL]: undefined; + [SignalType.SENTINEL_RARE_CLASS_AFFINITY]: undefined; [SignalType.ZENTROPI_LABELER]: undefined; [SignalType.CUSTOM]: undefined; }, @@ -73,6 +76,18 @@ export type RuntimeSignalArgsByType = Satisfies< [SignalType.OPEN_AI_SEXUAL_TEXT_MODEL]: undefined; [SignalType.OPEN_AI_VIOLENCE_IMAGE_MODEL]: undefined; [SignalType.OPEN_AI_VIOLENCE_TEXT_MODEL]: undefined; + [SignalType.SENTINEL_RARE_CLASS_AFFINITY]: { + /** + * The identifier of the thread/context item that this content belongs to. + * Used by the Sentinel signal to include thread-level text in scoring. + */ + threadIdentifier?: ItemIdentifier; + /** + * The name of the content field whose text value should be scored. + * Derived from the condition's input field name. + */ + contentTextFieldName?: string; + }; [SignalType.ZENTROPI_LABELER]: undefined; [SignalType.CUSTOM]: undefined; }, diff --git a/server/services/signalsService/types/SignalType.ts b/server/services/signalsService/types/SignalType.ts index 2c3c8074..877d1937 100644 --- a/server/services/signalsService/types/SignalType.ts +++ b/server/services/signalsService/types/SignalType.ts @@ -50,6 +50,7 @@ export const BuiltInThirdPartySignalType = makeEnumLike([ 'OPEN_AI_SEXUAL_TEXT_MODEL', 'OPEN_AI_VIOLENCE_IMAGE_MODEL', 'OPEN_AI_VIOLENCE_TEXT_MODEL', + 'SENTINEL_RARE_CLASS_AFFINITY', 'ZENTROPI_LABELER', ]); @@ -106,6 +107,8 @@ export function integrationForSignalType(type: SignalType | string) { case 'OPEN_AI_VIOLENCE_TEXT_MODEL': case 'OPEN_AI_WHISPER_TRANSCRIPTION': return Integration.OPEN_AI; + case 'SENTINEL_RARE_CLASS_AFFINITY': + return Integration.SENTINEL; case 'ZENTROPI_LABELER': return Integration.ZENTROPI; case 'AGGREGATION':