v1.1 (#5)

Major rework to add new detection types. 

* Closes #2 
* Closes #4

Author: fsvreddit

README.md

@@ -1,10 +1,18 @@
-A Dev Platform app for detecting AI content in images.
+A Dev Platform app for detecting AI or other unwanted content in images.
 
-This app uses the Sightengine API to analyse image posts for AI content.
+This app uses the Sightengine API to analyse image posts. Detections supported:
+
+* AI-generated images
+* Deepfake images
+* Minors in images
+* Poor quality (blurry) images
+* Offensive content (Nazi, white supremacist or terrorist imagery)
+* Spammy QR codes or text on an image
+* Drug imagery
 
 To use, you need to sign up to [Sightengine's platform](https://sightengine.com/) and obtain an API user ID and key. You must set the API User in the app settings, and the API key via the subreddit context menu.
 
-For many use cases, Sightengine's free tier will be adequate, permitting 100 AI detections per day up to 400/month. For higher usage needs, Sightengine offer paid plans.
+For many use cases, Sightengine's free tier will be adequate, permitting 500 "operations" per day up to 2,000/month. AI and Deepfake checks use 5 "operations" each, while other checks use 1 each. For higher usage needs, Sightengine offer paid plans.
 
 ## Checking images
 
@@ -26,6 +34,10 @@ If an image is detected as AI, a report like this will be made:
 
 ## Change History
 
+### v1.1
+
+* Add additional detection types
+
 ### v1.0
 
 * Initial Release
@@ -34,4 +46,4 @@ If an image is detected as AI, a report like this will be made:
 
 This app is open source under the BSD 3-Clause licence. The source code can be found [here](https://github.com/fsvreddit/image-moderator).
 
-Interested in detections for things other than AI? Get in touch by messaging /u/fsv! The list of possibilities can be seen in Sightengine's documentation [here](https://sightengine.com/docs/models).
+Interested in detections for things not yet supported? Get in touch by messaging /u/fsv! The list of possibilities can be seen in Sightengine's documentation [here](https://sightengine.com/docs/models). It would be really useful if you could outline the kind of content you're looking to avoid on your subreddit.

devvit.yaml

@@ -1,2 +1 @@
 name: image-moderator
-version: 1.0.0

package-lock.json

@@ -4,11 +4,17 @@
   "version": "0.0.0",
   "license": "BSD-3-Clause",
   "type": "module",
+  "scripts": {
+    "test": "npm run test:unit",
+    "test:unit": "vitest --run"
+  },
   "dependencies": {
-    "@devvit/protos": "^0.11.15",
-    "@devvit/public-api": "0.11.15",
-    "@devvit/shared-types": "^0.11.15",
+    "@devvit/protos": "0.11.17",
+    "@devvit/public-api": "0.11.17",
+    "@devvit/shared-types": "0.11.17",
+    "@types/lodash": "^4.17.18",
     "@types/luxon": "^3.6.2",
+    "lodash": "^4.17.21",
     "luxon": "^3.6.1"
   },
   "devDependencies": {

package.json

@@ -0,0 +1,45 @@
+import { MenuItemOnPressEvent, Context } from "@devvit/public-api";
+import { isLinkId } from "@devvit/shared-types/tid.js";
+import { getSightengineResults } from "./checkSightEngineAPI.js";
+import { getModels, getRelevantDetectors } from "./detections/allDetections.js";
+
+export async function checkPostFromMenu (event: MenuItemOnPressEvent, context: Context) {
+    const postId = event.targetId;
+    if (!postId || !isLinkId(postId)) {
+        context.ui.showToast("Could not get post ID from event.");
+        return;
+    }
+
+    const post = await context.reddit.getPostById(postId);
+    const settings = await context.settings.getAll();
+    const detectors = getRelevantDetectors(settings, "menu");
+
+    if (detectors.length === 0) {
+        context.ui.showToast("No detection models are enabled for this subreddit.");
+        console.log("No detection models are enabled for this subreddit.");
+        return;
+    }
+
+    const models = getModels(detectors);
+
+    const result = await getSightengineResults(post, models, context);
+    if (result.status === "error") {
+        context.ui.showToast(result.message ?? "Error checking post for AI content.");
+        console.error(`Menu: Error checking post ${postId}: ${result.message}`);
+        return;
+    }
+
+    console.log(JSON.stringify(result, null, 2));
+
+    const detectionResults: string[] = [];
+    for (const Detection of detectors) {
+        const detectionInstance = new Detection(settings);
+        const detectionResult = detectionInstance.detectByMenu(result);
+        if (detectionResult) {
+            detectionResults.push(detectionResult);
+        }
+    }
+
+    context.ui.showToast(detectionResults.join(", "));
+    console.log(`Active Check: Post ${postId} results ${detectionResults.join(", ")}.`);
+}

src/checkAIContentMenu.ts

@@ -5,6 +5,7 @@ import { postIsImage } from "./utility.js";
 import { AppSetting } from "./settings.js";
 import { DateTime } from "luxon";
 import { userIsModerator } from "./moderatorChecks.js";
+import { getModels, getRelevantDetectors } from "./detections/allDetections.js";
 
 function getFilterKeyForPost (postId: string) {
     return `filtered_post:${postId}`;
@@ -72,27 +73,36 @@ async function checkAndReportPost (postId: string, source: "PostCreate" | "PostA
         console.log(`${source}: User ${user.username} is a moderator. Skipping AI check.`);
     }
 
-    const result = await getSightengineResults(post, context);
-    if (result.status === "error") {
-        console.error(`PostCreate: Error checking post for AI content: ${result.message}`);
+    const detectors = getRelevantDetectors(settings, "menu");
+
+    if (detectors.length === 0) {
         return;
     }
 
-    if (!result.type?.ai_generated) {
-        console.log(`${source}: Post ${post.id} is not detected as AI content. Skipping report.`);
+    const models = getModels(detectors);
+
+    const result = await getSightengineResults(post, models, context);
+    if (result.status === "error") {
+        console.error(`PostCreate: Error checking post for AI content: ${result.message}`);
         return;
     }
 
-    const aiLikelihood = Math.round(result.type.ai_generated * 100);
+    const detectionResults: string[] = [];
+    for (const Detection of detectors) {
+        const detectionInstance = new Detection(settings);
+        const detectionResult = detectionInstance.detectByMenu(result);
+        if (detectionResult) {
+            detectionResults.push(detectionResult);
+        }
+    }
 
-    const thresholdToReport = settings[AppSetting.ThresholdToReport] as number | undefined;
-    if (thresholdToReport && aiLikelihood < thresholdToReport) {
-        console.log(`${source}: AI content likelihood for ${post.id} (${aiLikelihood}%) is below threshold (${thresholdToReport}%). Skipping report.`);
+    if (detectionResults.length === 0) {
+        console.log(`${source}: Post ${post.id} did not match any detectors. Skipping report.`);
         return;
     }
 
-    console.log(`${source}: Post ${post.id} is detected as AI content with likelihood ${aiLikelihood}%. Reporting post.`);
-    await context.reddit.report(post, { reason: `AI content likelihood: ${aiLikelihood}%` });
+    await context.reddit.report(post, { reason: detectionResults.join(", ") });
+    console.log(`${source}: Post ${post.id} matched: ${detectionResults.join(", ")}. Reported.`);
 }
 
 export async function handlePostCreate (event: PostCreate, context: TriggerContext) {
@@ -102,10 +112,6 @@ export async function handlePostCreate (event: PostCreate, context: TriggerConte
     }
 
     const settings = await context.settings.getAll();
-    if (!settings[AppSetting.AutoCheckEnabled]) {
-        console.log(`PostCreate: Auto check is disabled. Skipping AI check.`);
-        return;
-    }
 
     if (settings[AppSetting.CheckAfterApproval] && event.post.spam) {
         console.log(`PostCreate: Post ${event.post.id} is removed or filtered. Skipping AI check.`);
@@ -132,11 +138,6 @@ export async function handlePostApprovalAction (event: ModAction, context: Trigg
     }
 
     const settings = await context.settings.getAll();
-    if (!settings[AppSetting.AutoCheckEnabled]) {
-        console.log(`PostApprovalAction: Auto check is disabled. Skipping AI check.`);
-        return;
-    }
-
     if (!settings[AppSetting.CheckAfterApproval]) {
         console.log(`PostApprovalAction: Check after approval is disabled. Skipping AI check.`);
         return;

src/checkFromMenu.ts

@@ -3,7 +3,7 @@ import { DateTime } from "luxon";
 import { getAPIUserAndKey } from "./apiKeyManagement.js";
 import { getImageURLFromPost } from "./utility.js";
 
-interface SightengineResponse {
+export interface SightengineResponse {
     status: string;
     message?: string;
     request?: {
@@ -27,28 +27,118 @@ interface SightengineResponse {
             imagen?: number;
             ideogram?: number;
         };
+        deepfake?: number;
+        illustration?: number;
+        photo?: number;
+    };
+    quality?: {
+        score?: number;
+    };
+    faces?: [
+        {
+            x1?: number;
+            y1?: number;
+            x2?: number;
+            y2?: number;
+            features?: {
+                left_eye?: {
+                    x: number;
+                    y: number;
+                };
+                right_eye?: {
+                    x: number;
+                    y: number;
+                };
+                nose_tip?: {
+                    x: number;
+                    y: number;
+                };
+                left_mouth_corner?: {
+                    x: number;
+                    y: number;
+                };
+                right_mouth_corner?: {
+                    x: number;
+                    y: number;
+                };
+            };
+            attributes?: {
+                minor?: number;
+                sunglasses?: number;
+            };
+        },
+    ];
+    offensive?: {
+        nazi?: number;
+        confederate?: number;
+        supremacist?: number;
+        terrorist?: number;
+    };
+    qr?: {
+        personal: [];
+        link: {
+            type: string;
+            match: string;
+            category?: string;
+        }[];
+        social: {
+            type: string;
+            match: string;
+            score?: number;
+        }[];
+        profanity: [];
+        spam: {
+            type: string;
+            match: string;
+        }[];
+    };
+    text?: {
+        spam: {
+            type: string;
+            match: string;
+        }[];
+    };
+    recreational_drug?: {
+        prob?: number;
+        classes?: {
+            cannabis: number;
+            cannabis_logo_only: number;
+            cannabis_plant: number;
+            cannabis_drug: number;
+            recreational_drugs_not_cannabis: number;
+        };
     };
 }
 
+interface SightengineResponseWrapped {
+    detections: string[];
+    sightengineResponse: SightengineResponse;
+}
+
 function getFailureResponse (message: string): SightengineResponse {
     return {
         status: "error",
         message,
     };
 }
 
-export async function getSightengineResults (post: Post, context: TriggerContext): Promise<SightengineResponse> {
+export async function getSightengineResults (post: Post, detections: string[], context: TriggerContext): Promise<SightengineResponse> {
     const url = getImageURLFromPost(post);
     if (!url) {
         return getFailureResponse("This does not appear to be an image post.");
     }
 
-    const cachedResultKey = `sightengine_ai_check_${post.id}`;
+    const cachedResultKey = `sightengine_check_${post.id}`;
     const cachedResult = await context.redis.get(cachedResultKey);
 
     if (cachedResult) {
-        console.log("Using cached result");
-        return JSON.parse(cachedResult) as SightengineResponse;
+        console.log("Found cached result for post:", post.id);
+
+        const cachedResponse = JSON.parse(cachedResult) as SightengineResponseWrapped;
+        if (detections.every(detection => cachedResponse.detections.includes(detection))) {
+            console.log("Using cached result");
+            return cachedResponse.sightengineResponse;
+        }
     }
 
     const apiDetails = await getAPIUserAndKey(context);
@@ -62,7 +152,7 @@ export async function getSightengineResults (post: Post, context: TriggerContext
 
     const params = new URLSearchParams();
     params.append("url", url);
-    params.append("models", "genai");
+    params.append("models", detections.join(","));
     params.append("api_user", apiDetails.apiUser);
     params.append("api_secret", apiDetails.apiKey);
 
@@ -80,7 +170,12 @@ export async function getSightengineResults (post: Post, context: TriggerContext
         return getFailureResponse("Error checking post for AI content.");
     }
 
-    await context.redis.set(cachedResultKey, JSON.stringify(result), { expiration: DateTime.now().plus({ months: 1 }).toJSDate() });
+    const wrappedResult: SightengineResponseWrapped = {
+        detections,
+        sightengineResponse: result,
+    };
+
+    await context.redis.set(cachedResultKey, JSON.stringify(wrappedResult), { expiration: DateTime.now().plus({ months: 1 }).toJSDate() });
 
     return result;
 }