Split "matched_questions" into "keywords" and "questions" (#24)

* Split "matched_questions" to separate data better

* Update code for "matched_questions" split

* Update docker images

* Update GitHub Actions workflows

* Fix missing questions field

* Filter special characters from dictionary matching

* Add keywords to dictionary matching

Author: ButterscotchV

.github/workflows/codeql.yml

@@ -32,7 +32,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'csharp', 'javascript' ]
+        language: [ 'csharp', 'javascript', 'python' ]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
         # Use only 'java' to analyze code written in Java, Kotlin or both
         # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both

.github/workflows/frontend.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
 
       - name: Setup Pages
         uses: actions/configure-pages@v5

BingusApi/config/faq_config.json

@@ -14,7 +14,10 @@ public record FaqConfigEntry
             [JsonPropertyName("answer")]
             public string Answer { get; set; } = "";
 
-            [JsonPropertyName("matched_questions")]
+            [JsonPropertyName("keywords")]
+            public string[] Keywords { get; set; } = [];
+
+            [JsonPropertyName("questions")]
             public string[] Questions { get; set; } = [];
 
             public FaqConfigEntry() { }
@@ -25,9 +28,14 @@ public FaqConfigEntry(string answer)
                 Answer = answer;
             }
 
-            public FaqConfigEntry(string answer, IEnumerable<string> questions)
+            public FaqConfigEntry(
+                string answer,
+                IEnumerable<string> keywords,
+                IEnumerable<string> questions
+            )
                 : this(answer)
             {
+                Keywords = keywords.ToArray();
                 Questions = questions.ToArray();
             }
         }
@@ -50,6 +58,11 @@ public FaqConfigEntry(string answer, IEnumerable<string> questions)
         {
             foreach (var entry in FaqEntries)
             {
+                foreach (var keyword in entry.Keywords)
+                {
+                    yield return (entry.Title, keyword, entry.Answer);
+                }
+
                 foreach (var question in entry.Questions)
                 {
                     yield return (entry.Title, question, entry.Answer);

BingusLib/FaqHandling/FaqConfig.cs

@@ -1,8 +1,9 @@
+using System.Text.RegularExpressions;
 using static BingusLib.FaqHandling.FaqConfig;
 
 namespace BingusLib.FaqHandling
 {
-    public class FaqDict
+    public partial class FaqDict
     {
         private readonly Dictionary<string, FaqEntry> _faqDict = [];
 
@@ -20,6 +21,16 @@ public FaqDict(IEnumerable<FaqConfigEntry> faqConfigEntries)
                     Answer = entry.Answer,
                 };
 
+                foreach (var keyword in entry.Keywords)
+                {
+                    _faqDict[CleanQuery(keyword)] = new FaqEntry()
+                    {
+                        Title = entry.Title,
+                        Question = keyword,
+                        Answer = entry.Answer,
+                    };
+                }
+
                 foreach (var question in entry.Questions)
                 {
                     _faqDict[CleanQuery(question)] = new FaqEntry()
@@ -32,11 +43,15 @@ public FaqDict(IEnumerable<FaqConfigEntry> faqConfigEntries)
             }
         }
 
-        private static string CleanQuery(string query) => query.Trim().ToLowerInvariant();
+        private static string CleanQuery(string query) =>
+            QueryFilterRegex().Replace(query.ToLowerInvariant(), "");
 
         public FaqEntry? Search(string query)
         {
             return _faqDict.TryGetValue(CleanQuery(query), out var entry) ? entry : null;
         }
+
+        [GeneratedRegex("[^A-Za-z]")]
+        private static partial Regex QueryFilterRegex();
     }
 }

BingusLib/FaqHandling/FaqDict.cs

@@ -9,7 +9,7 @@ import { EmbedList, fetchBingus, fetchBingusData } from "../util.js";
 
 async function getFaqConfig() {
   return (await fetchBingusData()).faqs.flatMap((x) =>
-    x.matched_questions.filter((x) => x.length > 0 && x.length <= 100),
+    x.keywords.filter((x) => x.length > 0 && x.length <= 100),
   );
 }
 

bingus-bot/src/commands/ask.ts

@@ -168,11 +168,11 @@ export class EmbedList {
 }
 
 export interface FaqConfig {
-  average_questions: boolean;
   faqs: {
     title: string;
     answer: string;
-    matched_questions: string[];
+    keywords: string[];
+    questions: string[];
   }[];
 }
 

bingus-bot/src/util.ts

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.14-slim
 
 WORKDIR /usr/src/app
 

bingus-python-encoder/Dockerfile

@@ -69,7 +69,8 @@ def random_typo(str_err: StrErrer, random: Random) -> StrErrer:
 class FaqEntry(BaseModel):
     title: str | None
     answer: str
-    matched_questions: list[str]
+    keywords: list[str]
+    questions: list[str]
 
 
 class FaqConfig(BaseModel):
@@ -102,21 +103,21 @@ def iterate_answers(self):
 
     def iterate_questions(self):
         for faq in self.faqs:
-            for question in faq.matched_questions:
+            for question in faq.questions:
                 yield question
 
     def question_count(self):
-        return sum((len(faq.matched_questions) for faq in self.faqs))
+        return sum((len(faq.questions) for faq in self.faqs))
 
     def filter_short_questions(self, min_words: int):
         """
         Filters out questions shorter than min_words and removes empty entries.
         """
         for faq in self.faqs:
-            faq.matched_questions = [
-                q for q in faq.matched_questions if len(q.split()) >= min_words]
+            faq.questions = [
+                q for q in faq.questions if len(q.split()) >= min_words]
         self.faqs = [faq for faq in self.faqs if len(
-            faq.matched_questions) > 0]
+            faq.questions) > 0]
 
     def make_typos(
             self,
@@ -149,7 +150,7 @@ def make_typos(
         for faq in self.faqs:
             new_qs: list[str] = []
 
-            for question in faq.matched_questions:
+            for question in faq.questions:
                 q_min_typos = min_typos
                 q_max_typos = max_typos
                 if scale_max_per_word:
@@ -168,7 +169,7 @@ def make_typos(
                     new_qs.append(typo_q.result)
                     typo_count += num_typos
 
-            faq.matched_questions.extend(new_qs)
+            faq.questions.extend(new_qs)
             typo_entry_count += len(new_qs)
 
         return typo_entry_count, typo_count
@@ -178,7 +179,7 @@ def make_question_pairs(self) -> Dataset:
         Makes question-to-question pairs from the FAQs, where each question is paired with all
         other questions in its set (positive samples) and from other sets (negative sample).
         """
-        return make_entry_pairs([faq.matched_questions for faq in self.faqs])
+        return make_entry_pairs([faq.questions for faq in self.faqs])
 
     def make_question_answer_pairs(self) -> Dataset:
         """
@@ -188,7 +189,7 @@ def make_question_answer_pairs(self) -> Dataset:
         questions, answers, scores = [], [], []
 
         for faq in self.faqs:
-            for question in faq.matched_questions:
+            for question in faq.questions:
                 # Positive sample (correct answer)
                 questions.append(question)
                 answers.append(faq.answer)
@@ -212,7 +213,7 @@ def make_everything_pairs(self) -> Dataset:
         Makes pairs of titles, answers, and questions from the FAQs, where each set is paired with its correct
         answer (positive sample) and other incorrect answers (negative samples).
         """
-        return make_entry_pairs([[faq.title, faq.answer, *faq.matched_questions] for faq in self.faqs])
+        return make_entry_pairs([[faq.title, faq.answer, *faq.questions] for faq in self.faqs])
 
 
 def make_wiki_qa_dataset(faqs: FaqConfig, max_count: int = -1) -> Dataset:

bingus-python-encoder/data_utils.py

@@ -1,4 +1,4 @@
-FROM node:20-slim AS base
+FROM node:22-slim AS base
 
 COPY ./bingus-bot/ /app/bingus-bot/
 COPY ./package*.json /app/