confidence

Author: ahsd-coder

README.md

@@ -1,11 +1,11 @@
-# 训练模型
+# 一.训练模型
 
 ```commandline
 python3 training_code/train_tfidf.py
 python3 training_code/train_bert.py
 ```
 
-# 压测服务
+# 二.压测服务
 
 ```commandline
 cd test/
@@ -15,7 +15,7 @@ ab -n 100 -c 100 -p data.json -T 'application/json' -H 'accept: application/json
 ab -n 100 -c 100 -p data.json -T 'application/json' -H 'accept: application/json' 'http://0.0.0.0:8000/v1/text-cls/bert'
 ```
 
-# 接口
+# 三.接口
 
 ```commandline
 curl -X 'POST' \
@@ -28,21 +28,48 @@ curl -X 'POST' \
 }'
 ```
 
-# 部署
+# 四.部署
 
-## 只有后端时启动
+## (1)只有后端时启动
 ```commandline
 fastapi run main.py
 ```
-## 后端+前端启动
+## (2)后端+前端启动
 ```commandline
 uvicorn main:app --host 0.0.0.0 --port 8000
 解释：Uvicorn是一个ASGI服务器，负责接收HTTP请求并转发给FastAPI应用处理。FastAPI本身只是一个框架，不负责网络监听；uvicorn才是真正"跑起来"的那个进程
 main:app → main是Python模块名，对应main.py文件；app是该模块里的变量，即main.py里的app = FastAPI()
 uvicorn会先 import main,然后拿到main.py里的FastAPI实例，开始监听请求
 ```
 
-# 推广
+# 五.推广
 ```commandline
 可以重新找一个公开数据集（比如在魔搭社区的数据集），或直接标注一个文本分类数据集（推荐3个以上的类别个数），复现加载bert base 模型在新数据集上的微调过程。最终输入一个新的样本进行测试，验证分类效果是否准确。
 ```
+
+
+# 六.意图识别方式
+```commandline
+本项目介绍了意图识别常用的三个方法：
+1.关键词匹配。写规则。好处是简单快，坏处是说法稍微变一下就抓不到了，并且容易误命中
+2.模型分类。用一个专门训练过的语言理解模型做分类（本项目有TFIDF/BERT）
+3.大模型判断：直接把用户输入喂给大语言模型，让它判断意图，不需要训练，但速度慢、成本高，适合长尾场景或刚起步阶段。
+
+对于一个高频场景，通常的选择是：模型分类做主力，关键词规则兜底，大模型处理没见过的长尾说法。三个分开用，别指望一个方案全搞定。
+```
+
+# 七.置信度
+```commandline
+置信度就是模型对自己判断结果的把握程度，范围0到1。1是完全确定，0是完全不知道
+  ┌────────────────────┬────────────────────┬────────────┐
+  │        路由        │     置信度来源     │   值范围     │
+  ├────────────────────┼────────────────────┼────────────┤
+  │ /v1/text-cls/bert  │ softmax 概率       │ 0 ~ 1       │
+  ├────────────────────┼────────────────────┼────────────┤
+  │ /v1/text-cls/tfidf │ predict_proba 概率 │ 0 ~ 1       │
+  ├────────────────────┼────────────────────┼────────────┤
+  │ /v1/text-cls/regex │ 固定值             │ 1.0 或 0.0  │
+  ├────────────────────┼────────────────────┼────────────┤
+  │ /v1/text-cls/gpt   │ 固定值             │ 1.0         │
+  └────────────────────┴────────────────────┴────────────┘
+```

data_schema.py

@@ -20,5 +20,6 @@ class TextClassifyResponse(BaseModel):
     request_id: Optional[str] = Field(..., description="请求id")
     request_text: Union[str, List[str]] = Field(..., description="请求文本、字符串或列表")
     classify_result: Union[str, List[str]] = Field(..., description="分类结果")
+    classify_confidence: Union[float, List[float], None] = Field(None, description="分类置信度")
     classify_time: float = Field(..., description="分类耗时")
     error_msg: str = Field(..., description="异常信息")

main.py

@@ -32,16 +32,18 @@ def regex_classify(req: TextClassifyRequest) -> TextClassifyResponse:
         request_id=req.request_id,
         request_text=req.request_text,
         classify_result="",
+        classify_confidence=None,
         classify_time=0,
         error_msg=""
     )
 
     # logger.info(f"{req.request_id} {req.request_text}")  # 打印请求
     try:
-        response.classify_result = model_for_regex(req.request_text)
+        response.classify_result, response.classify_confidence = model_for_regex(req.request_text)
         response.error_msg = "ok"
     except Exception as err:
         response.classify_result = ""
+        response.classify_confidence = None
         response.error_msg = traceback.format_exc() # traceback.format_exec()可将异常调用结果返回
 
     response.classify_time = round(time.time() - start_time, 3)
@@ -60,16 +62,18 @@ def tfidf_classify(req: TextClassifyRequest) -> TextClassifyResponse:
         request_id=req.request_id,
         request_text=req.request_text,
         classify_result="",
+        classify_confidence=None,
         classify_time=0,
         error_msg=""
     )
     # logger.info(f"Get requst: {req.json()}")
 
     try:
-        response.classify_result = model_for_tfidf(req.request_text)
+        response.classify_result, response.classify_confidence = model_for_tfidf(req.request_text)
         response.error_msg = "ok"
     except Exception as err:
         response.classify_result = ""
+        response.classify_confidence = None
         response.error_msg = traceback.format_exc()
 
     response.classify_time = round(time.time() - start_time, 3)
@@ -89,16 +93,18 @@ def bert_classify(req: TextClassifyRequest) -> TextClassifyResponse:
         request_id=req.request_id,
         request_text=req.request_text,
         classify_result="",
+        classify_confidence=None,
         classify_time=0,
         error_msg=""
     )
     # info 日志
     try:
-        response.classify_result = model_for_bert(req.request_text)
+        response.classify_result, response.classify_confidence = model_for_bert(req.request_text)
         response.error_msg = "ok"
     except Exception as err:
         # error 日志
         response.classify_result = ""
+        response.classify_confidence = None
         response.error_msg = traceback.format_exc()
 
     response.classify_time = round(time.time() - start_time, 3)
@@ -117,15 +123,17 @@ def gpt_classify(req: TextClassifyRequest) -> TextClassifyResponse:
         request_id=req.request_id,
         request_text=req.request_text,
         classify_result="",
+        classify_confidence=None,
         classify_time=0,
         error_msg=""
     )
 
     try:
-        response.classify_result = model_for_gpt(req.request_text)
+        response.classify_result, response.classify_confidence = model_for_gpt(req.request_text)
         response.error_msg = "ok"
     except Exception as err:
         response.classify_result = ""
+        response.classify_confidence = None
         response.error_msg = traceback.format_exc()
 
     response.classify_time = round(time.time() - start_time, 3)

model/bert.py

@@ -1,4 +1,4 @@
-from typing import Union, List
+from typing import Union, List, Tuple
 
 import numpy as np
 import pandas as pd
@@ -32,8 +32,8 @@ def __len__(self):
         return len(self.labels)
 
 
-def model_for_bert(request_text: Union[str, List[str]]) -> Union[str, List[str]]:
-    classify_result: Union[str, List[str]] = None
+def model_for_bert(request_text: Union[str, List[str]]) -> Tuple[List[str], List[float]]:
+    classify_result: List[str] = None
 
     if isinstance(request_text, str):
         request_text = [request_text]
@@ -50,6 +50,7 @@ def model_for_bert(request_text: Union[str, List[str]]) -> Union[str, List[str]]
 
     model.eval()
     pred = []
+    confidence = []
     for batch in test_dataloader:
         with torch.no_grad(): # 关闭梯度计算，节省显存，加速推理
             input_ids = batch['input_ids'].to(device)
@@ -58,10 +59,12 @@ def model_for_bert(request_text: Union[str, List[str]]) -> Union[str, List[str]]
             outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
         logits = outputs[1]
         logits = logits.detach().cpu().numpy()
-        pred += list(np.argmax(logits, axis=1).flatten()) # +=是为多batch准备的，保持代码通用性，不管几条数据都能处理
+        probs = np.exp(logits) / np.exp(logits).sum(axis=1, keepdims=True)  # softmax
+        pred += list(np.argmax(probs, axis=1).flatten())
+        confidence += list(np.max(probs, axis=1).flatten())
 
     classify_result = [CATEGORY_NAME[x] for x in pred]
-    return classify_result
+    return classify_result, confidence
 
 '''
 以用户输入 request_text = "放个游戏视频看看" 为例，逐行追踪：

model/prompt.py

@@ -1,4 +1,4 @@
-from typing import Union, List
+from typing import Union, List, Tuple
 
 import openai
 import pandas as pd
@@ -36,8 +36,8 @@
 
 
 
-def model_for_gpt(request_text: Union[str, List[str]]) -> List[str]:
-    classify_result: Union[str, List[str]] = []
+def model_for_gpt(request_text: Union[str, List[str]]) -> Tuple[List[str], List[float]]:
+    classify_result: List[str] = []
 
     if isinstance(request_text, str):
         tfidf_feat = tfidf.transform([request_text]) # 一个文本
@@ -72,7 +72,8 @@ def model_for_gpt(request_text: Union[str, List[str]]) -> List[str]:
 
         classify_result.append(response.choices[0].message.content)
 
-    return classify_result
+    classify_confidence = [1.0] * len(classify_result)
+    return classify_result, classify_confidence
 
 
 # 算法算法项目测试

model/regex_rule.py

@@ -1,5 +1,5 @@
 import re
-from typing import Union, List
+from typing import Union, List, Tuple
 
 from config import REGEX_RULE
 
@@ -8,27 +8,34 @@
     REGEX_RULE_COMPILED[category] = re.compile("|".join(REGEX_RULE[category]))
 
 
-def model_for_regex(request_text: Union[str, List[str]]) -> Union[str, List[str]]:
-    classify_result: Union[str, List[str]] = []
+def model_for_regex(request_text: Union[str, List[str]]) -> Tuple[List[str], List[float]]:
+    classify_result: List[str] = []
+    classify_confidence: List[float] = []
 
     if isinstance(request_text, str):
         for category in REGEX_RULE_COMPILED.keys():
             if REGEX_RULE_COMPILED[category].findall(request_text):
                 classify_result.append(category)
+                classify_confidence.append(1.0)
         if not classify_result:
             classify_result.append("Other")
+            classify_confidence.append(0.0)
     elif isinstance(request_text, list):
         classify_result = []
+        classify_confidence = []
         for text in request_text:
             is_classified = False
             for category in REGEX_RULE_COMPILED.keys():
-                if REGEX_RULE_COMPILED[category].findall(request_text):
+                if REGEX_RULE_COMPILED[category].findall(text):
                     classify_result.append(category)
+                    classify_confidence.append(1.0)
                     is_classified = True
+                    break
 
             if not is_classified:
                 classify_result.append("Other")
+                classify_confidence.append(0.0)
     else:
         raise Exception("格式不支持")
 
-    return classify_result
+    return classify_result, classify_confidence

model/tfidf_ml.py

@@ -1,6 +1,7 @@
-from typing import Union, List
+from typing import Union, List, Tuple
 
 import jieba
+import numpy as np
 import pandas as pd
 from joblib import load
 from config import TFIDF_MODEL_PKL_PATH
@@ -12,20 +13,25 @@
 cn_stopwords = pd.read_csv('http://mirror.coggle.club/stopwords/baidu_stopwords.txt', header=None)[0].values
 
 
-def model_for_tfidf(request_text: Union[str, List[str]]) -> Union[str, List[str]]:
-    classify_result: Union[str, List[str]] = None
+def model_for_tfidf(request_text: Union[str, List[str]]) -> Tuple[List[str], List[float]]:
+    classify_result: List[str] = None
+    classify_confidence: List[float] = None
 
     if isinstance(request_text, str):
         query_words = " ".join([x for x in jieba.lcut(request_text) if x not in cn_stopwords])
-        classify_result = list(model.predict(tfidf.transform([query_words])))
+        proba = model.predict_proba(tfidf.transform([query_words]))
+        classify_result = list(model.classes_[np.argmax(proba, axis=1)])
+        classify_confidence = list(np.max(proba, axis=1).flatten())
     elif isinstance(request_text, list):
         query_words = []
         for text in request_text:
             query_words.append(
                 " ".join([x for x in jieba.lcut(text) if x not in cn_stopwords])
             )
-            classify_result = list(model.predict(tfidf.transform(query_words)))
+        proba = model.predict_proba(tfidf.transform(query_words))
+        classify_result = list(model.classes_[np.argmax(proba, axis=1)])
+        classify_confidence = list(np.max(proba, axis=1).flatten())
     else:
         raise Exception("格式不支持")
 
-    return classify_result
+    return classify_result, classify_confidence