Merge remote-tracking branch 'origin/develop_930' into develop_930

hefanli · hefanli · commit 97e599237aa6 · 2025-10-21T15:40:07.000+08:00
diff --git a/frontend/src/mock/mock-seed/data-cleansing.cjs b/frontend/src/mock/mock-seed/data-cleansing.cjs
@@ -10,6 +10,16 @@ function operatorItem() {
     inputs: Mock.Random.integer(1, 5),
     outputs: Mock.Random.integer(1, 5),
     settings: JSON.stringify({
+      fileLength: {
+        name: "文档字数",
+        description:
+          "过滤字数不在指定范围内的文档，如[10,10000000]。若输入为空，则不对字数上/下限做限制。",
+        type: "range",
+        defaultVal: [10, 10000000],
+        min: 0,
+        max: 10000000000000000,
+        step: 1,
+      },
       host: { type: "input", name: "主机地址", defaultVal: "localhost" },
       limit: {
         type: "range",
@@ -26,6 +36,12 @@ function operatorItem() {
         defaultVal: "utf-8",
         options: ["utf-8", "gbk", "ascii"],
       },
+      radio: {
+        type: "radio",
+        name: "radio",
+        defaultVal: "utf-8",
+        options: ["utf-8", "gbk", "ascii"],
+      },
       features: {
         type: "checkbox",
         name: "特征列",
diff --git a/frontend/src/pages/DataCleansing/Create/components/ParamConfig.tsx b/frontend/src/pages/DataCleansing/Create/components/ParamConfig.tsx
@@ -7,13 +7,14 @@ import {
   Form,
   InputNumber,
   Slider,
+  Space,
 } from "antd";
-import { OperatorI } from "@/pages/OperatorMarket/operator.model";
+import { ConfigI, OperatorI } from "@/pages/OperatorMarket/operator.model";
 
 interface ParamConfigProps {
   operator: OperatorI;
   paramKey: string;
-  param: any;
+  param: ConfigI;
   onParamChange?: (operatorId: string, paramKey: string, value: any) => void;
 }
 
@@ -23,15 +24,21 @@ const ParamConfig: React.FC<ParamConfigProps> = ({
   param,
   onParamChange,
 }) => {
+  if (!param) return null;
   const [value, setValue] = React.useState(param.value || param.defaultVal);
   const updateValue = (newValue: any) => {
     setValue(newValue);
     return onParamChange && onParamChange(operator.id, paramKey, newValue);
   };
+
   switch (param.type) {
     case "input":
       return (
-        <Form.Item label={param.name} tooltip={param.description} key={paramKey}>
+        <Form.Item
+          label={param.name}
+          tooltip={param.description}
+          key={paramKey}
+        >
           <Input
             value={value}
             onChange={(e) => updateValue(e.target.value)}
@@ -42,7 +49,11 @@ const ParamConfig: React.FC<ParamConfigProps> = ({
       );
     case "select":
       return (
-        <Form.Item label={param.name} tooltip={param.description} key={paramKey}>
+        <Form.Item
+          label={param.name}
+          tooltip={param.description}
+          key={paramKey}
+        >
           <Select
             value={value}
             onChange={updateValue}
@@ -58,7 +69,11 @@ const ParamConfig: React.FC<ParamConfigProps> = ({
       );
     case "radio":
       return (
-        <Form.Item label={param.name} tooltip={param.description} key={paramKey}>
+        <Form.Item
+          label={param.name}
+          tooltip={param.description}
+          key={paramKey}
+        >
           <Radio.Group
             value={value}
             onChange={(e) => updateValue(e.target.value)}
@@ -76,7 +91,11 @@ const ParamConfig: React.FC<ParamConfigProps> = ({
       );
     case "checkbox":
       return (
-        <Form.Item label={param.name} tooltip={param.description} key={paramKey}>
+        <Form.Item
+          label={param.name}
+          tooltip={param.description}
+          key={paramKey}
+        >
           <Checkbox.Group
             value={value}
             onChange={updateValue}
@@ -86,17 +105,22 @@ const ParamConfig: React.FC<ParamConfigProps> = ({
       );
     case "slider":
       return (
-        <Form.Item label={param.name} tooltip={param.description} key={paramKey}>
+        <Form.Item
+          label={param.name}
+          tooltip={param.description}
+          key={paramKey}
+        >
           <div className="flex items-center gap-1">
             <Slider
               value={value}
               onChange={updateValue}
               tooltip={{ open: true }}
               marks={{
-                [param.min || 0]: param.minLabel || `${param.min || 0}`,
-                [param.min + (param.max - param.min) / 2]:
-                  param.midLabel || `${(param.min + param.max) / 2}`,
-                [param.max || 100]: param.maxLabel || `${param.max || 100}`,
+                [param.min || 0]: `${param.min || 0}`,
+                [param.min + (param.max - param.min) / 2]: `${
+                  (param.min + param.max) / 2
+                }`,
+                [param.max || 100]: `${param.max || 100}`,
               }}
               min={param.min || 0}
               max={param.max || 100}
@@ -114,28 +138,94 @@ const ParamConfig: React.FC<ParamConfigProps> = ({
           </div>
         </Form.Item>
       );
-    case "range":
+    case "range": {
+      const min = param.min || 0;
+      const max = param.max || 100;
       return (
-        <Form.Item label={param.name} tooltip={param.description} key={paramKey}>
+        <Form.Item
+          label={param.name}
+          tooltip={param.description}
+          key={paramKey}
+        >
           <Slider
             value={Array.isArray(value) ? value : [value, value]}
             onChange={(val) =>
               updateValue(Array.isArray(val) ? val : [val, val])
             }
             range
-            marks={{
-              [param.min || 0]: param.minLabel || `${param.min || 0}`,
-              [param.min + (param.max - param.min) / 2]:
-                param.midLabel || `${(param.min + param.max) / 2}`,
-              [param.max || 100]: param.maxLabel || `${param.max || 100}`,
-            }}
-            min={param.min || 0}
-            max={param.max || 100}
+            min={min}
+            max={max}
             step={param.step || 1}
             className="w-full"
           />
+          <Space>
+            <InputNumber
+              min={min}
+              max={max}
+              value={value[0]}
+              onChange={(val1) => updateValue([val1, value[1]])}
+              changeOnWheel
+            />
+            ~
+            <InputNumber
+              min={min}
+              max={max}
+              value={value[1]}
+              onChange={(val2) => updateValue([value[0], val2])}
+              changeOnWheel
+            />
+          </Space>
         </Form.Item>
       );
+    }
+    case "inputNumber":
+      return (
+        <Form.Item
+          label={param.name}
+          tooltip={param.description}
+          key={paramKey}
+        >
+          <InputNumber
+            value={value}
+            onChange={(val) => updateValue(val)}
+            placeholder={`请输入${param.name}`}
+            className="w-full"
+            min={param.min}
+            max={param.max}
+            step={param.step || 1}
+          />
+        </Form.Item>
+      );
+
+    case "switch":
+      return (
+        <Form.Item
+          label={param.name}
+          tooltip={param.description}
+          key={paramKey}
+        >
+          <Checkbox
+            checked={value as boolean}
+            onChange={(e) => updateValue(e.target.checked)}
+          >
+            {param.name}
+          </Checkbox>
+        </Form.Item>
+      );
+    case "multiple":
+      return (
+        <div className="pl-4 border-l border-gray-300">
+          {param.properties.map((subParam) => (
+            <Config
+              key={subParam.key}
+              operator={operator}
+              paramKey={subParam.key}
+              param={subParam}
+              onParamChange={onParamChange}
+            />
+          ))}
+        </div>
+      );
     default:
       return null;
   }
diff --git a/frontend/src/pages/DataCleansing/Create/hooks/useOperatorOperations.ts b/frontend/src/pages/DataCleansing/Create/hooks/useOperatorOperations.ts
@@ -106,8 +106,6 @@ export function useOperatorOperations() {
     paramKey: string,
     value: any
   ) => {
-    console.log(operatorId, paramKey, value);
-
     setSelectedOperators((prev) =>
       prev.map((op) =>
         op.id === operatorId
diff --git a/frontend/src/pages/OperatorMarket/operator.model.ts b/frontend/src/pages/OperatorMarket/operator.model.ts
@@ -1,3 +1,26 @@
+export interface ConfigI {
+  type:
+    | "input"
+    | "select"
+    | "radio"
+    | "checkbox"
+    | "range"
+    | "slider"
+    | "inputNumber"
+    | "switch"
+    | "multiple";
+  value?: number | string | boolean | string[] | number[];
+  required?: boolean;
+  description?: string;
+  key: string;
+  defaultVal: number | string | boolean | string[];
+  options?: string[] | { label: string; value: string }[];
+  min?: number;
+  max?: number;
+  step?: number;
+  properties?: ConfigI[]; // 用于嵌套配置
+}
+
 export interface OperatorI {
   id: string;
   name: string;
@@ -12,15 +35,7 @@ export interface OperatorI {
   overrides?: { [key: string]: any }; // 用户配置的参数
   defaultParams?: { [key: string]: any }; // 默认参数
   configs: {
-    [key: string]: {
-      type: "input" | "select" | "radio" | "checkbox" | "range";
-      label: string;
-      value: any;
-      options?: string[] | { label: string; value: any }[];
-      min?: number;
-      max?: number;
-      step?: number;
-    };
+    [key: string]: ConfigI;
   };
 }
 
diff --git a/scripts/db/data-engine-operator-init.sql b/scripts/db/data-engine-operator-init.sql
@@ -74,7 +74,7 @@ VALUES ('TextFormatter', 'TXT文本抽取', '抽取TXT中的文本。', '1.0.0',
        ('FileWithHighSpecialCharRateFilter', '文档特殊字符率检查', '去除特殊字符过多的文档。', '1.0.0', 'text', 'text', null, '{"specialCharRatio": {"name": "文档特殊字符率", "description": "特殊字符的统计数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.3, "min": 0, "max": 1, "step": 0.1}}', '', 'false'),
        ('DuplicateFilesFilter', '相似文档去除', '相似文档去除。', '1.0.0', 'text', 'text', null, '{"fileDuplicateThreshold": {"name": "文档相似度", "description": "基于MinHash算法和Jaccard相似度，计算当前文档与数据集中其它文档相似性，超过设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}}', '', 'false'),
        ('FileWithManySensitiveWordsFilter', '文档敏感词率检查', '去除敏感词过多的文档。', '1.0.0', 'text', 'text', null, '{"sensitiveWordsRate": {"name": "文档敏感词率", "description": "敏感词的字数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.01, "min": 0, "max": 1, "step": 0.01}}', '', 'false'),
-       ('FileWithShortOrLongLengthFilter', '文档字数检查', '字数不在指定范围会被过滤掉。', '1.0.0', 'text', 'text', null, '{"fileLength": {"name": "文档字数", "description": "过滤字数不在指定范围内的文档，如[10,10000000]。若输入为空，则不对字数上/下限做限制。", "type": "range", "properties": [{"name": "fileMinimumLength", "type": "inputNumber", "defaultVal": 10, "min": 0, "max": 10000000000000000, "step": 1}, {"name": "fileMaximumLength", "type": "inputNumber", "defaultVal": 10000000, "min": 0, "max": 10000000000000000, "step": 1}]}}', '', 'false'),
+       ('FileWithShortOrLongLengthFilter', '文档字数检查', '字数不在指定范围会被过滤掉。', '1.0.0', 'text', 'text', null, '{"fileLength": {"name": "文档字数", "description": "过滤字数不在指定范围内的文档，如[10,10000000]。若输入为空，则不对字数上/下限做限制。", "type": "range", "defaultVal": [10, 10000000], "min": 0, "max": 10000000000000000, "step": 1}}', '', 'false'),
        ('ContentCleaner', '文档目录去除', '去除文档中的目录。', '1.0.0', 'text', 'text', null, null, '', 'false'),
        ('AnonymizedCreditCardNumber', '信用卡号匿名化', '信用卡号匿名化', '1.0.0', 'text', 'text', null, null, '', 'false'),
        ('EmailNumberCleaner', '邮件地址匿名化', '邮件地址匿名化', '1.0.0', 'text', 'text', null, null, '', 'false'),
