Merge remote-tracking branch 'origin/develop_930' into develop_930

# Conflicts:
#	scripts/db/data-cleaning-init.sql

Author: hhhhsc701

scripts/db/data-cleaning-init.sql

@@ -56,7 +56,8 @@ CREATE TABLE IF NOT EXISTS t_clean_result
 INSERT IGNORE INTO t_clean_template(id, name, description)
 VALUES ('ac2f2582-a990-11f0-9768-00155d09c825', '空模板', '空模板'),
        ('fb6d0d76-a990-11f0-92db-00155d09c825', '测试模板', '测试模板'),
-       ('26ae585c-8310-4679-adc0-e53215e6e69b', 'text文本清洗模板', 'text文本清洗模板');
+       ('26ae585c-8310-4679-adc0-e53215e6e69b', 'text文本清洗模板', 'text文本清洗模板'),
+       ('4421504e-c6c9-4760-b55a-509d17429597', '图片清洗模板', '图片清洗模板');
 
 INSERT IGNORE INTO t_operator_instance(instance_id, operator_id, op_index, settings_override)
 VALUES ('fb6d0d76-a990-11f0-92db-00155d09c825', 'TextFormatter', 1, null),
@@ -88,4 +89,18 @@ VALUES ('fb6d0d76-a990-11f0-92db-00155d09c825', 'TextFormatter', 1, null),
        ('26ae585c-8310-4679-adc0-e53215e6e69b', 'AnonymizedIpAddress', 25, null),
        ('26ae585c-8310-4679-adc0-e53215e6e69b', 'AnonymizedIdNumber', 26, null),
        ('26ae585c-8310-4679-adc0-e53215e6e69b', 'AnonymizedUrlCleaner', 27, null),
-       ('26ae585c-8310-4679-adc0-e53215e6e69b', 'FileExporter', 28, null);
+       ('26ae585c-8310-4679-adc0-e53215e6e69b', 'FileExporter', 28, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgFormatter', 1, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgBlurredImagesCleaner', 2, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgDuplicatedImagesCleaner', 3, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgSimilarImagesCleaner', 4, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgBrightness', 5, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgContrast', 6, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgSaturation', 7, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgSharpness', 8, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgDenoise', 9, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgShadowRemove', 10, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgPerspectiveTransformation', 11, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgResize', 12, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'ImgTypeUnify', 13, null),
+       ('4421504e-c6c9-4760-b55a-509d17429597', 'FileExporter', 14, null);

scripts/db/data-operator-init.sql

@@ -94,7 +94,20 @@ VALUES ('TextFormatter', 'TXT文本抽取', '抽取TXT中的文本。', '1.0.0',
        ('TraditionalChineseCleaner', '繁体转简体', '将繁体转换为简体。', '1.0.0', 'text', 'text', null, null, '', 'false'),
        ('UnicodeSpaceCleaner', '空格标准化', '将文档中不同的 unicode 空格，如 u2008，转换为正常空格\\u0020。', '1.0.0', 'text', 'text', null, null, '', 'false'),
        ('AnonymizedUrlCleaner', 'URL网址匿名化', '将文档中的url网址匿名化。', '1.0.0', 'text', 'text', null, null, '', 'false'),
-       ('XMLTagCleaner', 'XML标签去除', '去除XML中的标签。', '1.0.0', 'text', 'text', null, null, '', 'false');
+       ('XMLTagCleaner', 'XML标签去除', '去除XML中的标签。', '1.0.0', 'text', 'text', null, null, '', 'false'),
+       ('ImgFormatter', '读取图片文件', '读取图片文件。', '1.0.0', 'image', 'image', null, null, '', 'false'),
+       ('ImgBlurredImagesCleaner', '模糊图片过滤', '去除模糊的图片。', '1.0.0', 'image', 'image', '{"blurredThreshold": {"name": "梯度函数值", "name_en": "Gradient Value", "description": "梯度函数值取值越小，图片模糊度越高。", "description_en": "A smaller gradient value indicates a higher image blur.", "type": "slider", "defaultVal": 1000, "min": 1, "max": 10000, "step": 1}}', null, '', 'false'),
+       ('ImgBrightness', '图片亮度增强', '自适应调节图片的亮度。', '1.0.0', 'image', 'image', null, null, '', 'false'),
+       ('ImgContrast', '图片对比度增强', '自适应调节图片的对比度。', '1.0.0', 'image', 'image', null, null, '', 'false'),
+       ('ImgDenoise', '图片噪点去除', '去除图片中的噪点，主要适用于自然场景。', '1.0.0', 'image', 'image', null, null, '', 'false'),
+       ('ImgDuplicatedImagesCleaner', '重复图片去除', '去除重复的图片。', '1.0.0', 'image', 'image', null, null, '', 'false'),
+       ('ImgPerspectiveTransformation', '图片透视变换', '自适应校正图片的视角，主要适用于文档校正场景。', '1.0.0', 'image', 'image', null, null, '', 'false'),
+       ('ImgResize', '图片重采样', '将图片放大或缩小到指定像素。', '1.0.0', 'image', 'image', '{"targetSize": {"name": "重采样尺寸", "name_en": "Resample Size", "type": "multiple", "properties": [{"type": "inputNumber", "name": "宽度", "name_en": "Width", "description": "像素", "description_en": "Pixel", "defaultVal": 256, "min": 1, "max": 4096, "step": 1}, {"type": "inputNumber", "name": "高度", "name_en": "Height", "description": "像素", "description_en": "Pixel", "defaultVal": 256, "min": 1, "max": 4096, "step": 1}]}}', null, '', 'false'),
+       ('ImgSaturation', '图片饱和度增强', '自适应调节图片的饱和度，主要适用于自然场景图片。', '1.0.0', 'image', 'image', null, null, '', 'false'),
+       ('ImgShadowRemove', '图片阴影去除', '去除图片中的阴影，主要适用于文档场景。', '1.0.0', 'image', 'image', null, null, '', 'false'),
+       ('ImgSharpness', '图片锐度增强', '自适应调节图片的锐度，主要适用于自然场景图片。', '1.0.0', 'image', 'image', null, null, '', 'false'),
+       ('ImgSimilarImagesCleaner', '相似图片去除', '去除相似的图片。', '1.0.0', 'image', 'image', '{"similarThreshold": {"name": "相似度", "name_en": "Similarity", "description": "相似度取值越大，图片相似度越高。", "description_en": "A larger similarity value indicates a higher image similarity.", "type": "slider", "defaultVal": 0.8, "min": 0, "max": 1, "step": 0.01}}', null, '', 'false'),
+       ('ImgTypeUnify', '图片格式转换', '将图片编码格式统一为jpg、jpeg、png、bmp格式。', '1.0.0', 'image', 'image', '{"imgType": {"name": "图片编码格式", "name_en": "Image Encoding Format", "type": "select", "defaultVal": "jpg", "options": [{"label": "jpg", "label_en": "jpg", "value": "jpg"}, {"label": "png", "label_en": "png", "value": "png"}, {"label": "jpeg", "label_en": "jpeg", "value": "jpeg"}, {"label": "bmp", "label_en": "bmp", "value": "bmp"}]}}', null, '', 'false');
 
 
 INSERT IGNORE INTO t_operator_category_relation(category_id, operator_id)
@@ -128,6 +141,19 @@ VALUES (3, 'TextFormatter'),
        (3, 'SexualAndViolentWordCleaner'),
        (3, 'TraditionalChineseCleaner'),
        (3, 'UnicodeSpaceCleaner'),
+       (4, 'ImgFormatter'),
+       (4, 'ImgBlurredImagesCleaner'),
+       (4, 'ImgBrightness'),
+       (4, 'ImgContrast'),
+       (4, 'ImgDenoise'),
+       (4, 'ImgDuplicatedImagesCleaner'),
+       (4, 'ImgPerspectiveTransformation'),
+       (4, 'ImgResize'),
+       (4, 'ImgSaturation'),
+       (4, 'ImgShadowRemove'),
+       (4, 'ImgSharpness'),
+       (4, 'ImgSimilarImagesCleaner'),
+       (4, 'ImgTypeUnify'),
        (8, 'FileWithShortOrLongLengthFilter'),
        (8, 'FileWithHighRepeatPhraseRateFilter'),
        (8, 'FileWithHighRepeatWordRateFilter'),
@@ -181,4 +207,17 @@ VALUES (3, 'TextFormatter'),
        (11, 'PoliticalWordCleaner'),
        (11, 'SexualAndViolentWordCleaner'),
        (11, 'TraditionalChineseCleaner'),
-       (11, 'UnicodeSpaceCleaner');
+       (11, 'UnicodeSpaceCleaner'),
+       (11, 'ImgFormatter'),
+       (11, 'ImgBlurredImagesCleaner'),
+       (11, 'ImgBrightness'),
+       (11, 'ImgContrast'),
+       (11, 'ImgDenoise'),
+       (11, 'ImgDuplicatedImagesCleaner'),
+       (11, 'ImgPerspectiveTransformation'),
+       (11, 'ImgResize'),
+       (11, 'ImgSaturation'),
+       (11, 'ImgShadowRemove'),
+       (11, 'ImgSharpness'),
+       (11, 'ImgSimilarImagesCleaner'),
+       (11, 'ImgTypeUnify');