Skip to content

Commit 6e5d5ca

Browse files
baisui1981baisui1981
committed
add ontology for TIS
1 parent 64d4a57 commit 6e5d5ca

260 files changed

Lines changed: 10531 additions & 10 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@
6868
<module>tis-transformer</module>
6969
<!-- <module>tis-datax/tis-datax-dolphinscheduler-plugin</module>-->
7070
<module>tis-datax/tis-hive-shim-common</module>
71-
71+
<module>tis-ontology-plugin</module>
7272

7373

7474
<!-- <module>tis-solr-plugin</module>

‎tis-datax/tis-datax-doris-plugin/src/main/java/com/qlangtech/tis/plugin/ds/doris/DorisSourceFactory.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import java.util.Collections;
5959
import java.util.List;
6060
import java.util.Map;
61+
import java.util.Objects;
6162
import java.util.Optional;
6263
import java.util.Properties;
6364
import java.util.Set;
@@ -89,7 +90,7 @@ public class DorisSourceFactory extends BasicDataSourceFactory {
8990
@FormField(ordinal = 4, validate = {Validator.require})
9091
public DataSourceCatalog catalog;
9192

92-
@FormField(ordinal = 8, type = FormFieldType.TEXTAREA, validate = {Validator.require})
93+
@FormField(ordinal = 9, type = FormFieldType.TEXTAREA, validate = {Validator.require})
9394
public String loadUrl;
9495

9596
@Override
@@ -122,7 +123,8 @@ public String buidJdbcUrl(DBConfig db, String ip, String dbName) {
122123
jdbcUrl.append("jdbc:mysql://").append(ip).append(":").append(this.port);
123124

124125
if (StringUtils.isNotEmpty(dbName)) {
125-
catalog.appendJdbcUrl(jdbcUrl.append("/"), dbName);
126+
Objects.requireNonNull(catalog,"catalog can not be null")
127+
.appendJdbcUrl(jdbcUrl.append("/"), dbName);
126128
}
127129
return jdbcUrl.toString();
128130
}

‎tis-datax/tis-datax-doris-plugin/src/test/java/com/qlangtech/tis/plugin/ds/doris/TestDorisSourceFactory.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,19 @@
1818

1919
package com.qlangtech.tis.plugin.ds.doris;
2020

21+
import com.qlangtech.tis.common.utils.Assert;
22+
import com.qlangtech.tis.extension.IPropertyType;
23+
import com.qlangtech.tis.extension.impl.PropertyType;
2124
import com.qlangtech.tis.manage.common.CenterResource;
2225
import com.qlangtech.tis.manage.common.HttpUtils;
26+
import com.qlangtech.tis.plugin.annotation.Validator;
2327
import com.qlangtech.tis.plugin.ds.TableInDB;
2428
import com.qlangtech.tis.plugin.ds.impl.CatalogSpecific;
2529
import junit.framework.TestCase;
2630

31+
import java.util.Map;
32+
import java.util.Optional;
33+
2734
/**
2835
* @author: 百岁（baisui@qlangtech.com）
2936
* @create: 2021-09-07 14:01
@@ -39,7 +46,22 @@ public void setUp() throws Exception {
3946
CenterResource.setNotFetchFromCenterRepository();
4047
HttpUtils.addMockGlobalParametersConfig();
4148
}
42-
49+
public void testGetValidatorDisableRequireValidatorByJsonConfig() {
50+
Map<String, /*** fieldname */IPropertyType> props
51+
= PropertyType.buildPropertyTypes(Optional.empty(), DorisSourceFactory.class);
52+
PropertyType passwordPropery = (PropertyType) props.get("password");
53+
Assert.assertNotNull(passwordPropery);
54+
Validator[] validators = passwordPropery.getValidator();
55+
// contain 2 validator 'require' and 'none_blank'
56+
Assert.assertEquals(1, validators.length);
57+
boolean containRequire = false;
58+
for (Validator v : validators) {
59+
if (v == Validator.require) {
60+
containRequire = true;
61+
}
62+
}
63+
Assert.assertFalse("shall not containRequire", containRequire);
64+
}
4365

4466
public void testDataDumpers() throws Exception {
4567
DorisSourceFactory dataSourceFactory = getDorisSourceFactory();

‎tis-ontology-plugin/pom.xml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--~
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing, software
14+
~ distributed under the License is distributed on an "AS IS" BASIS,
15+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
~ See the License for the specific language governing permissions and
17+
~ limitations under the License.
18+
-->
19+
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
<parent>
25+
<groupId>com.qlangtech.tis.plugins</groupId>
26+
<artifactId>tis-plugin-parent</artifactId>
27+
<version>5.1.0</version>
28+
<relativePath>../pom.xml</relativePath>
29+
</parent>
30+
<packaging>tpi</packaging>
31+
32+
<artifactId>tis-ontology-plugin</artifactId>
33+
34+
<properties>
35+
36+
</properties>
37+
38+
</project>

‎tis-ontology-plugin/src/main/java/com/qlangtech/tis/plugin/ontology/AggregationFunc.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package com.qlangtech.tis.plugin.ontology;
19+
20+
/**
21+
* Derived property 聚合方式的 kind 标签。
22+
* 子类层次见 {@link com.qlangtech.tis.plugin.ontology.impl.aggregation.AggregationKind}。
23+
*
24+
* @author 百岁 (baisui@qlangtech.com)
25+
* @date 2026/5/9
26+
*/
27+
public enum AggregationFunc {
28+
COUNT,
29+
SUM,
30+
AVG,
31+
MIN,
32+
MAX,
33+
APPROX_CARDINALITY,
34+
COUNT_DISTINCT,
35+
COLLECT_LIST,
36+
COLLECT_SET
37+
}

‎tis-ontology-plugin/src/main/java/com/qlangtech/tis/plugin/ontology/EnableChatBI.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * <p> * http://www.apache.org/licenses/LICENSE-2.0 * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */package com.qlangtech.tis.plugin.ontology;import com.alibaba.citrus.turbine.Context;import com.qlangtech.tis.aiagent.llm.LLMProvider;import com.qlangtech.tis.extension.DescriptorUseableShortComment;import com.qlangtech.tis.extension.TISExtension;import com.qlangtech.tis.plugin.annotation.FormField;import com.qlangtech.tis.plugin.annotation.FormFieldType;import com.qlangtech.tis.plugin.annotation.Validator;import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;import com.qlangtech.tis.util.IPluginContext;import com.qlangtech.tis.util.UploadPluginMeta;import java.util.Optional;import static com.qlangtech.tis.manage.common.UserProfile.KEY_FIELD_LLM_NAME;/** * 开启智能问数，design/chat-bi/06-neo4j-ontology-sync.md * * @author 百岁 (baisui@qlangtech.com) * @date 2026/5/28 */public class EnableChatBI extends OntologyDomainManipulate { /** * 大模型接口 */ @FormField(type = FormFieldType.SELECTABLE, ordinal = 1, validate = {Validator.identity}) public String llm; @Override public void manipuldateProcess(IPluginContext pluginContext, UploadPluginMeta pluginMeta, Optional<Context> context) { } @TISExtension public static final class DftDesc extends OntologyDomainManipulate.BasicDesc implements DescriptorUseableShortComment { public DftDesc() { super(); // List<Pair<OntologyDomain, IPluginStore<OntologyDomain>>> domainList = OntologyDomain // .getDoaminList(); // List<OntologyDomain> domains = domainList.stream().map(Pair::getKey).toList(); // this.registerSelectOptions("ontologyDomain", () -> domains); this.registerSelectOptions(KEY_FIELD_LLM_NAME, LLMProvider::getExistProviders); } @Override protected boolean validateAll(IControlMsgHandler msgHandler, Context context, PostFormVals postFormVals) { // // return super.validateAll(msgHandler, context, postFormVals); // OntologyPluginMeta ometa = getOntologyPluginMeta((IPluginContext) msgHandler, Optional.of // (context)); // // List<OntologyObjectType> objectTypes = OntologyObjectType.loadAll(ometa.getDomain()); // if (objectTypes.isEmpty()) { // throw new IllegalStateException("domain '" + ometa.getDomain() // + "' has no ObjectType, please export tables first"); // } // Optional<OntologyProperty> pk = null; // List<OntologyObjectType> lackPkObjTypes = Lists.newArrayList(); // for (OntologyObjectType objType : objectTypes) { // pk = objType.getPk(); // if (!pk.isPresent()) { // lackPkObjTypes.add(objType); // } // } // // if (CollectionUtils.isNotEmpty(lackPkObjTypes)) { // msgHandler.addErrorMessage(context, // "对象还未设置主键，完成后继续此操作：" // // + lackPkObjTypes.stream() // // .map((obj) -> "'" + obj.getName() + "'").collect(Collectors.joining(", // "))); // return false; // } return true; } @Override public String getDisplayName() { return "Enable ChatBI"; } @Override public String shortComment() { return "开启自然语言统计问数功能"; } }}

‎tis-ontology-plugin/src/main/java/com/qlangtech/tis/plugin/ontology/ExportToOntologyInDataSource.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * <p> * http://www.apache.org/licenses/LICENSE-2.0 * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */package com.qlangtech.tis.plugin.ontology;import com.alibaba.citrus.turbine.Context;import com.google.common.collect.Lists;import com.qlangtech.tis.datax.DataXName;import com.qlangtech.tis.datax.StoreResourceType;import com.qlangtech.tis.extension.Descriptor;import com.qlangtech.tis.extension.OneStepOfMultiSteps;import com.qlangtech.tis.extension.TISExtension;import com.qlangtech.tis.manage.common.AppAndRuntime;import com.qlangtech.tis.manage.common.Option;import com.qlangtech.tis.plugin.IPluginStore;import com.qlangtech.tis.plugin.IdentityName;import com.qlangtech.tis.plugin.annotation.FormField;import com.qlangtech.tis.plugin.annotation.FormFieldType;import com.qlangtech.tis.plugin.annotation.Validator;import com.qlangtech.tis.plugin.ds.ColumnMetaData;import com.qlangtech.tis.plugin.ds.DBIdentity;import com.qlangtech.tis.plugin.ds.DataSourceFactory;import com.qlangtech.tis.plugin.ds.DataSourceFactoryManipulate;import com.qlangtech.tis.plugin.ds.TableInDB;import com.qlangtech.tis.plugin.ds.manipulate.ManipulateItemsProcessor;import com.qlangtech.tis.plugin.ds.manipulate.ManipuldateUtils;import com.qlangtech.tis.plugin.ontology.impl.objtype.DataSourceBinding;import com.qlangtech.tis.plugin.ontology.impl.objtype.DefaultOntologyObjectType;import com.qlangtech.tis.plugin.ontology.impl.objtype.DefaultOntologyProperty;import com.qlangtech.tis.plugin.ontology.impl.objtype.ObjectTypeProfile;import com.qlangtech.tis.plugin.ontology.impl.objtype.ObjectTypeProperties;import com.qlangtech.tis.plugin.ontology.impl.objtype.ObjectTypePropertiesRelevant;import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;import com.qlangtech.tis.sql.parser.tuple.creator.EntityName;import com.qlangtech.tis.util.IPluginContext;import com.qlangtech.tis.util.UploadPluginMeta;import org.apache.commons.collections.CollectionUtils;import org.apache.commons.lang3.StringUtils;import org.apache.commons.lang3.tuple.Pair;import java.util.List;import java.util.Objects;import java.util.Optional;/** * 从数据源中导出本体对象 * * @author 百岁 (baisui@qlangtech.com) * @date 2026/4/13 */public class ExportToOntologyInDataSource extends DataSourceFactoryManipulate { public static final String KEY_ONTOLOGY_PKG = "ontologyPkg"; /** * 目标本体管理域 */ @FormField(ordinal = 0, type = FormFieldType.SELECTABLE, validate = {Validator.require, Validator.identity}) public String ontologyPkg; /** * 需要导入到本体域的表对象 */ @FormField(ordinal = 200, type = FormFieldType.MULTI_SELECTABLE, validate = {Validator.require}) public List<IdentityName> targetTables = Lists.newArrayList(); public static List<Option> getCandidateTargetTables() { AppAndRuntime appAndRuntime = AppAndRuntime.getAppAndRuntime(); DataXName appName = Objects.requireNonNull(appAndRuntime, "appAndRuntime can not be null").getAppName(); DataSourceFactory dataSource = DataSourceFactory.load(appName.getPipelineName()); TableInDB tablesInDB = dataSource.getTablesInDB(); List<String> tabs = tablesInDB.getTabs(); return tabs.stream().map(Option::new).toList(); } // public static final String KEY_DATASOURCE_META = "ds.json"; @Override public void manipuldateProcess(IPluginContext pluginContext, UploadPluginMeta pluginMeta, Optional<Context> context) { if (StringUtils.isEmpty(this.ontologyPkg)) { throw new IllegalArgumentException("property ontologyPkg can not be null"); } Context ctx = context.orElseThrow(); ManipulateItemsProcessor itemsProcessor = ManipuldateUtils.instance(pluginContext, ctx, null, (meta) -> { meta.putExtraParams(DBIdentity.KEY_UPDATE, Boolean.FALSE.toString()); meta.putExtraParams(StoreResourceType.DATAX_DB_NAME, meta.getExtraParam(DBIdentity.KEY_DB_NAME)); }); DataXName ds = itemsProcessor.getPluginMeta().getDataXName(); DataSourceFactory fromSource = Objects.requireNonNull(DataSourceFactory.load(ds.getPipelineName()), "ds:" + ds.getPipelineName() + " relevant instance can not be null"); if (CollectionUtils.isEmpty(targetTables)) { throw new IllegalStateException("targetTables can not be null"); } // File objectTypeDir = new File(OntologyDomain.getObjectTypeDir(this.ontologyPkg), fromSource // .identityValue()); // new File(objectTypeDir,KEY_DATASOURCE_META); fromSource.visitFirstConnection((conn) -> { // try { EntityName entityName = null; for (IdentityName tab : targetTables) { entityName = EntityName.parse(tab.identityValue()); //JSONObject tabObj = new JSONObject(); List<ColumnMetaData> tableMetadata = fromSource.getTableMetadata(conn, false, entityName); OntologyObjectType objectType = new DefaultOntologyObjectType(); ObjectTypeProfile profile = new ObjectTypeProfile(); DataSourceBinding dsBinding = new DataSourceBinding(); dsBinding.dbName = fromSource.identityValue(); profile.binding = dsBinding; ObjectTypeProperties propsSetter = new ObjectTypeProperties(); profile.name = entityName.getTableName(); // objectType.setDataSourceName(fromSource.identityValue()); List<OntologyProperty> cols = Lists.newArrayList(); ObjectTypePropertiesRelevant propRelevant = new ObjectTypePropertiesRelevant(); for (ColumnMetaData c : tableMetadata) { cols.add(new DefaultOntologyProperty(c.getName(), c.isPk(), c.isNullable(), c.getComment(), OntologyType.convert(c.getType()))); if (c.isPk()) { propRelevant.pkField = c.getKey();// pkCol = c; } } propsSetter.setCols(cols); // propRelevant.pkField objectType.setSteps(new OneStepOfMultiSteps[]{profile, propsSetter, propRelevant}); IPluginStore<OntologyObjectType> objTypeStore = OntologyObjectType.getPluginStore(ontologyPkg, entityName.getTableName()); objTypeStore.setPlugins(null, Optional.empty(), List.of(new Descriptor.ParseDescribable<>(objectType))); } // } catch (IOException e) { // throw new RuntimeException(e); // } }); } @TISExtension public static final class DftDesc extends BasicDesc { public DftDesc() { super(); List<Pair<OntologyDomain, IPluginStore<OntologyDomain>>> doaminList = OntologyDomain.getDoaminList(); List<OntologyDomain> domains = doaminList.stream().map(Pair::getKey).toList(); this.registerSelectOptions(KEY_ONTOLOGY_PKG, () -> domains); } @Override protected boolean validateAll(IControlMsgHandler msgHandler, Context context, PostFormVals postFormVals) { ExportToOntologyInDataSource export = postFormVals.newInstance(); return super.validateAll(msgHandler, context, postFormVals); } @Override public String getDisplayName() { return "Export To Ontology"; } @Override public EndType getEndType() { return EndType.Ontology; } }}

0 commit comments

Comments
 (0)