diff --git a/pom.xml b/pom.xml
index 504c317..7eb0cd3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -84,6 +84,11 @@
commons-csv
${commons.csv.version}
+
+ com.google.code.gson
+ gson
+ 2.10.1
+
diff --git a/src/main/java/com/evolveum/polygon/connector/csv/CsvConfiguration.java b/src/main/java/com/evolveum/polygon/connector/csv/CsvConfiguration.java
index 5f11dc7..1e38112 100644
--- a/src/main/java/com/evolveum/polygon/connector/csv/CsvConfiguration.java
+++ b/src/main/java/com/evolveum/polygon/connector/csv/CsvConfiguration.java
@@ -197,6 +197,17 @@ public boolean isAuxiliary() {
return config.isAuxiliary();
}
+ @ConfigurationProperty(
+ displayMessageKey = "UI_CSV_GROUP_BY_ENABLED",
+ helpMessageKey = "UI_CSV_GROUP_BY_ENABLED_HELP")
+ public boolean isGroupByEnabled() {
+ return config.isGroupByEnabled();
+ }
+
+ public void setGroupByEnabled(boolean groupByEnabled) {
+ config.setGroupByEnabled(groupByEnabled);
+ }
+
public void setContainer(boolean container) {
config.setContainer(container);
}
diff --git a/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandler.java b/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandler.java
index 81ee89f..1d23f22 100644
--- a/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandler.java
+++ b/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandler.java
@@ -4,6 +4,7 @@
import com.evolveum.polygon.connector.csv.util.ConfigurationDetector;
import com.evolveum.polygon.connector.csv.util.StringAccessor;
import com.evolveum.polygon.connector.csv.util.Util;
+import com.google.gson.Gson;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVPrinter;
@@ -47,6 +48,7 @@ private enum Operation {
}
private static final Log LOG = Log.getLog(ObjectClassHandler.class);
+ private static final String ATTR_RAW_JSON = AttributeUtil.createSpecialName("RAW_JSON");
private ObjectClassHandlerConfiguration configuration;
@@ -260,6 +262,17 @@ private List createAttributeInfo(Map columns) {
infos.add(builder.build());
}
+ if (configuration.isGroupByEnabled()) {
+ AttributeInfoBuilder builder = new AttributeInfoBuilder(ATTR_RAW_JSON);
+ builder.setType(String.class);
+ builder.setNativeName(ATTR_RAW_JSON);
+ builder.setCreateable(false);
+ builder.setUpdateable(false);
+ builder.setSubtype(AttributeInfo.Subtypes.STRING_JSON);
+
+ infos.add(builder.build());
+ }
+
return infos;
}
@@ -434,6 +447,11 @@ public void executeQuery(ObjectClass oc, Filter filter, ResultsHandler handler,
CSVFormat csv = Util.createCsvFormatReader(configuration);
try (Reader reader = Util.createReader(configuration)) {
+ boolean shouldContinue = true;
+ List recordGroup = new ArrayList<>();
+ int uidIndex = this.getHeader().get(configuration.getUniqueAttribute()).getIndex();
+ String uid = extractUidFromFilter(filter);
+
CSVParser parser = csv.parse(reader);
Iterator iterator = parser.iterator();
while (iterator.hasNext()) {
@@ -442,13 +460,36 @@ public void executeQuery(ObjectClass oc, Filter filter, ResultsHandler handler,
continue;
}
- ConnectorObject obj = createConnectorObject(record);
+ ConnectorObject obj;
- String uid = extractUidFromFilter(filter);
+ if (configuration.isGroupByEnabled()) {
+ // Record group is empty, so push it then check next record
+ if (recordGroup.isEmpty()) {
+ recordGroup.add(record);
+ continue;
+ }
+ // If the current record has same uid with the current record group, push it then check next record
+ String currentUid = record.get(uidIndex);
+ String currentUidOfRecordGroup = recordGroup.get(0).get(uidIndex);
+ if (uidMatches(currentUid, currentUidOfRecordGroup, configuration.isIgnoreIdentifierCase())) {
+ recordGroup.add(record);
+ continue;
+ }
+
+ obj = createConnectorObject(recordGroup);
+
+ // Reset record group for next
+ recordGroup.clear();
+ recordGroup.add(record);
+
+ } else {
+ obj = createConnectorObject(record);
+ }
if (uid == null) {
if (filter == null || filter.accept(obj)) {
- if (!handler.handle(obj)) {
+ shouldContinue = handler.handle(obj);
+ if (!shouldContinue) {
break;
}
}
@@ -459,10 +500,31 @@ public void executeQuery(ObjectClass oc, Filter filter, ResultsHandler handler,
continue;
}
- if (!handler.handle(obj)) {
+ shouldContinue = handler.handle(obj);
+ if (!shouldContinue) {
break;
}
}
+
+ if (configuration.isGroupByEnabled()) {
+ // Handle remaining record group
+ if (shouldContinue && !recordGroup.isEmpty()) {
+ ConnectorObject obj = createConnectorObject(recordGroup);
+
+ if (uid == null) {
+ if (filter == null || filter.accept(obj)) {
+ handler.handle(obj);
+ }
+ return;
+ }
+
+ if (!uidMatches(uid, obj.getUid().getUidValue(), configuration.isIgnoreIdentifierCase())) {
+ return;
+ }
+
+ handler.handle(obj);
+ }
+ }
} catch (Exception ex) {
handleGenericException(ex, "Error during query execution");
}
@@ -934,7 +996,59 @@ private Map reverseHeaderMap() {
return reversed;
}
+ private ConnectorObject createConnectorObject(List recordGroup) {
+ Map header = reverseHeaderMap();
+ Set multivalueAttrs = StringUtil.isEmpty(configuration.getMultivalueAttributes()) || StringUtil.isEmpty(configuration.getMultivalueDelimiter()) ?
+ Collections.emptySet() : Set.of(configuration.getMultivalueAttributes().split(configuration.getMultivalueDelimiter()));
+
+ // Create base connectorObject using first record
+ ConnectorObjectBuilder builder = createConnectorObjectBuilder(recordGroup.get(0));
+
+ // Create rawJson and append to the connectorObject
+ List> data = new ArrayList<>();
+ for (CSVRecord record : recordGroup) {
+ if (header.size() != record.size()) {
+ throw new ConnectorException("Number of columns in header (" + header.size()
+ + ") doesn't match number of columns for record (" + record.size()
+ + "). File row number: " + record.getRecordNumber());
+ }
+
+ Map recordMap = new HashMap<>();
+ for (int i = 0; i < record.size(); i++) {
+ String name = header.get(i);
+ String value = record.get(i);
+
+ if (multivalueAttrs.contains(name)) {
+ // Multiple value
+ String[] values = Arrays.stream(value.split(configuration.getMultivalueDelimiter()))
+ .filter(StringUtil::isNotEmpty)
+ .toArray(String[]::new);
+ recordMap.put(name, values);
+ } else {
+ // Single value
+ if (StringUtil.isEmpty(value)) {
+ // Include empty string for JSON if empty
+ recordMap.put(name, "");
+ } else {
+ recordMap.put(name, value);
+ }
+ }
+ }
+ data.add(recordMap);
+ }
+ if (!data.isEmpty()) {
+ String json = new Gson().toJson(data);
+ builder.addAttribute(ATTR_RAW_JSON, json);
+ }
+
+ return builder.build();
+ }
+
private ConnectorObject createConnectorObject(CSVRecord record) {
+ return createConnectorObjectBuilder(record).build();
+ }
+
+ private ConnectorObjectBuilder createConnectorObjectBuilder(CSVRecord record) {
ConnectorObjectBuilder builder = new ConnectorObjectBuilder();
Map header = reverseHeaderMap();
@@ -974,7 +1088,7 @@ private ConnectorObject createConnectorObject(CSVRecord record) {
builder.addAttribute(name, createAttributeValues(value));
}
- return builder.build();
+ return builder;
}
private boolean isUniqueAndNameAttributeEqual() {
diff --git a/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandlerConfiguration.java b/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandlerConfiguration.java
index ba1aac7..79af94f 100644
--- a/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandlerConfiguration.java
+++ b/src/main/java/com/evolveum/polygon/connector/csv/ObjectClassHandlerConfiguration.java
@@ -55,6 +55,8 @@ public class ObjectClassHandlerConfiguration {
private boolean container = false;
private boolean auxiliary = false;
+ private boolean groupByEnabled = false;
+
public ObjectClassHandlerConfiguration() {
this(ObjectClass.ACCOUNT, null);
}
@@ -92,6 +94,8 @@ public ObjectClassHandlerConfiguration(ObjectClass oc, Map value
setContainer(Util.getSafeValue(values, "container", false, Boolean.class));
setAuxiliary(Util.getSafeValue(values, "auxiliary", false, Boolean.class));
+
+ setGroupByEnabled(Util.getSafeValue(values, "groupByEnabled", false, Boolean.class));
}
public void recompute() {
@@ -104,6 +108,14 @@ public void recompute() {
}
}
+ public boolean isGroupByEnabled() {
+ return groupByEnabled;
+ }
+
+ public void setGroupByEnabled(boolean groupByEnabled) {
+ this.groupByEnabled = groupByEnabled;
+ }
+
public boolean isContainer() {
return container;
}
diff --git a/src/main/resources/com/evolveum/polygon/connector/csv/Messages.properties b/src/main/resources/com/evolveum/polygon/connector/csv/Messages.properties
index 890c454..d043dfe 100644
--- a/src/main/resources/com/evolveum/polygon/connector/csv/Messages.properties
+++ b/src/main/resources/com/evolveum/polygon/connector/csv/Messages.properties
@@ -50,4 +50,6 @@ UI_MULTIVALUE_ATTRIBUTES_HELP=List all attributes that can have multiple values
UI_CONTAINER=Container
UI_CONTAINER_HELP=Should this object class be marked as container? Default is false.
UI_AUXILIARY=Auxiliary
-UI_AUXILIARY_HELP=Should this object class be marked as auxiliary? Default is false.
\ No newline at end of file
+UI_AUXILIARY_HELP=Should this object class be marked as auxiliary? Default is false.
+UI_CSV_GROUP_BY_ENABLED=Group by enabled
+UI_CSV_GROUP_BY_ENABLED_HELP=Enable group by unique attribute when reading csv (except for sync operation). The aggregated data is mapped to the "__RAW_JSON__" attribute with string type.
\ No newline at end of file
diff --git a/src/test/java/com/evolveum/polygon/connector/csv/SearchOpTest.java b/src/test/java/com/evolveum/polygon/connector/csv/SearchOpTest.java
index 26c111d..15ff0f2 100644
--- a/src/test/java/com/evolveum/polygon/connector/csv/SearchOpTest.java
+++ b/src/test/java/com/evolveum/polygon/connector/csv/SearchOpTest.java
@@ -3,6 +3,8 @@
import com.evolveum.polygon.connector.csv.util.ListResultHandler;
import org.identityconnectors.framework.api.ConnectorFacade;
import org.identityconnectors.framework.common.exceptions.ConnectorException;
+import org.identityconnectors.framework.common.objects.Attribute;
+import org.identityconnectors.framework.common.objects.AttributeUtil;
import org.identityconnectors.framework.common.objects.ConnectorObject;
import org.identityconnectors.framework.common.objects.ObjectClass;
import org.identityconnectors.framework.common.objects.Uid;
@@ -11,6 +13,7 @@
import org.testng.annotations.Test;
import java.io.File;
+import java.util.Collections;
import java.util.List;
/**
@@ -77,4 +80,102 @@ public void searchWrongNumberColumnCountInRow() throws Exception {
ConnectorFacade connector = setupConnector("/search-wrong-column-count-row.csv", config);
connector.search(ObjectClass.ACCOUNT, null, new ListResultHandler(), null);
}
+
+ @Test
+ public void findOneWithGroupByEnabled() throws Exception {
+ CsvConfiguration config = new CsvConfiguration();
+ config.setFilePath(new File(CSV_FILE_PATH));
+ config.setUniqueAttribute("id");
+ config.setMultivalueAttributes("tel");
+ config.setMultivalueDelimiter(",");
+ config.setGroupByEnabled(true);
+ ConnectorFacade connector = setupConnector("/search-grouping.csv", config);
+
+ ListResultHandler handler = new ListResultHandler();
+ EqualsFilter ef = new EqualsFilter(new Uid("1"));
+ connector.search(ObjectClass.ACCOUNT, ef, handler, null);
+
+ List objects = handler.getObjects();
+
+ AssertJUnit.assertEquals(1, objects.size());
+ ConnectorObject connectorObject = objects.get(0);
+ Attribute id = connectorObject.getAttributeByName(Uid.NAME);
+ AssertJUnit.assertEquals("1", AttributeUtil.getStringValue(id));
+ Attribute dept = connectorObject.getAttributeByName("dept");
+ AssertJUnit.assertEquals("abc", AttributeUtil.getStringValue(dept));
+ Attribute title = connectorObject.getAttributeByName("title");
+ AssertJUnit.assertEquals("engineer", AttributeUtil.getStringValue(title));
+ Attribute tel = connectorObject.getAttributeByName("tel");
+ AssertJUnit.assertEquals(List.of("1111"), tel.getValue());
+ Attribute rawJson = connectorObject.getAttributeByName("__RAW_JSON__");
+ AssertJUnit.assertNotNull(rawJson);
+ String jsonValue = AttributeUtil.getStringValue(rawJson);
+ AssertJUnit.assertEquals("[{\"name\":\"john\",\"tel\":[\"1111\"],\"id\":\"1\",\"dept\":\"abc\",\"title\":\"engineer\"},{\"name\":\"john\",\"tel\":[\"1111\"],\"id\":\"1\",\"dept\":\"efg\",\"title\":\"manager\"}]", jsonValue);
+ }
+
+ @Test
+ public void findAllWithGroupByEnabled() throws Exception {
+ CsvConfiguration config = new CsvConfiguration();
+ config.setFilePath(new File(CSV_FILE_PATH));
+ config.setUniqueAttribute("id");
+ config.setMultivalueAttributes("tel");
+ config.setMultivalueDelimiter(",");
+ config.setGroupByEnabled(true);
+ ConnectorFacade connector = setupConnector("/search-grouping.csv", config);
+
+ ListResultHandler handler = new ListResultHandler();
+ connector.search(ObjectClass.ACCOUNT, null, handler, null);
+
+ List objects = handler.getObjects();
+
+ AssertJUnit.assertEquals(3, objects.size());
+
+ ConnectorObject connectorObject = objects.get(0);
+ Attribute id = connectorObject.getAttributeByName(Uid.NAME);
+ AssertJUnit.assertEquals("1", AttributeUtil.getStringValue(id));
+ Attribute name = connectorObject.getAttributeByName("name");
+ AssertJUnit.assertEquals("john", AttributeUtil.getStringValue(name));
+ Attribute dept = connectorObject.getAttributeByName("dept");
+ AssertJUnit.assertEquals("abc", AttributeUtil.getStringValue(dept));
+ Attribute title = connectorObject.getAttributeByName("title");
+ AssertJUnit.assertEquals("engineer", AttributeUtil.getStringValue(title));
+ Attribute tel = connectorObject.getAttributeByName("tel");
+ AssertJUnit.assertEquals(List.of("1111"), tel.getValue());
+ Attribute rawJson = connectorObject.getAttributeByName("__RAW_JSON__");
+ AssertJUnit.assertNotNull(rawJson);
+ String jsonValue = AttributeUtil.getStringValue(rawJson);
+ AssertJUnit.assertEquals("[{\"name\":\"john\",\"tel\":[\"1111\"],\"id\":\"1\",\"dept\":\"abc\",\"title\":\"engineer\"},{\"name\":\"john\",\"tel\":[\"1111\"],\"id\":\"1\",\"dept\":\"efg\",\"title\":\"manager\"}]", jsonValue);
+
+ connectorObject = objects.get(1);
+ id = connectorObject.getAttributeByName(Uid.NAME);
+ AssertJUnit.assertEquals("2", AttributeUtil.getStringValue(id));
+ name = connectorObject.getAttributeByName("name");
+ AssertJUnit.assertEquals("jack", AttributeUtil.getStringValue(name));
+ dept = connectorObject.getAttributeByName("dept");
+ AssertJUnit.assertEquals("abc", AttributeUtil.getStringValue(dept));
+ title = connectorObject.getAttributeByName("title");
+ AssertJUnit.assertEquals("manager", AttributeUtil.getStringValue(title));
+ tel = connectorObject.getAttributeByName("tel");
+ AssertJUnit.assertEquals(List.of("1111", "2222"), tel.getValue());
+ rawJson = connectorObject.getAttributeByName("__RAW_JSON__");
+ AssertJUnit.assertNotNull(rawJson);
+ jsonValue = AttributeUtil.getStringValue(rawJson);
+ AssertJUnit.assertEquals("[{\"name\":\"jack\",\"tel\":[\"1111\",\"2222\"],\"id\":\"2\",\"dept\":\"abc\",\"title\":\"manager\"}]", jsonValue);
+
+ connectorObject = objects.get(2);
+ id = connectorObject.getAttributeByName(Uid.NAME);
+ AssertJUnit.assertEquals("3", AttributeUtil.getStringValue(id));
+ name = connectorObject.getAttributeByName("name");
+ AssertJUnit.assertEquals("bob", AttributeUtil.getStringValue(name));
+ dept = connectorObject.getAttributeByName("dept");
+ AssertJUnit.assertNull(dept);
+ title = connectorObject.getAttributeByName("title");
+ AssertJUnit.assertNull(title);
+ tel = connectorObject.getAttributeByName("tel");
+ AssertJUnit.assertNull(tel);
+ rawJson = connectorObject.getAttributeByName("__RAW_JSON__");
+ AssertJUnit.assertNotNull(rawJson);
+ jsonValue = AttributeUtil.getStringValue(rawJson);
+ AssertJUnit.assertEquals("[{\"name\":\"bob\",\"tel\":[],\"id\":\"3\",\"dept\":\"\",\"title\":\"\"}]", jsonValue);
+ }
}
diff --git a/src/test/resources/search-grouping.csv b/src/test/resources/search-grouping.csv
new file mode 100644
index 0000000..748ee60
--- /dev/null
+++ b/src/test/resources/search-grouping.csv
@@ -0,0 +1,5 @@
+id;name;dept;title;tel
+1;john;abc;engineer;1111
+1;john;efg;manager;1111
+2;jack;abc;manager;1111,2222
+3;bob;;;
\ No newline at end of file