Skip to content

Commit 7ce6078

Browse files
committed
feat(connection): implement connection testing functionality and enhance connection management
- Added connection testing feature in the UI, allowing users to test connections before saving. - Updated Connection model to include source information for connections (file or config). - Enhanced ConnectionRepository to manage connections from files and application.conf, ensuring proper handling of deletable and non-deletable connections. - Introduced ConnectionTestService to handle various connection types and provide detailed test results. - Updated UI components to reflect connection source and added test buttons for existing connections.
1 parent 3fa0c49 commit 7ce6078

9 files changed

Lines changed: 788 additions & 156 deletions

File tree

app/src/integrationTest/scala/io/github/datacatering/datacaterer/core/ui/plan/ConnectionRepositoryTest.scala

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,41 @@ class ConnectionRepositoryTest extends AnyFunSuiteLike with BeforeAndAfterAll wi
119119
probe.expectMessage(ConnectionRepository.ConnectionRemoved("nonExistentConnection", false))
120120
}
121121

122+
test("saved connections should have source=file when retrieved") {
123+
cleanFolder()
124+
val connection = Connection("testSourceConnection", "csv", Some(CONNECTION_GROUP_DATA_SOURCE), Map("key" -> "value"))
125+
val probe = testKit.createTestProbe[ConnectionRepository.ConnectionResponse]()
126+
127+
connectionRepository ! ConnectionRepository.SaveConnections(SaveConnectionsRequest(List(connection)), Some(probe.ref))
128+
probe.expectMessage(ConnectionRepository.ConnectionsSaved(1))
129+
130+
val connectionSaveFolder = s"$tempTestDirectory/connection"
131+
val retrievedConnection = ConnectionRepository.getConnection("testSourceConnection", connectionSaveFolder)
132+
133+
// Connections loaded from files should have source = "file"
134+
retrievedConnection.source shouldEqual "file"
135+
retrievedConnection.isFromFile shouldBe true
136+
retrievedConnection.isFromConfig shouldBe false
137+
}
138+
139+
test("getAllConnections should return connections with correct source field") {
140+
cleanFolder()
141+
val connection = Connection("testSourceConnection2", "csv", Some(CONNECTION_GROUP_DATA_SOURCE), Map("key" -> "value"))
142+
val saveProbe = testKit.createTestProbe[ConnectionRepository.ConnectionResponse]()
143+
144+
connectionRepository ! ConnectionRepository.SaveConnections(SaveConnectionsRequest(List(connection)), Some(saveProbe.ref))
145+
saveProbe.expectMessage(ConnectionRepository.ConnectionsSaved(1))
146+
147+
val getProbe = testKit.createTestProbe[GetConnectionsResponse]()
148+
connectionRepository ! ConnectionRepository.GetConnections(Some(CONNECTION_GROUP_DATA_SOURCE), getProbe.ref)
149+
150+
val response = getProbe.receiveMessage()
151+
val fileConnections = response.connections.filter(_.name == "testSourceConnection2")
152+
fileConnections should have size 1
153+
fileConnections.head.source shouldEqual "file"
154+
fileConnections.head.isFromFile shouldBe true
155+
}
156+
122157
private def cleanFolder(folder: String = "connection"): Unit = {
123158
val path = Paths.get(s"$tempTestDirectory/$folder").toFile
124159
if (path.exists()) {

app/src/integrationTest/scala/io/github/datacatering/datacaterer/core/ui/plan/PlanApiEndToEndTest.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ class PlanApiEndToEndTest extends AnyFunSuite with Matchers with BeforeAndAfterA
442442
val response = postJson("/run", planRunRequest)
443443

444444
response.statusCode() shouldBe 200
445-
response.body() should include("Plan started")
445+
response.body() should include("started")
446446
}
447447

448448
// ============================================================================

app/src/main/resources/application.conf

Lines changed: 0 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -114,116 +114,4 @@ runtime {
114114
}
115115
}
116116

117-
json {
118-
json {
119-
}
120-
}
121-
122-
csv {
123-
csv {
124-
}
125-
}
126-
127-
delta {
128-
delta {
129-
}
130-
}
131-
132-
iceberg {
133-
iceberg {
134-
}
135-
}
136-
137-
orc {
138-
orc {
139-
}
140-
}
141-
142-
parquet {
143-
parquet {
144-
}
145-
}
146-
147-
jdbc {
148-
postgres {
149-
url = "jdbc:postgresql://localhost:5432/customer"
150-
url = ${?POSTGRES_URL}
151-
user = "postgres"
152-
user = ${?POSTGRES_USER}
153-
password = "postgres"
154-
password = ${?POSTGRES_PASSWORD}
155-
driver = "org.postgresql.Driver"
156-
}
157-
mysql {
158-
url = "jdbc:mysql://localhost:3306/customer"
159-
url = ${?MYSQL_URL}
160-
user = "root"
161-
user = ${?MYSQL_USERNAME}
162-
password = "root"
163-
password = ${?MYSQL_PASSWORD}
164-
driver = "com.mysql.cj.jdbc.Driver"
165-
}
166-
}
167-
168-
169-
org.apache.spark.sql.cassandra {
170-
cassandra {
171-
spark.cassandra.connection.host = "localhost"
172-
spark.cassandra.connection.host = ${?CASSANDRA_HOST}
173-
spark.cassandra.connection.port = "9042"
174-
spark.cassandra.connection.port = ${?CASSANDRA_PORT}
175-
spark.cassandra.auth.username = "cassandra"
176-
spark.cassandra.auth.username = ${?CASSANDRA_USERNAME}
177-
spark.cassandra.auth.password = "cassandra"
178-
spark.cassandra.auth.password = ${?CASSANDRA_PASSWORD}
179-
}
180-
}
181-
182-
bigquery {
183-
bigquery {
184-
}
185-
}
186-
187-
http {
188-
http {
189-
}
190-
}
191-
192-
jms {
193-
solace {
194-
initialContextFactory = "com.solacesystems.jndi.SolJNDIInitialContextFactory"
195-
initialContextFactory = ${?SOLACE_INITIAL_CONTEXT_FACTORY}
196-
connectionFactory = "/jms/cf/default"
197-
connectionFactory = ${?SOLACE_CONNECTION_FACTORY}
198-
url = "smf://localhost:55554"
199-
url = ${?SOLACE_URL}
200-
user = "admin"
201-
user = ${?SOLACE_USER}
202-
password = "admin"
203-
password = ${?SOLACE_PASSWORD}
204-
vpnName = "default"
205-
vpnName = ${?SOLACE_VPN}
206-
}
207-
rabbitmq {
208-
connectionFactory = "com.rabbitmq.jms.admin.RMQConnectionFactory"
209-
connectionFactory = ${?RABBITMQ_CONNECTION_FACTORY}
210-
url = "amqp://localhost:5672"
211-
url = ${?RABBITMQ_URL}
212-
user = "guest"
213-
user = ${?RABBITMQ_USER}
214-
password = "guest"
215-
password = ${?RABBITMQ_PASSWORD}
216-
virtualHost = "/"
217-
virtualHost = ${?RABBITMQ_VIRTUAL_HOST}
218-
}
219-
}
220-
221-
kafka {
222-
kafka {
223-
kafka.bootstrap.servers = "localhost:9092"
224-
kafka.bootstrap.servers = ${?KAFKA_BOOTSTRAP_SERVERS}
225-
}
226-
}
227-
228-
229117
datastax-java-driver.advanced.metadata.schema.refreshed-keyspaces = [ "/.*/" ]

app/src/main/resources/ui/connection/connection.js

Lines changed: 184 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,133 @@ import {
1313
import {dataSourcePropertiesMap} from "../configuration-data.js";
1414
import {apiFetch} from "../config.js";
1515

16+
/**
17+
* Test a connection and display the result
18+
* @param {Object} connection - The connection object to test
19+
* @param {HTMLElement} button - The button element that was clicked (to show loading state)
20+
*/
21+
async function testConnection(connection, button) {
22+
const originalText = button.innerText;
23+
button.innerText = "Testing...";
24+
button.disabled = true;
25+
26+
try {
27+
const response = await apiFetch("/connection/test", {
28+
method: "POST",
29+
headers: {
30+
"Content-Type": "application/json"
31+
},
32+
body: JSON.stringify(connection)
33+
});
34+
35+
const result = await response.json();
36+
// Limit to 100 characters, show ellipsis if truncated
37+
const resultDetails = result.details ? ` - ${truncateString(result.details, 100)}` : "";
38+
39+
if (result.success) {
40+
createToast(
41+
`Test: ${connection.name}`,
42+
`${result.message}${resultDetails}`,
43+
"success"
44+
);
45+
} else {
46+
createToast(
47+
`Test: ${connection.name}`,
48+
`${result.message}${resultDetails}`,
49+
"fail"
50+
);
51+
}
52+
} catch (err) {
53+
// Limit to 100 characters, show ellipsis if truncated
54+
const resultDetails = err.message ? ` - ${truncateString(err.message, 100)}` : "";
55+
createToast(
56+
`Test: ${connection.name}`,
57+
`Connection test failed: ${resultDetails}`,
58+
"fail"
59+
);
60+
} finally {
61+
button.innerText = originalText;
62+
button.disabled = false;
63+
}
64+
}
65+
66+
/**
67+
* Test an existing saved connection by name
68+
* @param {string} connectionName - The name of the saved connection to test
69+
* @param {HTMLElement} button - The button element that was clicked (to show loading state)
70+
*/
71+
async function testExistingConnection(connectionName, button) {
72+
const originalText = button.innerText;
73+
button.innerText = "Testing...";
74+
button.disabled = true;
75+
76+
try {
77+
const response = await apiFetch(`/connection/${connectionName}/test`, {
78+
method: "POST"
79+
});
80+
81+
const result = await response.json();
82+
// Limit to 100 characters, show ellipsis if truncated
83+
const resultDetails = result.details ? ` - ${truncateString(result.details, 100)}` : "";
84+
if (result.success) {
85+
createToast(
86+
`Test: ${connectionName}`,
87+
`${result.message}${resultDetails}`,
88+
"success"
89+
);
90+
} else {
91+
createToast(
92+
`Test: ${connectionName}`,
93+
`${result.message}${resultDetails}`,
94+
"fail"
95+
);
96+
}
97+
} catch (err) {
98+
// Limit to 100 characters, show ellipsis if truncated
99+
const resultDetails = err.message ? ` - ${truncateString(err.message, 100)}` : "";
100+
createToast(
101+
`Test: ${connectionName}`,
102+
`Connection test failed: ${resultDetails}`,
103+
"fail"
104+
);
105+
} finally {
106+
button.innerText = originalText;
107+
button.disabled = false;
108+
}
109+
}
110+
111+
function truncateString(str, maxLength) {
112+
return str.length > maxLength ? str.substring(0, maxLength) + "..." : str;
113+
}
114+
115+
/**
116+
* Build a connection object from a data source container element
117+
* @param {HTMLElement} container - The data source container element
118+
* @returns {Object} The connection object
119+
*/
120+
function buildConnectionFromContainer(container) {
121+
let connection = {};
122+
let connectionOptions = {};
123+
let inputFields = Array.from(container.querySelectorAll(".input-field").values());
124+
125+
for (let inputField of inputFields) {
126+
let ariaLabel = inputField.getAttribute("aria-label");
127+
if (ariaLabel) {
128+
if (ariaLabel === "Name") {
129+
connection["name"] = inputField.value || "test-connection";
130+
} else if (ariaLabel === "Data source") {
131+
connection["type"] = inputField.value;
132+
} else {
133+
if (inputField.value !== "") {
134+
connectionOptions[ariaLabel] = inputField.value;
135+
}
136+
}
137+
}
138+
}
139+
connection["options"] = connectionOptions;
140+
return connection;
141+
}
142+
16143
const addDataSourceButton = document.getElementById("add-data-source-button");
17144
const dataSourceConfigRow = document.getElementById("add-data-source-config-row");
18145
const submitConnectionButton = document.getElementById("submit-connection");
@@ -105,17 +232,51 @@ async function getExistingConnections() {
105232
let connections = respJson.connections;
106233
for (let connection of connections) {
107234
numExistingConnections += 1;
108-
let accordionItem = createAccordionItem(numExistingConnections, connection.name, "", syntaxHighlight(connection));
109-
// add in button to delete connection
235+
236+
// Check if connection is from config (default connection) or file (user-created)
237+
let isFromConfig = connection.source === "config";
238+
let sourceLabel = isFromConfig ? " (default)" : "";
239+
let accordionItem = createAccordionItem(numExistingConnections, connection.name + sourceLabel, "", syntaxHighlight(connection));
240+
241+
// Add test connection button
242+
let testButton = createButton(`connection-test-${connection.name}`, "Test connection", "btn btn-info me-2", "Test");
243+
testButton.setAttribute("title", "Test this connection");
244+
testButton.addEventListener("click", async function () {
245+
await testExistingConnection(connection.name, testButton);
246+
});
247+
248+
// Add delete connection button
110249
let deleteButton = createButton(`connection-delete-${connection.name}`, "Connection delete", "btn btn-danger", "Delete");
250+
251+
// Disable delete button for config-based connections
252+
if (isFromConfig) {
253+
deleteButton.setAttribute("disabled", "true");
254+
deleteButton.setAttribute("title", "Default connections from application.conf cannot be deleted via UI");
255+
deleteButton.classList.add("btn-secondary");
256+
deleteButton.classList.remove("btn-danger");
257+
}
111258

112259
deleteButton.addEventListener("click", async function () {
113-
await apiFetch(`/connection/${connection.name}`, {method: "DELETE"});
114-
accordionConnections.removeChild(accordionItem);
115-
createToast(`${connection.name}`, `Connection ${connection.name} deleted!`, "success");
260+
if (isFromConfig) {
261+
createToast(`${connection.name}`, `Cannot delete default connection. Modify application.conf or environment variables to remove.`, "fail");
262+
return;
263+
}
264+
265+
try {
266+
const response = await apiFetch(`/connection/${connection.name}`, {method: "DELETE"});
267+
if (response.ok) {
268+
accordionConnections.removeChild(accordionItem);
269+
createToast(`${connection.name}`, `Connection ${connection.name} deleted!`, "success");
270+
} else {
271+
const errorText = await response.text();
272+
createToast(`${connection.name}`, `Failed to delete connection: ${errorText}`, "fail");
273+
}
274+
} catch (err) {
275+
createToast(`${connection.name}`, `Failed to delete connection: ${err.message}`, "fail");
276+
}
116277
});
117278

118-
let buttonGroup = createButtonGroup(deleteButton);
279+
let buttonGroup = createButtonGroup(testButton, deleteButton);
119280
let header = accordionItem.querySelector(".accordion-header");
120281
let divContainer = document.createElement("div");
121282
divContainer.setAttribute("class", "d-flex align-items-center");
@@ -149,6 +310,8 @@ function createDataSourceElement(index, hr) {
149310
colName.setAttribute("class", "col");
150311
let colSelect = document.createElement("div");
151312
colSelect.setAttribute("class", "col");
313+
let colTestButton = document.createElement("div");
314+
colTestButton.setAttribute("class", "col-auto");
152315

153316
let dataSourceName = createInput(`data-source-name-${index}`, "Name", "form-control input-field data-source-property", "text", `my-data-source-${index}`);
154317
let formFloatingName = createFormFloating("Name", dataSourceName);
@@ -181,6 +344,20 @@ function createDataSourceElement(index, hr) {
181344
}
182345
}
183346
}
347+
348+
// Create test connection button for new connections
349+
let testButton = createButton(`data-source-test-${index}`, "Test connection", "btn btn-info", "Test Connection");
350+
testButton.setAttribute("title", "Test this connection before saving");
351+
testButton.addEventListener("click", async function () {
352+
const connection = buildConnectionFromContainer(divContainer);
353+
if (!connection.type) {
354+
createToast("Test Connection", "Please select a data source type first", "fail");
355+
return;
356+
}
357+
await testConnection(connection, testButton);
358+
});
359+
colTestButton.append(testButton);
360+
184361
let closeButton = createCloseButton(divContainer);
185362

186363
createDataSourceOptions(dataSourceSelect);
@@ -190,7 +367,7 @@ function createDataSourceElement(index, hr) {
190367
if (hr) {
191368
divContainer.append(hr);
192369
}
193-
divContainer.append(colName, colSelect, closeButton);
370+
divContainer.append(colName, colSelect, colTestButton, closeButton);
194371
return divContainer;
195372
}
196373

0 commit comments

Comments
 (0)