chore: merge release into master (#3879)

Author: thoniTUB

.github/workflows/create-image.yml

@@ -39,15 +39,15 @@ jobs:
           # Fetch all tags
           fetch-depth: 0
       - name: Log in to the Container registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
@@ -81,15 +81,15 @@ jobs:
           fetch-depth: 0
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |

.github/workflows/test_cypress.yml

@@ -58,14 +58,14 @@ jobs:
           start: bash ./scripts/run_e2e_all.sh
           wait-on: "http://localhost:8000"
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v5
         if: failure()
         with:
           name: cypress-screenshots
           path: cypress/screenshots
           if-no-files-found: ignore
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v5
         if: failure()
         with:
           name: cypress-videos

autodoc/src/main/java/com/bakdata/conquery/Constants.java

@@ -38,7 +38,7 @@
 import com.bakdata.conquery.models.config.CSVConfig;
 import com.bakdata.conquery.models.config.ClusterConfig;
 import com.bakdata.conquery.models.config.ConqueryConfig;
-import com.bakdata.conquery.models.config.DatabaseConfig;
+import com.bakdata.conquery.models.config.DatabaseConnectionConfig;
 import com.bakdata.conquery.models.config.Dialect;
 import com.bakdata.conquery.models.config.FrontendConfig;
 import com.bakdata.conquery.models.config.LocaleConfig;
@@ -129,7 +129,7 @@ public class Constants {
 				 .otherClass(MinaConfig.class)
 				 .otherClass(FrontendConfig.CurrencyConfig.class)
 				 .otherClass(XodusConfig.class)
-				 .otherClasses(List.of(SqlConnectorConfig.class, DatabaseConfig.class, Dialect.class))
+				 .otherClasses(List.of(SqlConnectorConfig.class, DatabaseConnectionConfig.class, Dialect.class))
 				 .hide(Charset.class)
 				 .hide(Currency.class)
 				 .hide(InetAddress.class)

backend/pom.xml

@@ -13,7 +13,7 @@
     <properties>
         <mock-server.version>5.11.1</mock-server.version>
         <apache-poi.version>5.4.0</apache-poi.version>
-		<apache-arrow.version>16.0.0</apache-arrow.version>
+        <apache-arrow.version>16.0.0</apache-arrow.version>
         <apache-parquet.version>1.12.3</apache-parquet.version>
         <prometheus-metrics.version>1.3.5</prometheus-metrics.version>
         <openapi-generator.version>7.9.0</openapi-generator.version>
@@ -220,7 +220,7 @@
         <dependency>
             <groupId>org.apache.shiro</groupId>
             <artifactId>shiro-core</artifactId>
-            <version>2.0.2</version>
+            <version>2.1.0</version>
             <exclusions>
                 <!-- All these transitive deps are already bundled in shiro core-->
                 <exclusion>
@@ -361,25 +361,21 @@
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers</artifactId>
-            <version>1.21.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <version>1.20.6</version>
+            <artifactId>testcontainers-junit-jupiter</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>postgresql</artifactId>
-            <version>1.20.1</version>
+            <artifactId>testcontainers-postgresql</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>solr</artifactId>
-            <version>1.20.6</version>
+            <artifactId>testcontainers-solr</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -431,7 +427,7 @@
         <dependency>
             <groupId>org.apache.solr</groupId>
             <artifactId>solr-solrj</artifactId>
-            <version>9.8.0</version>
+            <version>9.10.1</version>
         </dependency>
     </dependencies>
 </project>

backend/src/main/java/com/bakdata/conquery/Conquery.java

@@ -1,5 +1,6 @@
 package com.bakdata.conquery;
 
+import java.time.Clock;
 import jakarta.validation.Validator;
 
 import ch.qos.logback.classic.Level;
@@ -39,6 +40,10 @@ public class Conquery extends Application<ConqueryConfig> {
 
 	private final String name;
 
+	protected Clock getQueryClock() {
+		return Clock.systemDefaultZone();
+	}
+
 	public Conquery() {
 		this("Conquery");
 	}
@@ -103,7 +108,7 @@ protected Level bootstrapLogLevel() {
 	@Override
 	public void run(ConqueryConfig configuration, Environment environment) throws Exception {
 		ManagerProvider provider = configuration.getSqlConnectorConfig().isEnabled() ?
-								   new LocalManagerProvider() : new ClusterManagerProvider();
+								   new LocalManagerProvider(getQueryClock()) : new ClusterManagerProvider();
 		Manager manager = provider.provideManager(configuration, environment);
 
 		ManagerNode managerNode = new ManagerNode();

backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java

@@ -1,6 +1,7 @@
 package com.bakdata.conquery.commands;
 
 import java.nio.file.Path;
+import java.time.Clock;
 import java.util.List;
 import java.util.Vector;
 
@@ -13,6 +14,7 @@
 import io.dropwizard.core.cli.ServerCommand;
 import io.dropwizard.core.setup.Environment;
 import lombok.Getter;
+import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import net.sourceforge.argparse4j.inf.Namespace;
 
@@ -24,6 +26,9 @@ public class DistributedStandaloneCommand extends ServerCommand<ConqueryConfig>
 	private final List<ShardNode> shardNodes = new Vector<>();
 	private ClusterManager manager;
 
+	@Setter
+	private Clock clock = Clock.systemDefaultZone();
+
 	public DistributedStandaloneCommand() {
 		super(new NoOpConquery(), "standalone", "starts a manager node and shard node(s) at the same time in a single JVM.");
 	}
@@ -46,6 +51,7 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig
 		for (int id = 0; id < configuration.getStandalone().getNumberOfShardNodes(); id++) {
 
 			ShardNode sc = new ShardNode(ShardNode.DEFAULT_NAME + id);
+			sc.setClock(clock);
 
 			shardNodes.add(sc);
 

backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java

@@ -1,5 +1,6 @@
 package com.bakdata.conquery.commands;
 
+import java.time.Clock;
 import java.util.Collection;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -36,6 +37,8 @@ public class ShardNode implements ConfiguredBundle<ConqueryConfig> {
 	@Setter
 	private ShardWorkers workers;
 	private ClusterConnectionShard clusterConnection;
+	@Setter
+	private Clock clock = Clock.systemDefaultZone();
 
 	public ShardNode() {
 		this(DEFAULT_NAME);
@@ -55,7 +58,8 @@ public void run(ConqueryConfig config, Environment environment) throws Exception
 		workers = new ShardWorkers(
 				config.getQueries().getExecutionPool(),
 				internalMapperFactory,
-				config.getQueries().getSecondaryIdSubPlanRetention()
+				config.getQueries().getSecondaryIdSubPlanRetention(),
+				clock
 		);
 
 		lifecycle.manage(workers);

backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java

@@ -1,5 +1,6 @@
 package com.bakdata.conquery.commands;
 
+import java.time.Clock;
 import java.util.List;
 
 import com.bakdata.conquery.mode.Manager;
@@ -15,4 +16,7 @@ public interface StandaloneCommand {
 
 	Environment getEnvironment();
 
+	void setClock(Clock clock);
+	Clock getClock();
+
 }

backend/src/main/java/com/bakdata/conquery/mode/local/ConnectionManager.java

@@ -0,0 +1,30 @@
+package com.bakdata.conquery.mode.local;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import com.bakdata.conquery.models.datasets.Dataset;
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.Getter;
+import org.jooq.DSLContext;
+
+@Data
+public class ConnectionManager {
+	@Getter(AccessLevel.NONE)
+	private final Map<String, ManagedConnection> connections = new HashMap<>();
+
+	public void addConnection(String name, ManagedConnection connection) {
+		connections.put(name, connection);
+	}
+
+	public DSLContext connect(Dataset dataset) {
+		return getConnection(dataset).connect();
+	}
+
+	public ManagedConnection getConnection(Dataset dataset) {
+		return Objects.requireNonNull(connections.get(dataset.getDataSource()), () -> "No connection available for datset=%s, dataSource=%s".formatted(dataset.getName(), dataset.getDataSource()));
+	}
+
+}

backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java

@@ -1,5 +1,6 @@
 package com.bakdata.conquery.mode.local;
 
+import java.time.Clock;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -14,31 +15,25 @@
 import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.models.worker.LocalNamespace;
 import com.bakdata.conquery.models.worker.ShardNodeInformation;
-import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory;
 import io.dropwizard.core.setup.Environment;
+import lombok.RequiredArgsConstructor;
 
+@RequiredArgsConstructor
 public class LocalManagerProvider implements ManagerProvider {
 
 	private static final Supplier<Collection<ShardNodeInformation>> EMPTY_NODE_PROVIDER = Collections::emptyList;
+	private final Clock clock;
 
-	private final SqlDialectFactory dialectFactory;
-
-	public LocalManagerProvider() {
-		this.dialectFactory = new SqlDialectFactory();
-	}
-
-	public LocalManagerProvider(SqlDialectFactory dialectFactory) {
-		this.dialectFactory = dialectFactory;
-	}
-
+	@Override
 	public DelegateManager<LocalNamespace> provideManager(ConqueryConfig config, Environment environment) {
 
+		final ConnectionManager connectionManager = config.getSqlConnectorConfig().toConnectionManager(environment);
+
 		final MetaStorage storage = new MetaStorage(config.getStorage());
 		final InternalMapperFactory internalMapperFactory = new InternalMapperFactory(config, environment.getValidator());
-		final NamespaceHandler<LocalNamespace> namespaceHandler = new LocalNamespaceHandler(config, internalMapperFactory, dialectFactory);
+		final NamespaceHandler<LocalNamespace> namespaceHandler = new LocalNamespaceHandler(config, internalMapperFactory, connectionManager, clock);
 		final DatasetRegistry<LocalNamespace> datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, internalMapperFactory);
 
-
 		return new DelegateManager<>(
 				config,
 				environment,