OpenTelemetry

backend/pom.xml

@@ -128,11 +128,7 @@
         </dependency>
         <dependency>
             <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-micrometer</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.quarkus</groupId>
-            <artifactId>quarkus-micrometer-registry-prometheus</artifactId>
+            <artifactId>quarkus-opentelemetry</artifactId>
         </dependency>
     </dependencies>
 

backend/src/main/java/org/cryptomator/hub/api/AuditLogResource.java

@@ -3,6 +3,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import io.opentelemetry.instrumentation.annotations.WithSpan;
 import jakarta.annotation.security.RolesAllowed;
 import jakarta.inject.Inject;
 import jakarta.ws.rs.BadRequestException;
@@ -74,6 +75,7 @@ public class AuditLogResource {
 	@APIResponse(responseCode = "400", description = "startDate or endDate not specified, startDate > endDate, order specified and not in ['asc','desc'], pageSize not in [1 .. 100] or type is not valid")
 	@APIResponse(responseCode = "402", description = "Community license used or license expired")
 	@APIResponse(responseCode = "403", description = "requesting user does not have admin role")
+	@WithSpan("AuditLogResource.getAllEvents")
 	public List<AuditEventDto> getAllEvents(@QueryParam("startDate") Instant startDate, @QueryParam("endDate") Instant endDate, @QueryParam("type") List<String> type, @QueryParam("paginationId") Long paginationId, @QueryParam("order") @DefaultValue("desc") String order, @QueryParam("pageSize") @DefaultValue("20") int pageSize) {
 		if (license.getEntitlements().auditLogRetentionDays() == 0 || license.isExpired()) {
 			throw new PaymentRequiredException("Community license used or license expired");

backend/src/main/java/org/cryptomator/hub/entities/EffectiveGroupMembership.java

@@ -1,5 +1,6 @@
 package org.cryptomator.hub.entities;
 
+import io.opentelemetry.instrumentation.annotations.WithSpan;
 import io.quarkus.hibernate.orm.panache.PanacheRepositoryBase;
 import io.quarkus.panache.common.Parameters;
 import jakarta.enterprise.context.ApplicationScoped;
@@ -87,6 +88,7 @@ public void fullUpdate() {
 					.executeUpdate();
 		}
 
+		@WithSpan("EffectiveGroupMembership.Repository.updateGroups")
 		public void updateGroups(Collection<String> groupIds) {
 			Batch.of(200).run(groupIds, batch -> {
 				getEntityManager()
@@ -100,6 +102,7 @@ public void updateGroups(Collection<String> groupIds) {
 			});
 		}
 
+		@WithSpan("EffectiveGroupMembership.Repository.updateUsers")
 		public void updateUsers(Collection<String> userIds) {
 			Batch.of(200).run(userIds, batch -> {
 				delete("#EffectiveGroupMembership.deleteUsers", Parameters.with("userIds", batch));

backend/src/main/java/org/cryptomator/hub/entities/EffectiveVaultAccess.java

@@ -1,5 +1,6 @@
 package org.cryptomator.hub.entities;
 
+import io.opentelemetry.instrumentation.annotations.WithSpan;
 import io.quarkus.hibernate.orm.panache.PanacheRepositoryBase;
 import io.quarkus.panache.common.Parameters;
 import jakarta.enterprise.context.ApplicationScoped;
@@ -131,6 +132,7 @@ public long countSeatOccupyingUsers() {
 			return count("#EffectiveVaultAccess.countSeatOccupyingUsers");
 		}
 
+		@WithSpan("EffectiveVaultAccess.Repository.countSeatOccupyingUsersWithAccessToken")
 		public long countSeatOccupyingUsersWithAccessToken() {
 			return count("#EffectiveVaultAccess.countSeatOccupyingUsersWithAccessToken");
 		}

backend/src/main/java/org/cryptomator/hub/keycloak/KeycloakAdminService.java

@@ -1,5 +1,6 @@
 package org.cryptomator.hub.keycloak;
 
+import io.opentelemetry.instrumentation.annotations.WithSpan;
 import jakarta.annotation.PostConstruct;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
@@ -61,6 +62,7 @@ public void setup() {
 		this.realm = keycloak.realm(keycloakRealm);
 	}
 
+	@WithSpan("KeycloakAdminService.createUser")
 	public UserRepresentation createUser(String username, String email, String firstName, String lastName, String password, String pictureUrl, Set<String> groupIds) {
 		UserRepresentation user = new UserRepresentation();
 		user.setUsername(username);
@@ -199,6 +201,7 @@ public boolean isUserReadOnly(String userId) {
 
 	// TODO deduplicate with KeycloakAuthorityPuller
 	@Transactional
+	@WithSpan("KeycloakAdminService.syncUser")
 	public User syncUser(String userId) {
 		UserResource userResource = realm.users().get(userId);
 		UserRepresentation keycloakUser = userResource.toRepresentation();
@@ -230,6 +233,7 @@ public User syncUser(String userId) {
 
 	// TODO deduplicate with KeycloakAuthorityPuller
 	@Transactional
+	@WithSpan("KeycloakAdminService.syncGroup")
 	public Group syncGroup(String groupId) {
 		GroupResource groupResource = realm.groups().group(groupId);
 		GroupRepresentation keycloakGroup = groupResource.toRepresentation();
@@ -285,6 +289,7 @@ public void removeUserFromGroup(String groupId, String userId) {
 	}
 
 	@Transactional
+	@WithSpan("KeycloakAdminService.updateUserRoles")
 	public void updateUserRoles(String userId, Set<RealmRole> roles) {
 		// remove roles that are not in the provided set:
 		var rolesToRemove = EnumSet.allOf(RealmRole.class);

backend/src/main/java/org/cryptomator/hub/keycloak/KeycloakAuthorityPuller.java

@@ -1,5 +1,6 @@
 package org.cryptomator.hub.keycloak;
 
+import io.opentelemetry.instrumentation.annotations.WithSpan;
 import io.quarkus.scheduler.Scheduled;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
@@ -30,6 +31,7 @@ public class KeycloakAuthorityPuller {
 	EffectiveGroupMembership.Repository effectiveGroupMembershipRepo;
 
 	@Scheduled(every = "{hub.keycloak.syncer-period}")
+	@WithSpan("KeycloakAuthorityPuller.sync")
 	void sync() {
 		var keycloakGroups = remoteUserProvider.groups().stream().collect(Collectors.toMap(KeycloakGroupDto::id, Function.identity()));
 		var keycloakUsers = remoteUserProvider.users().stream().collect(Collectors.toMap(KeycloakUserDto::id, Function.identity()));

backend/src/main/java/org/cryptomator/hub/license/LicenseHolder.java

@@ -2,6 +2,7 @@
 
 import com.auth0.jwt.exceptions.JWTVerificationException;
 import com.auth0.jwt.interfaces.DecodedJWT;
+import io.opentelemetry.instrumentation.annotations.WithSpan;
 import io.quarkus.scheduler.Scheduled;
 import io.smallrye.common.annotation.RunOnVirtualThread;
 import jakarta.annotation.PostConstruct;
@@ -97,6 +98,7 @@ void init() {
 	 * @throws JWTVerificationException if the license is invalid
 	 */
 	@Transactional
+	@WithSpan("LicenseHolder.ensureLicenseExists")
 	DecodedJWT ensureLicenseExists() throws JWTVerificationException, WebApplicationException {
 		var settings = settingsRepo.get();
 		if (settings.getLicenseKey() != null && settings.getHubId() != null) {
@@ -136,6 +138,7 @@ DecodedJWT validateAndApplyInitLicense(Settings settings, String initialLicenseT
 	}
 
 	@Transactional(Transactional.TxType.MANDATORY)
+	@WithSpan("LicenseHolder.requestAnonTrialLicense")
 	DecodedJWT requestAnonTrialLicense(Settings settings) throws WebApplicationException {
 		LOG.info("No license found. Requesting trial license...");
 		var solution = solveChallenge();

backend/src/main/java/org/cryptomator/hub/metrics/SystemUsageMetrics.java

@@ -1,71 +1,73 @@
 package org.cryptomator.hub.metrics;
 
-import io.micrometer.core.instrument.Gauge;
-import io.micrometer.core.instrument.MeterRegistry;
-import io.quarkus.scheduler.Scheduled;
-import jakarta.annotation.PostConstruct;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.metrics.ObservableLongGauge;
+import io.opentelemetry.api.metrics.ObservableLongMeasurement;
+import io.quarkus.narayana.jta.QuarkusTransaction;
+import io.quarkus.runtime.ShutdownEvent;
 import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
 import jakarta.inject.Inject;
-import jakarta.transaction.Transactional;
 import org.cryptomator.hub.entities.Device;
 import org.cryptomator.hub.entities.EffectiveVaultAccess;
 import org.cryptomator.hub.entities.Vault;
 
-import java.util.EnumMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
 @ApplicationScoped
 public class SystemUsageMetrics {
 
-	private static final String VAULTS_TOTAL_METRIC = "hub_vaults_total";
-	private static final String ACTIVE_USERS_TOTAL_METRIC = "hub_active_users_total";
-	private static final String DEVICES_TOTAL_METRIC = "hub_devices_total";
-
-	@Inject
-	MeterRegistry meterRegistry;
-
-	@Inject
-	Vault.Repository vaultRepo;
+	private static final String VAULTS_METRIC = "hub_vaults";
+	private static final String ACTIVE_USERS_METRIC = "hub_active_users";
+	private static final String DEVICES_METRIC = "hub_devices";
+	private static final AttributeKey<String> DEVICE_TYPE_KEY = AttributeKey.stringKey("type");
 
-	@Inject
-	EffectiveVaultAccess.Repository effectiveVaultAccessRepo;
+	private final Vault.Repository vaultRepo;
+	private final EffectiveVaultAccess.Repository effectiveVaultAccessRepo;
+	private final Device.Repository deviceRepo;
+	private final ObservableLongGauge vaultsGauge;
+	private final ObservableLongGauge activeUsersGauge;
+	private final ObservableLongGauge devicesGauge;
 
 	@Inject
-	Device.Repository deviceRepo;
-
-	private final AtomicLong vaultsTotal = new AtomicLong(0);
-	private final AtomicLong activeUsersTotal = new AtomicLong(0);
-	private final Map<Device.Type, AtomicLong> devicesPerType = new EnumMap<>(Device.Type.class);
+	SystemUsageMetrics(Meter meter, Vault.Repository vaultRepo, EffectiveVaultAccess.Repository effectiveVaultAccessRepo, Device.Repository deviceRepo) {
+		this.vaultRepo = vaultRepo;
+		this.effectiveVaultAccessRepo = effectiveVaultAccessRepo;
+		this.deviceRepo = deviceRepo;
+		this.vaultsGauge = meter.gaugeBuilder(VAULTS_METRIC)
+				.ofLongs()
+				.setDescription("Number of vaults")
+				.buildWithCallback(this::recordVaultCount);
+		this.activeUsersGauge = meter.gaugeBuilder(ACTIVE_USERS_METRIC)
+				.ofLongs()
+				.setDescription("Number of unique users with access to any non-archived vault")
+				.buildWithCallback(this::recordSeatCount);
+		this.devicesGauge = meter.gaugeBuilder(DEVICES_METRIC)
+				.ofLongs()
+				.setDescription("Number of devices grouped by type")
+				.buildWithCallback(this::recordDeviceCount);
+	}
 
-	@PostConstruct
-	void registerMetrics() {
-		Gauge.builder(VAULTS_TOTAL_METRIC, vaultsTotal, AtomicLong::get)
-				.description("Number of vaults")
-				.register(meterRegistry);
+	private void recordVaultCount(ObservableLongMeasurement measurement) {
+		measurement.record(QuarkusTransaction.requiringNew().call(vaultRepo::count));
+	}
 
-		Gauge.builder(ACTIVE_USERS_TOTAL_METRIC, activeUsersTotal, AtomicLong::get)
-				.description("Number of unique users with access to any non-archived vault")
-				.register(meterRegistry);
+	private void recordSeatCount(ObservableLongMeasurement measurement) {
+		measurement.record(QuarkusTransaction.requiringNew().call(effectiveVaultAccessRepo::countSeatOccupyingUsers));
+	}
 
-		for (var deviceType : Device.Type.values()) {
-			var value = new AtomicLong(0);
-			devicesPerType.put(deviceType, value);
-			Gauge.builder(DEVICES_TOTAL_METRIC, value, AtomicLong::get)
-					.description("Number of devices grouped by type")
-					.tag("type", deviceType.name())
-					.register(meterRegistry);
-		}
+	private void recordDeviceCount(ObservableLongMeasurement measurement) {
+		QuarkusTransaction.requiringNew().run(() -> {
+			for (var type : Device.Type.values()) {
+				measurement.record(deviceRepo.count("type", type), Attributes.of(DEVICE_TYPE_KEY, type.name()));
+			}
+		});
 	}
 
-	@Scheduled(every = "24h", delayed = "10s")
-	@Transactional
-	void collect() {
-		vaultsTotal.set(vaultRepo.count());
-		activeUsersTotal.set(effectiveVaultAccessRepo.countSeatOccupyingUsers());
-		for (var deviceType : Device.Type.values()) {
-			var count = deviceRepo.count("type", deviceType);
-			devicesPerType.get(deviceType).set(count);
-		}
+	void onStop(@Observes ShutdownEvent event) {
+		vaultsGauge.close();
+		activeUsersGauge.close();
+		devicesGauge.close();
 	}
+
 }

backend/src/main/java/org/cryptomator/hub/metrics/VaultUnlockMetrics.java

@@ -1,51 +1,47 @@
 package org.cryptomator.hub.metrics;
 
-import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Gauge;
-import io.micrometer.core.instrument.MeterRegistry;
-import jakarta.annotation.PostConstruct;
+import io.opentelemetry.api.metrics.LongCounter;
+import io.opentelemetry.api.metrics.LongGauge;
+import io.opentelemetry.api.metrics.Meter;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
 
-import java.util.concurrent.atomic.AtomicLong;
-
 @ApplicationScoped
 public class VaultUnlockMetrics {
 
 	private static final String UNLOCKS_TOTAL_METRIC = "hub_vault_unlocks_total";
 	private static final String LAST_SUCCESS_METRIC = "hub_vault_unlock_last_success_epoch_seconds";
 	private static final String LAST_FAILURE_METRIC = "hub_vault_unlock_last_failure_epoch_seconds";
 
+	private final LongCounter unlockCounter;
+	private final LongGauge lastSuccessGauge;
+	private final LongGauge lastFailureGauge;
+
 	@Inject
-	MeterRegistry meterRegistry;
-
-	private final AtomicLong lastSuccessEpochSeconds = new AtomicLong(0);
-	private final AtomicLong lastFailureEpochSeconds = new AtomicLong(0);
-	private Counter unlockCounter;
-
-	@PostConstruct
-	void registerGauges() {
-		unlockCounter = Counter.builder(UNLOCKS_TOTAL_METRIC)
-				.description("Total number of vault unlock attempts")
-				.register(meterRegistry);
-		Gauge.builder(LAST_SUCCESS_METRIC, lastSuccessEpochSeconds, AtomicLong::get)
-				.description("Epoch timestamp in seconds of the last successful vault unlock")
-				.register(meterRegistry);
-		Gauge.builder(LAST_FAILURE_METRIC, lastFailureEpochSeconds, AtomicLong::get)
-				.description("Epoch timestamp in seconds of the last failed vault unlock")
-				.register(meterRegistry);
+	VaultUnlockMetrics(Meter meter) {
+		this.unlockCounter = meter.counterBuilder(UNLOCKS_TOTAL_METRIC)
+				.setDescription("Total number of vault unlock attempts")
+				.build();
+		this.lastSuccessGauge = meter.gaugeBuilder(LAST_SUCCESS_METRIC)
+				.ofLongs()
+				.setDescription("Epoch timestamp in seconds of the last successful vault unlock")
+				.build();
+		this.lastFailureGauge = meter.gaugeBuilder(LAST_FAILURE_METRIC)
+				.ofLongs()
+				.setDescription("Epoch timestamp in seconds of the last failed vault unlock")
+				.build();
 	}
 
 	public void recordUnlock() {
-		unlockCounter.increment();
+		unlockCounter.add(1);
 	}
 
 	public void recordSuccess() {
-		lastSuccessEpochSeconds.set(currentEpochSeconds());
+		lastSuccessGauge.set(currentEpochSeconds());
 	}
 
 	public void recordFailure() {
-		lastFailureEpochSeconds.set(currentEpochSeconds());
+		lastFailureGauge.set(currentEpochSeconds());
 	}
 
 	private long currentEpochSeconds() {

backend/src/main/resources/application.properties

@@ -59,10 +59,10 @@ quarkus.swagger-ui.enabled=false
 # Management Interface
 quarkus.management.enabled=true
 quarkus.management.port=9000
-## Metrics
-quarkus.micrometer.enabled=true
-quarkus.micrometer.export.prometheus.enabled=true
-quarkus.datasource.metrics.enabled=true
+# OpenTelemetry. Tracing is on by default; metrics and logs are opt-in.
+quarkus.otel.metrics.enabled=true
+quarkus.otel.logs.enabled=true
+%test.quarkus.otel.sdk.disabled=true
 ## Health
 quarkus.smallrye-health.enabled=true
 quarkus.datasource.health.enabled=true

charts/cryptomator-hub/Chart.yaml

@@ -5,7 +5,7 @@ icon: https://cryptomator.org/img/logo.png
 description: Helm chart for deploying Cryptomator Hub with optional Keycloak and PostgreSQL
 type: application
 version: 0.1.3
-appVersion: "1.5.0-alpha3"
+appVersion: "63024bb"
 kubeVersion: ">=1.27.0-0"
 sources:
   - https://github.com/cryptomator/hub