Merge pull request #61 from scality/improvement/WKBCH-18

WKBCH-18: Fix version detection and update ClickHouse schema

Author: dvasilas

cmd/util.go

@@ -142,8 +142,9 @@ func copyFile(src, dest string) (err error) {
 }
 
 // detectCloudserverVersion extracts the major version from a cloudserver image tag.
-// Returns "v7" for version 7.x images, "v9" for version 9+ images
-// Defaults to "v9" for non-numeric tags (latest, dev, etc.) or when version cannot be determined.
+// Returns "v7" for version 7.x images (e.g., 7.70.77), "v9" for version 9+ images.
+// Only recognizes semver-style tags (digits followed by a dot). Git SHAs and other
+// non-semver tags default to "v9".
 func detectCloudserverVersion(image string) string {
 	parts := strings.Split(image, ":")
 	if len(parts) < 2 || parts[1] == "" {
@@ -159,7 +160,9 @@ func detectCloudserverVersion(image string) string {
 			endIdx++
 		}
 
-		if endIdx > 0 {
+		// Must have a dot after the major version to be considered semver
+		// This prevents git SHAs like "7aae6b6..." from being detected as v7
+		if endIdx > 0 && endIdx < len(tag) && tag[endIdx] == '.' {
 			majorVersionStr := tag[0:endIdx]
 			if majorVersion, err := strconv.Atoi(majorVersionStr); err == nil {
 				if majorVersion == 7 {
@@ -172,6 +175,6 @@ func detectCloudserverVersion(image string) string {
 		}
 	}
 
-	// Default to v9 for non-numeric tags (latest, dev, etc.)
+	// Default to v9 for non-semver tags (latest, dev, git SHAs, etc.)
 	return "v9"
 }

templates/clickhouse/init.d/02-create-ingest-table.sql

@@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS logs.access_logs_ingest
     hostname               LowCardinality(Nullable(String)),
 
     -- AWS access server logs fields https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html
-    startTime              DateTime64(3), -- AWS "Time" field
+    startTime              Int64, -- AWS "Time" field (epoch milliseconds)
     requester              Nullable(String),
     operation              Nullable(String),
     requestURI             Nullable(String),

templates/clickhouse/init.d/03-create-storage-table.sql

@@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS logs.access_logs
     hostname               LowCardinality(Nullable(String)),
 
     -- AWS access server logs fields https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html
-    startTime              DateTime64(3), -- AWS "Time" field
+    startTime              Int64, -- AWS "Time" field (epoch milliseconds)
     requester              Nullable(String),
     operation              Nullable(String),
     requestURI             Nullable(String),

templates/clickhouse/init.d/04-create-offsets-table.sql

@@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS logs.offsets
     bucketName                String,
     raftSessionID             UInt16,
     lastProcessedInsertedAt   DateTime,
-    lastProcessedStartTime    DateTime64(3),
+    lastProcessedStartTime    Int64,
     lastProcessedReqId        String
 )
 ENGINE = ReplacingMergeTree(lastProcessedInsertedAt)

templates/global/values.yaml

@@ -26,7 +26,7 @@ features:
     enabled: false
 
 cloudserver:
-  image: ghcr.io/scality/cloudserver:9.2.14
+  image: ghcr.io/scality/cloudserver:9.2.19
 
 vault:
   image: ghcr.io/scality/vault:7.83.0