diff --git a/CHANGES.txt b/CHANGES.txt index eda42deec..b705075ae 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,6 @@ 0.5.0 ----- + * Fix the spurious oldest segment age in CdcRawDirectorySpaceCleaner (CASSSIDECAR-484) * Implement durable operational job tracker (CASSSIDECAR-374) * Remove filesystem path from Http response (CASSSIDECAR-477) * Add basic configuration retrieval logic to ConfigurationManager (CASSSIDECAR-427) diff --git a/server/src/main/java/org/apache/cassandra/sidecar/tasks/CdcRawDirectorySpaceCleaner.java b/server/src/main/java/org/apache/cassandra/sidecar/tasks/CdcRawDirectorySpaceCleaner.java index 03ed40a14..fca6c370a 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/tasks/CdcRawDirectorySpaceCleaner.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/tasks/CdcRawDirectorySpaceCleaner.java @@ -232,8 +232,13 @@ protected void cleanUpCdcRawDirectory(File cdcRawDirectory) Collections.sort(segmentFiles); long nowInMillis = timeProvider.currentTimeMillis(); - // track the age of the oldest commit log segment to give indication of the time-window buffer available - cdcMetrics.oldestSegmentAge.metric.setValue((int) MILLISECONDS.toSeconds(nowInMillis - segmentFiles.get(0).lastModified())); + // track the age of the oldest commit log segment to give indication of the time-window buffer available. + // Skip emission if lastModified is 0 (file was deleted before we could snapshot its lastModified time). + long oldestLastModified = segmentFiles.get(0).lastModified(); + if (oldestLastModified > 0) + { + cdcMetrics.oldestSegmentAge.metric.setValue((int) MILLISECONDS.toSeconds(nowInMillis - oldestLastModified)); + } LOGGER.debug("Cdc data cleaner directorySizeBytes={} maxedUsageBytes={} upperLimitBytes={}", directorySizeBytes, maxUsageBytes, upperLimitBytes); @@ -254,31 +259,47 @@ protected void cleanUpCdcRawDirectory(File cdcRawDirectory) while (i < segmentFiles.size() - 1 && directorySizeBytes > upperLimitBytes) { CdcRawSegmentFile segment = segmentFiles.get(i); - long ageMillis = nowInMillis - segment.lastModified(); - - if (ageMillis < criticalMillis) - { - LOGGER.error("Insufficient Cdc buffer size to maintain {}-minute window segment={} maxSize={} ageMinutes={}", - MILLISECONDS.toMinutes(criticalMillis), segment, upperLimitBytes, - MILLISECONDS.toMinutes(ageMillis)); - cdcMetrics.criticalCdcRawSpace.metric.update(1); - } - else if (ageMillis < lowMillis) - { - LOGGER.warn("Insufficient Cdc buffer size to maintain {}-minute window segment={} maxSize={} ageMinutes={}", - MILLISECONDS.toMinutes(lowMillis), segment, upperLimitBytes, - MILLISECONDS.toMinutes(ageMillis)); - cdcMetrics.lowCdcRawSpace.metric.update(1); - } + long segmentLastModified = segment.lastModified(); long length = 0; - try + // When lastModified is 0 the segment was already reclaimed between the directory + // Skip the buffer-window alerts and the metrics update but still discount the cached size + // from the local budget so the outer loop stops at the right point instead of over-cleaning + // subsequent live segments. + if (segmentLastModified > 0) { - length = deleteSegment(segment); - cdcMetrics.deletedSegment.metric.update(length); + long ageMillis = nowInMillis - segmentLastModified; + + if (ageMillis < criticalMillis) + { + LOGGER.error("Insufficient Cdc buffer size to maintain {}-minute window segment={} maxSize={} ageMinutes={}", + MILLISECONDS.toMinutes(criticalMillis), segment, upperLimitBytes, + MILLISECONDS.toMinutes(ageMillis)); + cdcMetrics.criticalCdcRawSpace.metric.update(1); + } + else if (ageMillis < lowMillis) + { + LOGGER.warn("Insufficient Cdc buffer size to maintain {}-minute window segment={} maxSize={} ageMinutes={}", + MILLISECONDS.toMinutes(lowMillis), segment, upperLimitBytes, + MILLISECONDS.toMinutes(ageMillis)); + cdcMetrics.lowCdcRawSpace.metric.update(1); + } + + try + { + length = deleteSegment(segment); + cdcMetrics.deletedSegment.metric.update(length); + } + catch (IOException e) + { + LOGGER.warn("Failed to delete cdc segment", e); + } } - catch (IOException e) + else { - LOGGER.warn("Failed to delete cdc segment", e); + LOGGER.warn("Skipping delete for already-reclaimed cdc segment {}; a concurrent reclaim " + + "(e.g. Cassandra reclaiming cdc_raw) occurred between the directory scan and " + + "the deletion pass", segment); + length = segment.length(); } directorySizeBytes -= length; i++; @@ -398,6 +419,10 @@ protected static class CdcRawSegmentFile implements Comparable