Merge branch 'master' into managed-jdbcio

Author: shunping

.asf.yaml

@@ -51,6 +51,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.68: {}
     release-2.67.0-postrelease: {}
     release-2.67: {}
     release-2.66.0-postrelease: {}

CHANGES.md

@@ -510,6 +510,254 @@ tasks.register("pythonFormatterPreCommit") {
   dependsOn("sdks:python:test-suites:tox:pycommon:formatter")
 }
 
+tasks.register("formatChanges") {
+  group = "formatting"
+  description = "Formats CHANGES.md according to the template structure"
+
+  doLast {
+    val changesFile = file("CHANGES.md")
+    if (!changesFile.exists()) {
+      throw GradleException("CHANGES.md file not found")
+    }
+
+    val content = changesFile.readText()
+    val lines = content.lines().toMutableList()
+
+    // Find template end (after --> that follows <!-- Template -->)
+    var templateStartIndex = -1
+    var templateEndIndex = -1
+
+    for (i in lines.indices) {
+      if (lines[i].trim() == "<!-- Template -->") {
+        templateStartIndex = i
+      } else if (templateStartIndex != -1 && lines[i].trim() == "-->") {
+        templateEndIndex = i
+        break
+      }
+    }
+
+    if (templateEndIndex == -1) {
+      throw GradleException("Template end marker not found in CHANGES.md")
+    }
+
+    // Process each release section
+    var i = templateEndIndex + 1
+    val formattedLines = mutableListOf<String>()
+
+    // Keep header and template exactly as-is (lines 0 to templateEndIndex inclusive)
+    formattedLines.addAll(lines.subList(0, templateEndIndex + 1))
+
+    // Always add blank line after template
+    formattedLines.add("")
+
+    while (i < lines.size) {
+      val line = lines[i]
+
+      // Check if this is a release header
+      if (line.startsWith("# [")) {
+        formattedLines.add(line)
+        i++
+
+        // Expected sections in order (following template)
+        val expectedSections = listOf(
+          "## Beam 3.0.0 Development Highlights",
+          "## Highlights",
+          "## I/Os",
+          "## New Features / Improvements",
+          "## Breaking Changes",
+          "## Deprecations",
+          "## Bugfixes",
+          "## Security Fixes",
+          "## Known Issues"
+        )
+
+        val sectionContent = mutableMapOf<String, MutableList<String>>()
+        var currentSection = ""
+
+        // Parse existing sections
+        while (i < lines.size && !lines[i].startsWith("# [")) {
+          val currentLine = lines[i]
+
+          if (currentLine.startsWith("## ")) {
+            currentSection = currentLine
+            if (!sectionContent.containsKey(currentSection)) {
+              sectionContent[currentSection] = mutableListOf()
+            }
+          } else if (currentSection.isNotEmpty()) {
+            sectionContent[currentSection]!!.add(currentLine)
+          }
+          i++
+        }
+
+        // Only add sections that actually exist with content
+        for (section in expectedSections) {
+          if (sectionContent.containsKey(section)) {
+            formattedLines.add("")
+            formattedLines.add(section)
+            formattedLines.add("")
+
+            // Remove empty lines at start and end
+            val content = sectionContent[section]!!
+            while (content.isNotEmpty() && content.first().trim().isEmpty()) {
+              content.removeAt(0)
+            }
+            while (content.isNotEmpty() && content.last().trim().isEmpty()) {
+              content.removeAt(content.size - 1)
+            }
+
+            // Format content according to template rules
+            val formattedContent = content.map { line ->
+              // Convert SDK language references from [Language] to (Language)
+              line.replace(Regex("\\[([^\\]]*(?:Java|Python|Go|Kotlin|TypeScript|YAML)[^\\]]*)\\]")) { matchResult ->
+                val languages = matchResult.groupValues[1]
+                // Only convert if it's clearly a language reference (not a link or other content)
+                if (languages.matches(Regex(".*(?:Java|Python|Go|Kotlin|TypeScript|YAML).*"))) {
+                  "($languages)"
+                } else {
+                  matchResult.value
+                }
+              }
+            }
+
+            formattedLines.addAll(formattedContent)
+          }
+        }
+
+        if (i < lines.size) {
+          formattedLines.add("")
+        }
+      } else {
+        i++
+      }
+    }
+
+    // Write formatted content back
+    changesFile.writeText(formattedLines.joinToString("\n"))
+    println("CHANGES.md has been formatted according to template structure")
+  }
+}
+
+tasks.register("validateChanges") {
+  group = "verification"
+  description = "Validates CHANGES.md follows required formatting rules"
+
+  doLast {
+    val changesFile = file("CHANGES.md")
+    if (!changesFile.exists()) {
+      throw GradleException("CHANGES.md file not found")
+    }
+
+    val content = changesFile.readText()
+    val lines = content.lines()
+    val errors = mutableListOf<String>()
+
+    // Find template section boundaries
+    var templateStartIndex = -1
+    var templateEndIndex = -1
+
+    for (i in lines.indices) {
+      if (lines[i].trim() == "<!-- Template -->") {
+        templateStartIndex = i
+        println("Found template start at line ${i+1}")
+      } else if (templateStartIndex != -1 && lines[i].trim() == "-->") {
+        templateEndIndex = i
+        println("Found template end at line ${i+1}")
+        break
+      }
+    }
+
+    if (templateStartIndex == -1 || templateEndIndex == -1) {
+      throw GradleException("Template section not found in CHANGES.md")
+    }
+
+    println("Template section: lines ${templateStartIndex+1} to ${templateEndIndex+1}")
+
+    // Find unreleased section after the template section
+    var unreleasedSectionStart = -1
+    for (i in (templateEndIndex + 1) until lines.size) {
+      if (lines[i].startsWith("# [") && lines[i].contains("Unreleased")) {
+        unreleasedSectionStart = i
+        println("Found unreleased section at line ${i+1}: ${lines[i]}")
+        break
+      }
+    }
+
+    if (unreleasedSectionStart == -1) {
+      throw GradleException("Unreleased section not found in CHANGES.md")
+    }
+
+    // Check entries in the unreleased section
+    var i = unreleasedSectionStart + 1
+    println("Starting validation from line ${i+1}")
+
+    while (i < lines.size && !lines[i].startsWith("# [")) {
+      val line = lines[i].trim()
+
+      if (line.startsWith("* ") && line.isNotEmpty()) {
+        println("Checking line ${i+1}: $line")
+
+        // Skip comment lines
+        if (line.startsWith("* [comment]:")) {
+          println("  Skipping comment line")
+        } else {
+          // Rule 1: Check if language references use parentheses instead of brackets
+          val languagePattern = "\\[(Java|Python|Go|Kotlin|TypeScript|YAML)(?:/(?:Java|Python|Go|Kotlin|TypeScript|YAML))*\\]"
+          val languageRegex = Regex(languagePattern)
+
+          // Check if there's a language reference in brackets
+          val matches = languageRegex.findAll(line).toList()
+          if (matches.isNotEmpty()) {
+            for (match in matches) {
+              val matchText = match.value
+              val matchPosition = match.range.first
+              println("  Found language reference: $matchText at position $matchPosition")
+
+              // Check if this is part of an issue link or URL
+              val beforeMatch = if (matchPosition > 0) line.substring(0, matchPosition) else ""
+              val isPartOfLink = beforeMatch.contains("[#") ||
+                                beforeMatch.contains("http") ||
+                                line.contains("CVE-")
+
+              println("  Is part of link: $isPartOfLink")
+
+              if (!isPartOfLink) {
+                val error = "Line ${i+1}: Language references should use parentheses () instead of brackets []: $line"
+                println("  Adding error: $error")
+                errors.add(error)
+              }
+            }
+          } else {
+            println("  No bracketed language reference found")
+          }
+
+          // Rule 2: Check if each entry has an issue link
+          val issueLinkPattern = "\\(\\[#[0-9a-zA-Z]+\\]\\(https://github\\.com/apache/beam/issues/[0-9a-zA-Z]+\\)\\)"
+          val issueLinkRegex = Regex(issueLinkPattern)
+
+          val hasIssueLink = issueLinkRegex.containsMatchIn(line)
+          println("  Has issue link: $hasIssueLink")
+
+          if (!hasIssueLink) {
+            val error = "Line ${i+1}: Missing or malformed issue link. Each entry should end with ([#X](https://github.com/apache/beam/issues/X)): $line"
+            println("  Adding error: $error")
+            errors.add(error)
+          }
+        }
+      }
+
+      i++
+    }
+
+    println("Found ${errors.size} errors")
+
+    if (errors.isNotEmpty()) {
+      throw GradleException("CHANGES.md validation failed with the following errors:\n${errors.joinToString("\n")}\n\nYou can run ./gradlew formatChanges to correct some issues.")
+    }
+
+    println("CHANGES.md validation successful")
+  }
+}
+
 tasks.register("python39PostCommit") {
   dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py39:postCommitIT")

build.gradle.kts

@@ -53,7 +53,7 @@ dependencies {
   runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5")                                         // Adds a 'taskTree' task to print task dependency tree
   runtimeOnly("net.linguica.gradle:maven-settings-plugin:0.5")
   runtimeOnly("gradle.plugin.io.pry.gradle.offline_dependencies:gradle-offline-dependencies-plugin:0.5.0") // Enable creating an offline repository
-  runtimeOnly("net.ltgt.gradle:gradle-errorprone-plugin:3.1.0")                                            // Enable errorprone Java static analysis
+  runtimeOnly("net.ltgt.gradle:gradle-errorprone-plugin:4.2.0")                                            // Enable errorprone Java static analysis
   runtimeOnly("org.ajoberstar.grgit:grgit-gradle:5.3.2")                                                   // Enable website git publish to asf-site branch
   runtimeOnly("com.avast.gradle:gradle-docker-compose-plugin:0.16.12")                                     // Enable docker compose tasks
   runtimeOnly("ca.cutterslade.gradle:gradle-dependency-analyze:1.8.3")                                     // Enable dep analysis