Backport some GHA changes from trunk (#18428)

Reviewers: Chia-Ping Tsai <chia7712@gmail.com>

.github/actions/run-gradle/action.yml

@@ -16,8 +16,8 @@
 # under the License.
 #
 ---
-name: "Gradle Setup"
-description: "Setup Java and Gradle"
+name: "Run Tests with Gradle"
+description: "Run specified Gradle test tasks with configuration for timeout and test catalog."
 inputs:
   # Composite actions do not support typed parameters. Everything is treated as a string
   # See: https://github.com/actions/runner/issues/2238

.github/scripts/junit.py

@@ -402,10 +402,19 @@ if __name__ == "__main__":
     print("<hr/>")
 
     # Print special message if there was a timeout
-    exit_code = get_env("GRADLE_EXIT_CODE", int)
-    if exit_code == 124:
+    test_exit_code = get_env("GRADLE_TEST_EXIT_CODE", int)
+    quarantined_test_exit_code = get_env("GRADLE_QUARANTINED_TEST_EXIT_CODE", int)
+
+    if test_exit_code == 124 or quarantined_test_exit_code == 124:
+        # Special handling for timeouts. The exit code 124 is emitted by 'timeout' command used in build.yml.
+        # A watchdog script "thread-dump.sh" will use jstack to force a thread dump for any Gradle process
+        # still running after the timeout. We capture the exit codes of the two test tasks and pass them to
+        # this script. If either "test" or "quarantinedTest" fails due to timeout, we want to fail the overall build.
         thread_dump_url = get_env("THREAD_DUMP_URL")
-        logger.debug(f"Gradle command timed out. These are partial results!")
+        if test_exit_code == 124:
+            logger.debug(f"Gradle task for 'test' timed out. These are partial results!")
+        else:
+            logger.debug(f"Gradle task for 'quarantinedTest' timed out. These are partial results!")
         logger.debug(summary)
         if thread_dump_url:
             print(f"\nThe JUnit tests were cancelled due to a timeout. Thread dumps were generated before the job was cancelled. "
@@ -414,7 +423,7 @@ if __name__ == "__main__":
         else:
             logger.debug(f"Failing this step because the tests timed out. Thread dumps were not archived, check logs in JUnit step.")
         exit(1)
-    elif exit_code in (0, 1):
+    elif test_exit_code in (0, 1):
         logger.debug(summary)
         if total_failures > 0:
             logger.debug(f"Failing this step due to {total_failures} test failures")
@@ -425,5 +434,5 @@ if __name__ == "__main__":
         else:
             exit(0)
     else:
-        logger.debug(f"Gradle had unexpected exit code {exit_code}. Failing this step")
+        logger.debug(f"Gradle had unexpected exit code {test_exit_code}. Failing this step")
         exit(1)

.github/workflows/build.yml

@@ -21,14 +21,10 @@ name: Check and Test
 on:
   workflow_call:
     inputs:
-      gradle-cache-read-only:
-        description: "Should the Gradle cache be read-only?"
+      is-trunk:
+        description: "Is this a trunk build?"
         default: true
         type: boolean
-      gradle-cache-write-only:
-        description: "Should the Gradle cache be write-only?"
-        default: false
-        type: boolean
       is-public-fork:
         description: "Is this CI run from a public fork?"
         default: true
@@ -105,8 +101,8 @@ jobs:
         uses: ./.github/actions/setup-gradle
         with:
           java-version: 23
-          gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }}
-          gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }}
+          gradle-cache-read-only: ${{ !inputs.is-trunk }}
+          gradle-cache-write-only: ${{ inputs.is-trunk }}
           develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Compile and validate
         env:
@@ -117,7 +113,7 @@ jobs:
         # --scan:         Publish the build scan. This will only work on PRs from apache/kafka and trunk
         # --no-scan:      For public fork PRs, we won't attempt to publish the scan
         run: |
-          ./gradlew --build-cache --info $SCAN_ARG check -x test
+          ./gradlew --build-cache --info $SCAN_ARG check siteDocTar -x test
       - name: Archive check reports
         if: always()
         uses: actions/upload-artifact@v4
@@ -137,6 +133,16 @@ jobs:
         run: python .github/scripts/rat.py
         env:
           GITHUB_WORKSPACE: ${{ github.workspace }}
+      - name: Check generated documentation
+        # Check if there are any empty files under ./site-docs/generated, If any empty files are found, print an error 
+        # message and list the empty files
+        run: |
+          tar zxvf core/build/distributions/kafka_2.13-$(./gradlew properties | grep version: | awk '{print $NF}' | head -n 1)-site-docs.tgz
+          if find ./site-docs/generated -type f -exec grep -L "." {} \; | grep -q "."; then
+            echo "One or more documentation files are empty!" >&2
+            find ./site-docs/generated -type f -exec grep -L "." {} \; >&2
+            exit 1
+          fi
 
   test:
     needs: [validate, load-catalog]
@@ -146,6 +152,8 @@ jobs:
       fail-fast: false
       matrix:
         java: [ 23, 17 ]  # If we change these, make sure to adjust ci-complete.yml
+    outputs:
+      timed-out: ${{ (steps.junit-test.outputs.gradle-exitcode == '124' || steps.junit-quarantined-test.outputs.gradle-exitcode == '124') }}
     name: JUnit tests Java ${{ matrix.java }}
     steps:
       - name: Checkout code
@@ -158,8 +166,8 @@ jobs:
         uses: ./.github/actions/setup-gradle
         with:
           java-version: ${{ matrix.java }}
-          gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }}
-          gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }}
+          gradle-cache-read-only: ${{ !inputs.is-trunk }}
+          gradle-cache-write-only: ${{ inputs.is-trunk }}
           develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
 
       # If the load-catalog job failed, we won't be able to download the artifact. Since we don't want this to fail
@@ -176,7 +184,7 @@ jobs:
         uses: ./.github/actions/run-gradle
         with:
           test-task: quarantinedTest
-          timeout-minutes: 30
+          timeout-minutes: 180
           test-catalog-path: ${{ steps.load-test-catalog.outputs.download-path }}/combined-test-catalog.txt
           build-scan-artifact-name: build-scan-quarantined-test-${{ matrix.java }}
 
@@ -225,7 +233,8 @@ jobs:
           GITHUB_WORKSPACE: ${{ github.workspace }}
           JUNIT_REPORT_URL: ${{ steps.junit-upload-artifact.outputs.artifact-url }}
           THREAD_DUMP_URL: ${{ steps.thread-dump-upload-artifact.outputs.artifact-url }}
-          GRADLE_EXIT_CODE: ${{ steps.junit-test.outputs.gradle-exitcode }}
+          GRADLE_TEST_EXIT_CODE: ${{ steps.junit-test.outputs.gradle-exitcode }}
+          GRADLE_QUARANTINED_TEST_EXIT_CODE: ${{ steps.junit-quarantined-test.outputs.gradle-exitcode }}
 
       - name: Archive Test Catalog
         if: ${{ always() && matrix.java == '23' }}
@@ -239,7 +248,7 @@ jobs:
   update-test-catalog:
     name: Update Test Catalog
     needs: test
-    if: ${{ always() && !inputs.is-public-fork }}
+    if: ${{ always() && inputs.is-trunk && needs.test.outputs.timed-out == 'false' }}
     runs-on: ubuntu-latest
     permissions:
       contents: write

.github/workflows/ci.yml

@@ -38,8 +38,7 @@ jobs:
   build:
     uses: ./.github/workflows/build.yml
     with:
-      gradle-cache-read-only: ${{ github.ref != 'refs/heads/trunk' }}
-      gradle-cache-write-only: ${{ github.ref == 'refs/heads/trunk' }}
+      is-trunk: ${{ github.ref == 'refs/heads/trunk' }}
       is-public-fork: ${{ github.event.pull_request.head.repo.fork || false }}
     secrets:
       inherit