diff --git a/.github/actions/run-gradle/action.yml b/.github/actions/run-gradle/action.yml index a5bc4e55237..18d6bdeb1f5 100644 --- a/.github/actions/run-gradle/action.yml +++ b/.github/actions/run-gradle/action.yml @@ -16,8 +16,8 @@ # under the License. # --- -name: "Gradle Setup" -description: "Setup Java and Gradle" +name: "Run Tests with Gradle" +description: "Run specified Gradle test tasks with configuration for timeout and test catalog." inputs: # Composite actions do not support typed parameters. Everything is treated as a string # See: https://github.com/actions/runner/issues/2238 diff --git a/.github/scripts/junit.py b/.github/scripts/junit.py index 142e6c09076..b23f444720b 100644 --- a/.github/scripts/junit.py +++ b/.github/scripts/junit.py @@ -402,10 +402,19 @@ if __name__ == "__main__": print("
") # Print special message if there was a timeout - exit_code = get_env("GRADLE_EXIT_CODE", int) - if exit_code == 124: + test_exit_code = get_env("GRADLE_TEST_EXIT_CODE", int) + quarantined_test_exit_code = get_env("GRADLE_QUARANTINED_TEST_EXIT_CODE", int) + + if test_exit_code == 124 or quarantined_test_exit_code == 124: + # Special handling for timeouts. The exit code 124 is emitted by 'timeout' command used in build.yml. + # A watchdog script "thread-dump.sh" will use jstack to force a thread dump for any Gradle process + # still running after the timeout. We capture the exit codes of the two test tasks and pass them to + # this script. If either "test" or "quarantinedTest" fails due to timeout, we want to fail the overall build. thread_dump_url = get_env("THREAD_DUMP_URL") - logger.debug(f"Gradle command timed out. These are partial results!") + if test_exit_code == 124: + logger.debug(f"Gradle task for 'test' timed out. These are partial results!") + else: + logger.debug(f"Gradle task for 'quarantinedTest' timed out. These are partial results!") logger.debug(summary) if thread_dump_url: print(f"\nThe JUnit tests were cancelled due to a timeout. Thread dumps were generated before the job was cancelled. " @@ -414,7 +423,7 @@ if __name__ == "__main__": else: logger.debug(f"Failing this step because the tests timed out. Thread dumps were not archived, check logs in JUnit step.") exit(1) - elif exit_code in (0, 1): + elif test_exit_code in (0, 1): logger.debug(summary) if total_failures > 0: logger.debug(f"Failing this step due to {total_failures} test failures") @@ -425,5 +434,5 @@ if __name__ == "__main__": else: exit(0) else: - logger.debug(f"Gradle had unexpected exit code {exit_code}. Failing this step") + logger.debug(f"Gradle had unexpected exit code {test_exit_code}. Failing this step") exit(1) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e860073ecdd..d878a52ddac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,14 +21,10 @@ name: Check and Test on: workflow_call: inputs: - gradle-cache-read-only: - description: "Should the Gradle cache be read-only?" + is-trunk: + description: "Is this a trunk build?" default: true type: boolean - gradle-cache-write-only: - description: "Should the Gradle cache be write-only?" - default: false - type: boolean is-public-fork: description: "Is this CI run from a public fork?" default: true @@ -105,8 +101,8 @@ jobs: uses: ./.github/actions/setup-gradle with: java-version: 23 - gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }} - gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }} + gradle-cache-read-only: ${{ !inputs.is-trunk }} + gradle-cache-write-only: ${{ inputs.is-trunk }} develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }} - name: Compile and validate env: @@ -117,7 +113,7 @@ jobs: # --scan: Publish the build scan. This will only work on PRs from apache/kafka and trunk # --no-scan: For public fork PRs, we won't attempt to publish the scan run: | - ./gradlew --build-cache --info $SCAN_ARG check -x test + ./gradlew --build-cache --info $SCAN_ARG check siteDocTar -x test - name: Archive check reports if: always() uses: actions/upload-artifact@v4 @@ -137,6 +133,16 @@ jobs: run: python .github/scripts/rat.py env: GITHUB_WORKSPACE: ${{ github.workspace }} + - name: Check generated documentation + # Check if there are any empty files under ./site-docs/generated, If any empty files are found, print an error + # message and list the empty files + run: | + tar zxvf core/build/distributions/kafka_2.13-$(./gradlew properties | grep version: | awk '{print $NF}' | head -n 1)-site-docs.tgz + if find ./site-docs/generated -type f -exec grep -L "." {} \; | grep -q "."; then + echo "One or more documentation files are empty!" >&2 + find ./site-docs/generated -type f -exec grep -L "." {} \; >&2 + exit 1 + fi test: needs: [validate, load-catalog] @@ -146,6 +152,8 @@ jobs: fail-fast: false matrix: java: [ 23, 17 ] # If we change these, make sure to adjust ci-complete.yml + outputs: + timed-out: ${{ (steps.junit-test.outputs.gradle-exitcode == '124' || steps.junit-quarantined-test.outputs.gradle-exitcode == '124') }} name: JUnit tests Java ${{ matrix.java }} steps: - name: Checkout code @@ -158,8 +166,8 @@ jobs: uses: ./.github/actions/setup-gradle with: java-version: ${{ matrix.java }} - gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }} - gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }} + gradle-cache-read-only: ${{ !inputs.is-trunk }} + gradle-cache-write-only: ${{ inputs.is-trunk }} develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }} # If the load-catalog job failed, we won't be able to download the artifact. Since we don't want this to fail @@ -176,7 +184,7 @@ jobs: uses: ./.github/actions/run-gradle with: test-task: quarantinedTest - timeout-minutes: 30 + timeout-minutes: 180 test-catalog-path: ${{ steps.load-test-catalog.outputs.download-path }}/combined-test-catalog.txt build-scan-artifact-name: build-scan-quarantined-test-${{ matrix.java }} @@ -225,7 +233,8 @@ jobs: GITHUB_WORKSPACE: ${{ github.workspace }} JUNIT_REPORT_URL: ${{ steps.junit-upload-artifact.outputs.artifact-url }} THREAD_DUMP_URL: ${{ steps.thread-dump-upload-artifact.outputs.artifact-url }} - GRADLE_EXIT_CODE: ${{ steps.junit-test.outputs.gradle-exitcode }} + GRADLE_TEST_EXIT_CODE: ${{ steps.junit-test.outputs.gradle-exitcode }} + GRADLE_QUARANTINED_TEST_EXIT_CODE: ${{ steps.junit-quarantined-test.outputs.gradle-exitcode }} - name: Archive Test Catalog if: ${{ always() && matrix.java == '23' }} @@ -239,7 +248,7 @@ jobs: update-test-catalog: name: Update Test Catalog needs: test - if: ${{ always() && !inputs.is-public-fork }} + if: ${{ always() && inputs.is-trunk && needs.test.outputs.timed-out == 'false' }} runs-on: ubuntu-latest permissions: contents: write diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0d531f267d..c02c47c7584 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,7 @@ jobs: build: uses: ./.github/workflows/build.yml with: - gradle-cache-read-only: ${{ github.ref != 'refs/heads/trunk' }} - gradle-cache-write-only: ${{ github.ref == 'refs/heads/trunk' }} + is-trunk: ${{ github.ref == 'refs/heads/trunk' }} is-public-fork: ${{ github.event.pull_request.head.repo.fork || false }} secrets: inherit