KAFKA-17767 Load test catalog data [3/n] (#17654)

Add a new "load-catalog" job to the workflow. This job will checkout the test-catalog branch at 7 days prior and generate a text file of all the tests that were known at that time. This file is then passed down to the two parallel "test" jobs to be used as a source of data for the quarantined test behavior.

Reviewers: Chia-Ping Tsai <chia7712@gmail.com>

.github/actions/setup-python/action.yml

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Python Setup"
+description: "Setup Python and install dependencies"
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+    - name: Pip install
+      shell: bash
+      run: pip install -r .github/scripts/requirements.txt

.github/scripts/format-test-catalog.py

@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from glob import glob
+import logging
+import os
+import sys
+
+import yaml
+
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+handler = logging.StreamHandler(sys.stderr)
+handler.setLevel(logging.DEBUG)
+logger.addHandler(handler)
+
+
+def yaml_to_all_tests(glob_path: str, out_file: str):
+    yamls = glob(pathname=glob_path, recursive=True)
+    logger.debug(f"Found {len(yamls)} YAML files")
+    class_count = 0
+    method_count = 0
+    with open(out_file, "w") as fp:
+        for yaml_file in yamls:
+            with open(yaml_file, "r") as yamp_fp:
+                tests = yaml.safe_load(yamp_fp)
+                for clazz, methods in tests.items():
+                    class_count += 1
+                    for method in methods:
+                        method_count += 1
+                        fp.write(f"{clazz}#{method}\n")
+
+    logger.debug(f"Wrote {method_count} test methods from {class_count} classes to {out_file}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Convert the test catalog to a single text file")
+
+    parser.add_argument("--path",
+                        required=False,
+                        default="test-catalog/**/*.yaml",
+                        help="Path to module YAML files. Glob patterns are supported.")
+    parser.add_argument("--output-file",
+                        required=False,
+                        default="combined-test-catalog.txt",
+                        help="Output file location")
+
+    if not os.getenv("GITHUB_WORKSPACE"):
+        print("This script is intended to by run by GitHub Actions.")
+        exit(1)
+
+    args = parser.parse_args()
+    yaml_to_all_tests(args.path, args.output_file)

.github/workflows/build.yml

@@ -35,6 +35,50 @@ on:
         type: boolean
 
 jobs:
+  load-catalog:
+    runs-on: ubuntu-latest
+    name: Load Test Catalog
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Checkout test-catalog
+        uses: actions/checkout@v4
+        with:
+          ref: 'test-catalog'
+          persist-credentials: false
+          fetch-depth: 100  # Needs to be large enough to ensure we can fetch N days ago
+          path: test-catalog
+
+      - name: Checkout catalog at earlier date
+        run: |
+          cd test-catalog
+          SHA=$(git rev-list -1 --before 7.days.ago origin/test-catalog)
+          echo $SHA
+          git switch --detach $SHA
+          git show --no-patch
+
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
+
+      # Prior to this step, we don't expect any errors. For the rest of this "load-catalog" job, we need to ensure
+      # we do not fail as this will fail the overall workflow.
+      - name: Combine Catalog into single file
+        id: combine-catalog
+        continue-on-error: true
+        run: |
+          python .github/scripts/format-test-catalog.py --path "test-catalog/test-catalog/**/*.yaml"
+
+      - name: Archive Combined Test Catalog
+        if: steps.combine-catalog.outcome == 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: combined-test-catalog
+          path: combined-test-catalog.txt
+          compression-level: 9
+
   validate:
     runs-on: ubuntu-latest
     name: Compile and Check Java
@@ -55,9 +99,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Setup Gradle
         uses: ./.github/actions/setup-gradle
         with:
@@ -96,7 +139,7 @@ jobs:
           GITHUB_WORKSPACE: ${{ github.workspace }}
 
   test:
-    needs: validate
+    needs: [validate, load-catalog]
     if: ${{ ! needs.validate.outputs.is-draft }}
     runs-on: ubuntu-latest
     strategy:
@@ -109,10 +152,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-      - run: pip install -r .github/scripts/requirements.txt
+      - name: Setup Python
+        uses: ./.github/actions/setup-python
       - name: Setup Gradle
         uses: ./.github/actions/setup-gradle
         with:
@@ -120,6 +161,15 @@ jobs:
           gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }}
           gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }}
           develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
+
+      # If the load-catalog job failed, we won't be able to download the artifact. Since we don't want this to fail
+      # the overall workflow, so we'll continue here without a test catalog.
+      - name: Load Test Catalog
+        uses: actions/download-artifact@v4
+        continue-on-error: true
+        with:
+          name: combined-test-catalog
+
       - name: Test
         # Gradle flags
         # --build-cache:  Let Gradle restore the build cache