KAFKA-7844: Use regular subproject for generator to fix *All targets (#6182)

The presence of the buildSrc subproject is causing problems when we try
to run installAll, jarAll, and the other "all" targets. It's easier
just to make the generator code a regular subproject and use the
JavaExec gradle task to run the code. This also makes it more
straightforward to run the generator unit tests.

Reviewers: David Arthur <mumrah@gmail.com>, Ismael Juma <ismael@juma.me.uk>

Co-authored-by: Colin P. Mccabe <cmccabe@confluent.io>
Co-authored-by: Stanislav Kozlovski <stanislav_kozlovski@outlook.com>

build.gradle

@@ -249,10 +249,11 @@ subprojects {
       exceptionFormat = testExceptionFormat
     }
 
+    if (it.project.name != 'generator') {
       useJUnit {
         excludeCategories 'org.apache.kafka.test.IntegrationTest'
       }
-    
+    }
   }
 
   jar {
@@ -823,6 +824,23 @@ project(':examples') {
   }
 }
 
+project(':generator') {
+  dependencies {
+    compile libs.jacksonDatabind
+    compile libs.jacksonJDK8Datatypes
+    compile libs.jacksonJaxrsJsonProvider
+    testCompile libs.junit
+  }
+
+  integrationTest {
+    enabled = false
+  }
+
+  javadoc {
+    enabled = false
+  }
+}
+
 project(':clients') {
   archivesBaseName = "kafka-clients"
 
@@ -899,9 +917,12 @@ project(':clients') {
     delete "$buildDir/kafka/"
   }
 
-  task processMessages(type:org.apache.kafka.task.ProcessMessagesTask) {
-    inputDirectory = file("src/main/resources/common/message")
-    outputDirectory = file("src/generated/java/org/apache/kafka/common/message")
+  task processMessages(type:JavaExec) {
+    main = "org.apache.kafka.message.MessageGenerator"
+    classpath = project(':generator').sourceSets.main.runtimeClasspath
+    args = [ "src/generated/java/org/apache/kafka/common/message", "src/main/resources/common/message" ]
+    inputs.dir("src/main/resources/common/message")
+    outputs.dir("src/generated/java/org/apache/kafka/common/message")
   }
 
   sourceSets {
@@ -912,8 +933,7 @@ project(':clients') {
     }
     test {
       java {
-        srcDirs = ["src/generated/java", "src/test/java",
-            "$rootDir/buildSrc/src/main/java/org/apache/kafka/message/"]
+        srcDirs = ["src/generated/java", "src/test/java"]
       }
     }
   }

buildSrc/build.gradle

@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-repositories {
-  mavenCentral()
-}
-
-dependencies {
-  compile "com.fasterxml.jackson.core:jackson-databind:2.9.6"
-  compile "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.6"
-  compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.9.6"
-}
-
-test.enabled=false

buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java

@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kafka.task;
-
-import org.apache.kafka.message.MessageGenerator;
-import org.gradle.api.DefaultTask;
-import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.OutputFiles;
-import org.gradle.api.tasks.TaskAction;
-
-import java.io.File;
-import java.util.Map;
-
-/**
- * A gradle task which processes a directory full of JSON files into an output directory.
- */
-public class ProcessMessagesTask extends DefaultTask {
-    /**
-     * The directory where we should read the input JSON from.
-     */
-    public File inputDirectory;
-
-    /**
-     * The directory that we should write output JSON to.
-     */
-    public File outputDirectory;
-
-    @InputDirectory
-    public File getInputDirectory() {
-        return inputDirectory;
-    }
-
-    /**
-     * Define the task outputs.
-     *
-     * Gradle consults this to see if the task is up-to-date.
-     */
-    @OutputFiles
-    public Map<String, File> getOutputFiles() throws Exception {
-        return MessageGenerator.getOutputFiles(
-            outputDirectory.toString(), inputDirectory.toString());
-    }
-
-    @TaskAction
-    public void run() {
-        try {
-            MessageGenerator.processDirectories(
-                outputDirectory.toString(), inputDirectory.toString());
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-}

generator/src/main/java/org/apache/kafka/message/MessageGenerator.java

@@ -24,14 +24,12 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 
 import java.io.BufferedWriter;
-import java.io.File;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Locale;
-import java.util.Map;
 
 /**
  * The Kafka message generator.
@@ -94,25 +92,14 @@ public final class MessageGenerator {
         JSON_SERDE.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
     }
 
-    public static Map<String, File> getOutputFiles(String outputDir, String inputDir) throws Exception {
-        HashMap<String, File> outputFiles = new HashMap<>();
-        for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
-            String jsonName = inputPath.getFileName().toString();
-            String javaName = jsonName.substring(0, jsonName.length() - JSON_SUFFIX.length()) + "Data.java";
-            File outputFile = new File(outputDir, javaName);
-            outputFiles.put(outputFile.toString(), outputFile);
-        }
-        File factoryFile = new File(outputDir, API_MESSAGE_FACTORY_JAVA);
-        outputFiles.put(factoryFile.toString(), factoryFile);
-        return outputFiles;
-    }
-
     public static void processDirectories(String outputDir, String inputDir) throws Exception {
         Files.createDirectories(Paths.get(outputDir));
         int numProcessed = 0;
         ApiMessageFactoryGenerator messageFactoryGenerator = new ApiMessageFactoryGenerator();
         HashSet<String> outputFileNames = new HashSet<>();
-        for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
+        try (DirectoryStream<Path> directoryStream = Files
+                .newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
+            for (Path inputPath : directoryStream) {
                 try {
                     MessageSpec spec = JSON_SERDE.
                         readValue(inputPath.toFile(), MessageSpec.class);
@@ -130,6 +117,7 @@ public final class MessageGenerator {
                     throw new RuntimeException("Exception while processing " + inputPath.toString(), e);
                 }
             }
+        }
         Path factoryOutputPath = Paths.get(outputDir, API_MESSAGE_FACTORY_JAVA);
         outputFileNames.add(API_MESSAGE_FACTORY_JAVA);
         try (BufferedWriter writer = Files.newBufferedWriter(factoryOutputPath)) {
@@ -137,11 +125,17 @@ public final class MessageGenerator {
             messageFactoryGenerator.write(writer);
         }
         numProcessed++;
-        for (Path outputPath : Files.newDirectoryStream(Paths.get(outputDir))) {
-            if (!outputFileNames.contains(outputPath.getFileName().toString())) {
+        try (DirectoryStream<Path> directoryStream = Files.
+                newDirectoryStream(Paths.get(outputDir))) {
+            for (Path outputPath : directoryStream) {
+                Path fileName = outputPath.getFileName();
+                if (fileName != null) {
+                    if (!outputFileNames.contains(fileName.toString())) {
                         Files.delete(outputPath);
                     }
                 }
+            }
+        }
         System.out.printf("MessageGenerator: processed %d Kafka message JSON files(s).%n", numProcessed);
     }
 

settings.gradle

@@ -17,4 +17,4 @@ include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:streams-scal
         'streams:upgrade-system-tests-0100', 'streams:upgrade-system-tests-0101', 'streams:upgrade-system-tests-0102',
         'streams:upgrade-system-tests-0110', 'streams:upgrade-system-tests-10', 'streams:upgrade-system-tests-11', 'streams:upgrade-system-tests-20',
         'streams:upgrade-system-tests-21' , 'log4j-appender', 'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file',
-        'connect:basic-auth-extension', 'jmh-benchmarks'
+        'connect:basic-auth-extension', 'jmh-benchmarks', 'generator'