KAFKA-18142 Switch to `com.gradleup.shadow` (#18018)

**Prologue (related PR's):**
-  https://github.com/apache/kafka/pull/16295
-  https://github.com/apache/kafka/pull/17218
-  https://github.com/apache/kafka/pull/16489 

**Action points:**
- switch shadow plugin from `io.github.goooler.shadow` to
`com.gradleup.shadow`
- upgrade plugin version from **8.1.3** to **8.3.5** (release notes:
https://gradleup.com/shadow/changes/#v8-3-5-2024-11-03)

**Rationale:** both previous one and currently used shadow plugins are
now deprecated (in favor of `com.gradleup.shadow`):
- `com.github.johnrengelman.shadow` maintenance was transferred to
`com.gradleup.shadow`:
https://github.com/GradleUp/shadow/tree/8.3.5?tab=readme-ov-file#gradle-shadow
- `io.github.goooler.shadow`: changes are ported to
`com.gradleup.shadow`:
https://github.com/Goooler/shadow?tab=readme-ov-file#gradle-shadow

**Pitfall (to keep in mind):** 
- https://github.com/apache/kafka/pull/15532 

Co-authored-by: Goooler <wangzongler@gmail.com>
Reviewers: Apoorv Mittal <apoorvmittal10@gmail.com>, David Arthur <mumrah@gmail.com>

build.gradle

@@ -41,7 +41,7 @@ plugins {
 
   id "com.github.spotbugs" version '6.0.25' apply false
   id 'org.scoverage' version '8.0.3' apply false
-  id 'io.github.goooler.shadow' version '8.1.3' apply false
+  id 'com.gradleup.shadow' version '8.3.6' apply false
   id 'com.diffplug.spotless' version "6.25.0"
 }
 
@@ -364,17 +364,14 @@ subprojects {
           if (!shouldPublishWithShadow) {
             from components.java
           } else {
-            apply plugin: 'io.github.goooler.shadow'
-            project.shadow.component(mavenJava)
+            apply plugin: 'com.gradleup.shadow'
+            from components.shadow
 
             // Fix for avoiding inclusion of runtime dependencies marked as 'shadow' in MANIFEST Class-Path.
-            // https://github.com/johnrengelman/shadow/issues/324
+            // https://github.com/GradleUp/shadow/issues/324
             afterEvaluate {
               pom.withXml { xml ->
-                if (xml.asNode().get('dependencies') == null) {
-                  xml.asNode().appendNode('dependencies')
-                }
-                def dependenciesNode = xml.asNode().get('dependencies').get(0)
+                def dependenciesNode = xml.asNode().get('dependencies') ?: xml.asNode().appendNode('dependencies')
                 project.configurations.shadowed.allDependencies.each {
                   def dependencyNode = dependenciesNode.appendNode('dependency')
                   dependencyNode.appendNode('groupId', it.group)
@@ -1826,6 +1823,10 @@ project(':clients') {
     generator project(':generator')
   }
 
+  tasks.withType(GenerateModuleMetadata) {
+    enabled = false
+  }
+
   task createVersionFile() {
     def receiptFile = file("${layout.buildDirectory.get().asFile.path}/kafka/$buildVersionFileName")
     inputs.property "commitId", commitId
@@ -1860,7 +1861,7 @@ project(':clients') {
     // dependencies excluded from the final jar, since they are declared as runtime dependencies
     dependencies {
       project.configurations.shadowed.allDependencies.each {
-        exclude(dependency(it.group + ':' + it.name))
+        exclude(dependency(it))
       }
       // exclude proto files from the jar
       exclude "**/opentelemetry/proto/**/*.proto"
@@ -3245,7 +3246,7 @@ project(':streams:upgrade-system-tests-39') {
 
 project(':jmh-benchmarks') {
 
-  apply plugin: 'io.github.goooler.shadow'
+  apply plugin: 'com.gradleup.shadow'
 
   shadowJar {
     archiveBaseName = 'kafka-jmh-benchmarks'