diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots-trigger.yml new file mode 100644 index 000000000000..7f1c639770ae --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots-trigger.yml @@ -0,0 +1,6 @@ +--- +jjbb-template: periodic-trigger-lgc.yml +vars: + - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot + - lgc-job: elastic+elasticsearch+%BRANCH%+intake + - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots.yml new file mode 100644 index 000000000000..2a48e38f9e8f --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+dra-snapshots.yml @@ -0,0 +1,47 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+dra-snapshot + display-name: "elastic / elasticsearch # %BRANCH% - DRA snapshot" + description: "Publishing Daily Releasable Artifacts (DRAs) of Elasticsearch %BRANCH% snapshots.\n" + node: "ubuntu-18.04 || ubuntu-20.04" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + ES_VERSION=$(cat build-tools-internal/version.properties \ + | grep elasticsearch \ + | sed "s/elasticsearch//g" \ + | sed "s/ //g" \ + | sed "s/=//") + BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats master)" + ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp master)" + set -euo pipefail + set +x + VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) + export VAULT_TOKEN + $WORKSPACE/.ci/scripts/run-gradle.sh assemble -Ddra.artifacts=true -Ddra.artifacts.dependency.beats=${BEATS_BUILD_ID} -Ddra.artifacts.dependency.ml-cpp=${ML_CPP_BUILD_ID} + unset VAULT_TOKEN + set -x + $WORKSPACE/x-pack/plugin/sql/connectors/tableau/package.sh + # Artifacts should be generated + docker run --rm \ + --name release-manager \ + -e VAULT_ADDR \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project elasticsearch \ + --branch "master" \ + --commit "$GIT_COMMIT" \ + --workflow "snapshot" \ + --version "$ES_VERSION" \ + --artifact-set main \ + --dependency https://artifacts-snapshot.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}-SNAPSHOT.json \ + --dependency https://artifacts-snapshot.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}-SNAPSHOT.json diff --git a/.ci/scripts/resolve-dra-manifest.sh b/.ci/scripts/resolve-dra-manifest.sh new file mode 100755 index 000000000000..7ccbaf3819e4 --- /dev/null +++ b/.ci/scripts/resolve-dra-manifest.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e +ARTIFACT="${ARTIFACT:-$1}" +BRANCH="${BRANCH:-$2}" + +curl -sS https://artifacts-snapshot.elastic.co/$ARTIFACT/latest/$BRANCH.json \ + | jq -r '.build_id' diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 7387d5beafe2..707faf8749c1 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -51,6 +51,10 @@ gradlePlugin { id = 'elasticsearch.docs-test' implementationClass = 'org.elasticsearch.gradle.internal.doc.DocsTestPlugin' } + draArtifacts { + id = 'elasticsearch.dra-artifacts' + implementationClass = 'org.elasticsearch.gradle.internal.dra.DraResolvePlugin' + } globalBuildInfo { id = 'elasticsearch.global-build-info' implementationClass = 'org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin' diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy index 823c9e4c878b..1926cf9f0395 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/LocalRepositoryFixture.groovy @@ -17,7 +17,7 @@ class LocalRepositoryFixture extends ExternalResource { private TemporaryFolder temporaryFolder - LocalRepositoryFixture(){ + LocalRepositoryFixture() { this.temporaryFolder = new TemporaryFolder() } @@ -33,22 +33,27 @@ class LocalRepositoryFixture extends ExternalResource { temporaryFolder.after() } - void generateJar(String group, String module, String version, String... clazzNames){ + void generateJar(String group, String module, String version, String... clazzNames) { def baseGroupFolderPath = group.replace('.', '/') def targetFolder = new File(repoDir, "${baseGroupFolderPath}/$module/$version") targetFolder.mkdirs() def jarFile = new File(targetFolder, "${module}-${version}.jar") - clazzNames.each {clazzName -> - DynamicType.Unloaded dynamicType = new ByteBuddy().subclass(Object.class) - .name(clazzName) - .make() - if(jarFile.exists()) { - dynamicType.inject(jarFile); - }else { - dynamicType.toJar(jarFile); + if (clazzNames.size() == 0) { + jarFile.write("blubb") + } else { + clazzNames.each { clazzName -> + DynamicType.Unloaded dynamicType = new ByteBuddy().subclass(Object.class) + .name(clazzName) + .make() + if (jarFile.exists()) { + dynamicType.inject(jarFile); + } else { + dynamicType.toJar(jarFile); + } } } + } void configureBuild(File buildFile) { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy new file mode 100644 index 000000000000..40f36f05dfd0 --- /dev/null +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/dra/DraResolvePluginFuncTest.groovy @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.dra + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.elasticsearch.gradle.fixtures.LocalRepositoryFixture +import org.elasticsearch.gradle.fixtures.WiremockFixture +import org.gradle.testkit.runner.TaskOutcome +import org.junit.ClassRule +import spock.lang.Shared + +class DraResolvePluginFuncTest extends AbstractGradleFuncTest { + + @Shared + @ClassRule + public LocalRepositoryFixture repository = new LocalRepositoryFixture() + + def setup() { + configurationCacheCompatible = false + + buildFile << """ + plugins { + id 'elasticsearch.dra-artifacts' + } + + repositories.all { + // for supporting http testing repos here + allowInsecureProtocol = true + } + """ + } + + def "provides flag indicating dra usage"() { + setup: + repository.generateJar("org.acme", "ml-cpp", "8.6.0-SNAPSHOT") + buildFile << """ + if(useDra == false) { + repositories { + maven { + name = "local-test" + url = "${repository.getRepoDir().toURI()}" + metadataSources { + artifact() + } + } + } + } + """ + + buildFile << """ + configurations { + someConfig + } + + dependencies { + someConfig "org.acme:ml-cpp:8.6.0-SNAPSHOT" + } + + tasks.register('resolveArtifacts') { + doLast { + configurations.someConfig.files.each { println it } + } + } + """ + + when: + def result = gradleRunner("resolveArtifacts").build() + + then: + result.task(":resolveArtifacts").outcome == TaskOutcome.SUCCESS + + when: + result = gradleRunner("resolveArtifacts", "-Ddra.artifacts=true").buildAndFail() + + then: + result.task(":resolveArtifacts").outcome == TaskOutcome.FAILED + result.output.contains("Cannot resolve external dependency org.acme:ml-cpp:8.6.0-SNAPSHOT because no repositories are defined.") + } + + def "configures repositories to resolve #draKey like dra #artifactType artifacts"() { + setup: + repository.generateJar("some.group", "bar", "1.0.0") + repository.generateJar("some.group", "baz", "1.0.0-SNAPSHOT") + repository.configureBuild(buildFile) + buildFile << """ + configurations { + someConfig + } + + dependencies { + someConfig "some.group:bar:1.0.0" + someConfig "some.group:baz:1.0.0-SNAPSHOT" + someConfig "org.acme:$draArtifact:$draVersion:deps@zip" + } + + tasks.register('resolveArtifacts') { + doLast { + configurations.someConfig.files.each { println it } + } + } + """ + + when: + def result = WiremockFixture.withWireMock(expectedRequest, "content".getBytes('UTF-8')) { server -> + gradleRunner("resolveArtifacts", + '-Ddra.artifacts=true', + "-Ddra.artifacts.dependency.${draKey}=$buildId", + "-Ddra.artifacts.url.repo.${artifactType}.prefix=${server.baseUrl()}").build() + } + + then: + result.task(":resolveArtifacts").outcome == TaskOutcome.SUCCESS + + where: + artifactType | buildId | draVersion | draKey | draArtifact | expectedRequest + "snapshot" | '8.6.0-f633b1d7' | "8.6.0-SNAPSHOT" | "ml-cpp" | "ml-cpp" | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}-deps.zip" + "release" | '8.6.0-f633b1d7' | "8.6.0" | "ml-cpp" | "ml-cpp" | "/$draKey/${buildId}/downloads/$draArtifact/${draArtifact}-${draVersion}-deps.zip" + "snapshot" | '8.6.0-f633b1d7' | "8.6.0-SNAPSHOT" | "beats" | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}-deps.zip" + "release" | '8.6.0-f633b1d7' | "8.6.0" | "beats" | "metricbeat" | "/$draKey/${buildId}/downloads/$draKey/$draArtifact/${draArtifact}-${draVersion}-deps.zip" + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java new file mode 100644 index 000000000000..432e3cbc5e72 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.dra; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; +import org.gradle.initialization.layout.BuildLayout; + +import java.util.Map; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import static java.util.Map.Entry; + +public class DraResolvePlugin implements Plugin { + + public static final String USE_DRA_ARTIFACTS_FLAG = "dra.artifacts"; + public static final String DRA_ARTIFACTS_DEPENDENCY_PREFIX = "dra.artifacts.dependency"; + private final ProviderFactory providerFactory; + private BuildLayout buildLayout; + + private final Provider snapshotRepositoryPrefix; + private final Provider releaseRepositoryPrefix; + + @Inject + public DraResolvePlugin(ProviderFactory providerFactory, BuildLayout buildLayout) { + this.providerFactory = providerFactory; + this.buildLayout = buildLayout; + this.snapshotRepositoryPrefix = providerFactory.systemProperty("dra.artifacts.url.repo.snapshot.prefix"); + this.releaseRepositoryPrefix = providerFactory.systemProperty("dra.artifacts.url.repo.release.prefix"); + } + + @Override + public void apply(Project project) { + boolean useDra = providerFactory.systemProperty(USE_DRA_ARTIFACTS_FLAG).map(Boolean::parseBoolean).getOrElse(false); + project.getExtensions().getExtraProperties().set("useDra", useDra); + if (useDra) { + resolveBuildIdProperties().get().forEach((key, buildId) -> { + configureDraRepository( + project, + "dra-snapshot-artifacts-" + key, + key, + buildId, + snapshotRepositoryPrefix.orElse("https://artifacts-snapshot.elastic.co/"), + ".*SNAPSHOT" + ); + configureDraRepository( + project, + "dra-release-artifacts-" + key, + key, + buildId, + releaseRepositoryPrefix.orElse("https://artifacts.elastic.co/"), + "^(.(?!SNAPSHOT))*$" + ); + }); + } + } + + private void configureDraRepository( + Project project, + String repositoryName, + String draKey, + String buildId, + Provider repoPrefix, + String includeVersionRegex + ) { + project.getRepositories().ivy(repo -> { + repo.setName(repositoryName); + repo.setUrl(repoPrefix.get()); + repo.patternLayout(patternLayout -> { + patternLayout.artifact( + String.format("/%s/%s/downloads/%s/[module]-[revision]-[classifier].[ext]", draKey, buildId, draKey) + ); + patternLayout.artifact( + String.format("/%s/%s/downloads/%s/[module]/[module]-[revision]-[classifier].[ext]", draKey, buildId, draKey) + ); + }); + repo.metadataSources(metadataSources -> metadataSources.artifact()); + repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeVersionByRegex(".*", ".*", includeVersionRegex)); + }); + } + + private Provider> resolveBuildIdProperties() { + return providerFactory.systemPropertiesPrefixedBy(DRA_ARTIFACTS_DEPENDENCY_PREFIX) + .map( + stringStringMap -> stringStringMap.entrySet() + .stream() + .collect( + Collectors.toMap(entry -> entry.getKey().substring(DRA_ARTIFACTS_DEPENDENCY_PREFIX.length() + 1), Entry::getValue) + ) + ); + } +} diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 845d505d88d3..b2d4628b5ab2 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -8,6 +8,7 @@ import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin import org.elasticsearch.gradle.internal.docker.DockerSupportService import org.elasticsearch.gradle.internal.docker.ShellRetry import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter +import org.elasticsearch.gradle.internal.dra.DraResolvePlugin import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.util.GradleUtils @@ -17,9 +18,10 @@ import java.time.temporal.ChronoUnit apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.test.fixtures' apply plugin: 'elasticsearch.internal-distribution-download' +apply plugin: 'elasticsearch.dra-artifacts' String buildId = providers.systemProperty('build.id').getOrNull() -boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false +boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false && useDra == false repositories { // Define a repository that allows Gradle to fetch a resource from GitHub. This @@ -35,29 +37,33 @@ repositories { metadataSources { artifact() } content { includeGroup 'krallin' } } +} - // Cloud builds bundle some beats - ivy { - name = 'beats' - if (useLocalArtifacts) { - url "file://${buildDir}/artifacts/" - patternLayout { - artifact '/[organisation]/[module]-[revision]-[classifier].[ext]' - } - } else { - url "https://artifacts-snapshot.elastic.co/" - patternLayout { - if (VersionProperties.isElasticsearchSnapshot()) { - artifact '/[organization]/[revision]/downloads/[organization]/[module]/[module]-[revision]-[classifier].[ext]' - } else { - // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`. - // Release builds are always done with a local repo. - artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/[module]/[module]-[revision]-SNAPSHOT-[classifier].[ext]' +if(useDra == false) { + repositories { + // Cloud builds bundle some beats + ivy { + name = 'beats' + if (useLocalArtifacts) { + url "file://${buildDir}/artifacts/" + patternLayout { + artifact '/[organisation]/[module]-[revision]-[classifier].[ext]' + } + } else { + url "https://artifacts-snapshot.elastic.co/" + patternLayout { + if (VersionProperties.isElasticsearchSnapshot()) { + artifact '/[organization]/[revision]/downloads/[organization]/[module]/[module]-[revision]-[classifier].[ext]' + } else { + // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`. + // Release builds are always done with a local repo. + artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/[module]/[module]-[revision]-SNAPSHOT-[classifier].[ext]' + } } } + metadataSources { artifact() } + content { includeGroup 'beats' } } - metadataSources { artifact() } - content { includeGroup 'beats' } } } diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle index 86802009abde..a7ff0a68b58e 100644 --- a/x-pack/plugin/ml/build.gradle +++ b/x-pack/plugin/ml/build.gradle @@ -1,8 +1,10 @@ import org.elasticsearch.gradle.VersionProperties +import org.elasticsearch.gradle.internal.dra.DraResolvePlugin apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' +apply plugin: 'elasticsearch.dra-artifacts' esplugin { name 'x-pack-ml' @@ -13,39 +15,41 @@ esplugin { } def localRepo = providers.systemProperty('build.ml_cpp.repo').orNull - -repositories { - exclusiveContent { - filter { - includeGroup 'org.elasticsearch.ml' - } - forRepository { - ivy { - name "ml-cpp" - metadataSources { - // no repository metadata, look directly for the artifact - artifact() - } - if (localRepo) { - url localRepo - patternLayout { - artifact "maven/[orgPath]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]" +if (useDra == false) { + repositories { + exclusiveContent { + filter { + includeGroup 'org.elasticsearch.ml' + } + forRepository { + ivy { + name "ml-cpp" + metadataSources { + // no repository metadata, look directly for the artifact + artifact() } - } else { - url "https://artifacts-snapshot.elastic.co/" - patternLayout { - if (VersionProperties.isElasticsearchSnapshot()) { - artifact '/ml-cpp/[revision]/downloads/ml-cpp/[module]-[revision]-[classifier].[ext]' - } else { - // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`. - // Release builds are always done with a local repo. - artifact '/ml-cpp/[revision]-SNAPSHOT/downloads/ml-cpp/[module]-[revision]-SNAPSHOT-[classifier].[ext]' + if (localRepo) { + url localRepo + patternLayout { + artifact "maven/[orgPath]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]" + } + } else { + url "https://artifacts-snapshot.elastic.co/" + patternLayout { + if (VersionProperties.isElasticsearchSnapshot()) { + artifact '/ml-cpp/[revision]/downloads/ml-cpp/[module]-[revision]-[classifier].[ext]' + } else { + // When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`. + // Release builds are always done with a local repo. + artifact '/ml-cpp/[revision]-SNAPSHOT/downloads/ml-cpp/[module]-[revision]-SNAPSHOT-[classifier].[ext]' + } } } } } } } + } configurations { @@ -126,4 +130,4 @@ tasks.named("dependencyLicenses").configure { mapping from: /lucene-.*/, to: 'lucene' } -addQaCheckDependencies(project) +addQaCheckDependencies(project) \ No newline at end of file