Revision 47b6eb39e5e68fd58a83a9c9f47a72477ec3438e authored by Moritz Mack on 21 March 2024, 17:12:22 UTC, committed by GitHub on 21 March 2024, 17:12:22 UTC
Mute test until remaining races are addressed. (related to #104652)
1 parent 7c46c73
build.gradle
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
import com.avast.gradle.dockercompose.tasks.ComposePull
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.ObjectMapper
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin
import org.elasticsearch.gradle.internal.ResolveAllDependencies
import org.elasticsearch.gradle.internal.info.BuildParams
import org.elasticsearch.gradle.util.GradleUtils
import org.gradle.plugins.ide.eclipse.model.AccessRule
import org.gradle.plugins.ide.eclipse.model.ProjectDependency
import java.nio.file.Files
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING
import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
plugins {
id 'lifecycle-base'
id 'elasticsearch.docker-support'
id 'elasticsearch.global-build-info'
id 'elasticsearch.build-complete'
id 'elasticsearch.build-scan'
id 'elasticsearch.jdk-download'
id 'elasticsearch.internal-distribution-download'
id 'elasticsearch.runtime-jdk-provision'
id 'elasticsearch.ide'
id 'elasticsearch.forbidden-dependencies'
id 'elasticsearch.local-distribution'
id 'elasticsearch.fips'
id 'elasticsearch.internal-testclusters'
id 'elasticsearch.run'
id 'elasticsearch.run-ccs'
id 'elasticsearch.release-tools'
id 'elasticsearch.versions'
}
/**
* This is a convenient method for declaring test artifact dependencies provided by the internal
* test artifact plugin. It replaces basically the longer dependency notation with explicit capability
* declaration like this:
*
* testImplementation(project(xpackModule('repositories-metering-api'))) {
* capabilities {
* requireCapability("org.elasticsearch.gradle:repositories-metering-api-test-artifacts")
* }
* }
*
* */
ext.testArtifact = { p, String name = "test" ->
def projectDependency = p.dependencies.create(p)
projectDependency.capabilities {
requireCapabilities("org.elasticsearch.gradle:${projectDependency.name}-${name}-artifacts")
};
}
class StepExpansion {
String templatePath
List<Version> versions
String variable
}
class ListExpansion {
List<Version> versions
String variable
}
tasks.register("updateCIBwcVersions") {
def writeVersions = { File file, List<Version> versions ->
file.text = ""
file << "BWC_VERSION:\n"
versions.each {
file << " - \"$it\"\n"
}
}
def writeBuildkitePipeline = { String outputFilePath, String pipelineTemplatePath, List<ListExpansion> listExpansions, List<StepExpansion> stepExpansions = [] ->
def outputFile = file(outputFilePath)
def pipelineTemplate = file(pipelineTemplatePath)
def pipeline = pipelineTemplate.text
listExpansions.each { expansion ->
def listString = "[" + expansion.versions.collect { "\"${it}\"" }.join(", ") + "]"
pipeline = pipeline.replaceAll('\\$' + expansion.variable, listString)
}
stepExpansions.each { expansion ->
def steps = ""
expansion.versions.each {
steps += "\n" + file(expansion.templatePath).text.replaceAll('\\$BWC_VERSION', it.toString())
}
pipeline = pipeline.replaceAll(' *\\$' + expansion.variable, steps)
}
outputFile.text = "# This file is auto-generated. See ${pipelineTemplatePath}\n" + pipeline
}
// Writes a Buildkite pipelime from a template, and replaces $BWC_LIST with an array of versions
// Useful for writing a list of versions in a matrix configuration
def expandBwcList = { String outputFilePath, String pipelineTemplatePath, List<Version> versions ->
writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [new ListExpansion(versions: versions, variable: "BWC_LIST")])
}
// Writes a Buildkite pipeline from a template, and replaces $BWC_STEPS with a list of steps, one for each version
// Useful when you need to configure more versions than are allowed in a matrix configuration
def expandBwcSteps = { String outputFilePath, String pipelineTemplatePath, String stepTemplatePath, List<Version> versions ->
writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [], [new StepExpansion(templatePath: stepTemplatePath, versions: versions, variable: "BWC_STEPS")])
}
doLast {
writeVersions(file(".ci/bwcVersions"), BuildParams.bwcVersions.allIndexCompatible)
writeVersions(file(".ci/snapshotBwcVersions"), BuildParams.bwcVersions.unreleasedIndexCompatible)
expandBwcList(
".buildkite/pipelines/intake.yml",
".buildkite/pipelines/intake.template.yml",
BuildParams.bwcVersions.unreleasedIndexCompatible
)
writeBuildkitePipeline(
".buildkite/pipelines/periodic.yml",
".buildkite/pipelines/periodic.template.yml",
[
new ListExpansion(versions: BuildParams.bwcVersions.unreleasedIndexCompatible, variable: "BWC_LIST"),
],
[
new StepExpansion(templatePath: ".buildkite/pipelines/periodic.bwc.template.yml", versions: BuildParams.bwcVersions.allIndexCompatible, variable: "BWC_STEPS"),
]
)
expandBwcSteps(
".buildkite/pipelines/periodic-packaging.yml",
".buildkite/pipelines/periodic-packaging.template.yml",
".buildkite/pipelines/periodic-packaging.bwc.template.yml",
BuildParams.bwcVersions.allIndexCompatible
)
}
}
tasks.register("verifyVersions") {
def verifyCiYaml = { File file, List<Version> versions ->
String ciYml = file.text
versions.each {
if (ciYml.contains("\"$it\"\n") == false) {
throw new Exception("${file} is outdated, run `./gradlew updateCIBwcVersions` and check in the results")
}
}
}
doLast {
if (gradle.startParameter.isOffline()) {
throw new GradleException("Must run in online mode to verify versions")
}
// Read the list from maven central.
// Fetch the metadata and parse the xml into Version instances because it's more straight forward here
// rather than bwcVersion ( VersionCollection ).
new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
BuildParams.bwcVersions.compareToAuthoritative(
new XmlParser().parse(s)
.versioning.versions.version
.collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }
.collect { Version.fromString(it) }
)
}
verifyCiYaml(file(".ci/bwcVersions"), BuildParams.bwcVersions.allIndexCompatible)
verifyCiYaml(file(".ci/snapshotBwcVersions"), BuildParams.bwcVersions.unreleasedIndexCompatible)
// Make sure backport bot config file is up to date
JsonNode backportConfig = new ObjectMapper().readTree(file(".backportrc.json"))
BuildParams.bwcVersions.forPreviousUnreleased { unreleasedVersion ->
boolean valid = backportConfig.get("targetBranchChoices").elements().any { branchChoice ->
if (branchChoice.isObject()) {
return branchChoice.get("name").textValue() == unreleasedVersion.branch
} else {
return branchChoice.textValue() == unreleasedVersion.branch
}
}
if (valid == false) {
throw new GradleException("No branch choice exists for development branch ${unreleasedVersion.branch} in .backportrc.json.")
}
}
String versionMapping = backportConfig.get("branchLabelMapping").fields().find { it.value.textValue() == 'main' }.key
String expectedMapping = "^v${versions.elasticsearch.replaceAll('-SNAPSHOT', '')}\$"
if (versionMapping != expectedMapping) {
throw new GradleException(
"Backport label mapping for branch 'main' is '${versionMapping}' but should be " +
"'${expectedMapping}'. Update .backportrc.json."
)
}
}
}
/*
* When adding backcompat behavior that spans major versions, temporarily
* disabling the backcompat tests is necessary. This flag controls
* the enabled state of every bwc task. It should be set back to true
* after the backport of the backcompat code is complete.
*/
boolean bwc_tests_enabled = true
// place a PR link here when committing bwc changes:
String bwc_tests_disabled_issue = ""
if (bwc_tests_enabled == false) {
if (bwc_tests_disabled_issue.isEmpty()) {
throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
}
println "========================= WARNING ========================="
println " Backwards compatibility tests are disabled!"
println "See ${bwc_tests_disabled_issue}"
println "==========================================================="
}
if (project.gradle.startParameter.taskNames.any { it.startsWith("checkPart") || it == 'functionalTests' }) {
// Disable BWC tests for checkPart* tasks and platform support tests as it's expected that this will run on it's own check
bwc_tests_enabled = false
}
subprojects { proj ->
apply plugin: 'elasticsearch.base'
}
allprojects {
// We disable this plugin for now till we shaked out the issues we see
// e.g. see https://github.com/elastic/elasticsearch/issues/72169
// apply plugin:'elasticsearch.internal-test-rerun'
plugins.withType(BaseInternalPluginBuildPlugin).whenPluginAdded {
project.dependencies {
compileOnly project(":server")
testImplementation project(":test:framework")
}
}
// injecting groovy property variables into all projects
project.ext {
// for ide hacks...
isEclipse = providers.systemProperty("eclipse.launcher").isPresent() || // Detects gradle launched from Eclipse's IDE
providers.systemProperty("eclipse.application").isPresent() || // Detects gradle launched from the Eclipse compiler server
gradle.startParameter.taskNames.contains('eclipse') || // Detects gradle launched from the command line to do eclipse stuff
gradle.startParameter.taskNames.contains('cleanEclipse')
}
ext.bwc_tests_enabled = bwc_tests_enabled
// eclipse configuration
apply plugin: 'elasticsearch.eclipse'
/*
* Allow accessing com/sun/net/httpserver in projects that have
* configured forbidden apis to allow it.
*/
plugins.withType(ForbiddenApisPlugin) {
eclipse.classpath.file.whenMerged { classpath ->
if (false == forbiddenApisTest.bundledSignatures.contains('jdk-non-portable')) {
classpath.entries
.findAll { it.kind == "con" && it.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER") }
.each {
it.accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
}
}
}
}
tasks.register('resolveAllDependencies', ResolveAllDependencies) {
configs = project.configurations
resolveJavaToolChain = true
if (project.path.contains("fixture")) {
dependsOn tasks.withType(ComposePull)
}
}
plugins.withId('lifecycle-base') {
if (project.path.startsWith(":x-pack:")) {
if (project.path.contains("security") || project.path.contains(":ml")) {
tasks.register('checkPart4') { dependsOn 'check' }
} else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) {
tasks.register('checkPart3') { dependsOn 'check' }
} else {
tasks.register('checkPart2') { dependsOn 'check' }
}
} else {
tasks.register('checkPart1') { dependsOn 'check' }
}
tasks.register('functionalTests') { dependsOn 'check' }
}
/*
* Remove assemble/dependenciesInfo on all qa projects because we don't
* need to publish artifacts for them.
*/
if (project.name.equals('qa') || project.path.contains(':qa:')) {
maybeConfigure(project.tasks, 'assemble') {
it.enabled = false
}
maybeConfigure(project.tasks, 'dependenciesInfo') {
it.enabled = false
}
}
project.afterEvaluate {
// Ensure similar tasks in dependent projects run first. The projectsEvaluated here is
// important because, while dependencies.all will pickup future dependencies,
// it is not necessarily true that the task exists in both projects at the time
// the dependency is added.
if (project.path == ':test:framework') {
// :test:framework:test cannot run before and after :server:test
return
}
tasks.matching { it.name.equals('integTest') }.configureEach { integTestTask ->
integTestTask.mustRunAfter tasks.matching { it.name.equals("test") }
}
configurations.matching { it.canBeResolved }.all { Configuration configuration ->
dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
Project upstreamProject = dep.dependencyProject
if (project.path != upstreamProject?.path) {
for (String taskName : ['test', 'integTest']) {
project.tasks.matching { it.name == taskName }.configureEach { task ->
task.shouldRunAfter(upstreamProject.tasks.matching { upStreamTask -> upStreamTask.name == taskName })
}
}
}
}
}
}
apply plugin: 'elasticsearch.formatting'
}
tasks.register("verifyBwcTestsEnabled") {
doLast {
if (bwc_tests_enabled == false) {
throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.')
}
}
}
tasks.register("branchConsistency") {
description 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
group 'Verification'
dependsOn ":verifyVersions", ":verifyBwcTestsEnabled"
}
tasks.named("wrapper").configure {
distributionType = 'ALL'
doLast {
// copy wrapper properties file to build-tools-internal to allow seamless idea integration
def file = new File("build-tools-internal/gradle/wrapper/gradle-wrapper.properties")
Files.copy(wrapper.getPropertiesFile().toPath(), file.toPath(), REPLACE_EXISTING)
// copy wrapper properties file to plugins/examples to allow seamless idea integration
def examplePluginsWrapperProperties = new File("plugins/examples/gradle/wrapper/gradle-wrapper.properties")
Files.copy(wrapper.getPropertiesFile().toPath(), examplePluginsWrapperProperties.toPath(), REPLACE_EXISTING)
// Update build-tools to reflect the Gradle upgrade
// TODO: we can remove this once we have tests to make sure older versions work.
project.file('build-tools-internal/src/main/resources/minimumGradleVersion').text = gradleVersion
println "Updated minimum Gradle Version"
}
}
gradle.projectsEvaluated {
// Having the same group and name for distinct projects causes Gradle to consider them equal when resolving
// dependencies leading to hard to debug failures. Run a check across all project to prevent this from happening.
// see: https://github.com/gradle/gradle/issues/847
Map coordsToProject = [:]
project.allprojects.forEach { p ->
String coords = "${p.group}:${p.name}"
if (false == coordsToProject.putIfAbsent(coords, p)) {
throw new GradleException(
"Detected that two projects: ${p.path} and ${coordsToProject[coords].path} " +
"have the same name and group: ${coords}. " +
"This doesn't currently work correctly in Gradle, see: " +
"https://github.com/gradle/gradle/issues/847"
)
}
}
}
tasks.named("precommit") {
dependsOn gradle.includedBuild('build-tools').task(':precommit')
dependsOn gradle.includedBuild('build-tools-internal').task(':precommit')
}
tasks.named("checkPart1").configure {
dependsOn gradle.includedBuild('build-tools').task(':check')
dependsOn gradle.includedBuild('build-tools-internal').task(':check')
}
tasks.named("assemble").configure {
dependsOn gradle.includedBuild('build-tools').task(':assemble')
}
tasks.named("cleanEclipse").configure {
dependsOn gradle.includedBuild('build-conventions').task(':cleanEclipse')
dependsOn gradle.includedBuild('build-tools').task(':cleanEclipse')
dependsOn gradle.includedBuild('build-tools-internal').task(':cleanEclipse')
}
tasks.named("eclipse").configure {
dependsOn gradle.includedBuild('build-conventions').task(':eclipse')
dependsOn gradle.includedBuild('build-tools').task(':eclipse')
dependsOn gradle.includedBuild('build-tools-internal').task(':eclipse')
}
tasks.register("buildReleaseArtifacts").configure {
group = 'build'
description = 'Builds all artifacts required for release manager'
dependsOn allprojects.findAll {
it.path.startsWith(':distribution:docker') == false
&& it.path.startsWith(':ml-cpp') == false
&& it.path.startsWith(':distribution:bwc') == false
&& it.path.startsWith(':test:fixture') == false
}
.collect { GradleUtils.findByName(it.tasks, 'assemble') }
.findAll { it != null }
}
tasks.register("spotlessApply").configure {
dependsOn gradle.includedBuild('build-tools').task(':spotlessApply')
dependsOn gradle.includedBuild('build-tools').task(':reaper:spotlessApply')
dependsOn gradle.includedBuild('build-tools-internal').task(':spotlessApply')
}
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...