Revision 52e7b926a95fbc3ef1ad7f99abbf7e3d00264bf4 authored by Armin Braun on 10 May 2021, 11:40:57 UTC, committed by GitHub on 10 May 2021, 11:40:57 UTC
Use an iterator instead of a list when passing around what to delete. In the case of very large deletes the iterator is a much smaller than the actual list of files to delete (since we save all the prefixes which adds up if the individual shard folders contain lots of deletes). Also this commit as a side-effect adjusts a few spots in logging where the log messages could be catastrophic in size when trace logging is activated.
1 parent a7ff5e7
settings.gradle
plugins {
id "com.gradle.enterprise" version "3.5.1"
}
rootProject.name = "elasticsearch"
List projects = [
'build-tools',
'build-tools:reaper',
'rest-api-spec',
'docs',
'client:rest',
'client:rest-high-level',
'client:sniffer',
'client:transport',
'client:test',
'client:client-benchmark-noop-api-plugin',
'client:benchmark',
'benchmarks',
'distribution:archives:integ-test-zip',
'distribution:archives:windows-zip',
'distribution:archives:no-jdk-windows-zip',
'distribution:archives:darwin-tar',
'distribution:archives:darwin-aarch64-tar',
'distribution:archives:no-jdk-darwin-tar',
'distribution:archives:no-jdk-darwin-aarch64-tar',
'distribution:archives:linux-aarch64-tar',
'distribution:archives:linux-tar',
'distribution:archives:no-jdk-linux-tar',
'distribution:docker',
'distribution:docker:docker-aarch64-export',
'distribution:docker:docker-export',
'distribution:docker:ironbank-docker-aarch64-export',
'distribution:docker:ironbank-docker-export',
'distribution:docker:ubi-docker-aarch64-export',
'distribution:docker:ubi-docker-export',
'distribution:packages:aarch64-deb',
'distribution:packages:deb',
'distribution:packages:no-jdk-deb',
'distribution:packages:aarch64-rpm',
'distribution:packages:rpm',
'distribution:packages:no-jdk-rpm',
'distribution:bwc:bugfix',
'distribution:bwc:maintenance',
'distribution:bwc:minor',
'distribution:bwc:staged',
'distribution:tools:java-version-checker',
'distribution:tools:launchers',
'distribution:tools:plugin-cli',
'distribution:tools:keystore-cli',
'distribution:tools:geoip-cli',
'server',
'server:cli',
'test:framework',
'test:fixtures:azure-fixture',
'test:fixtures:gcs-fixture',
'test:fixtures:hdfs-fixture',
'test:fixtures:krb5kdc-fixture',
'test:fixtures:minio-fixture',
'test:fixtures:old-elasticsearch',
'test:fixtures:s3-fixture',
'test:fixtures:geoip-fixture',
'test:fixtures:url-fixture',
'test:fixtures:nginx-fixture',
'test:logger-usage'
]
/**
* Iterates over sub directories, looking for build.gradle, and adds a project if found
* for that dir with the given path prefix. Note that this requires each level
* of the dir hierarchy to have a build.gradle. Otherwise we would have to iterate
* all files/directories in the source tree to find all projects.
*/
void addSubProjects(String path, File dir) {
if (dir.isDirectory() == false) return;
if (dir.name == 'buildSrc') return;
if (new File(dir, 'build.gradle').exists() == false) return;
if (findProject(dir) != null) return;
final String projectName = "${path}:${dir.name}"
include projectName
if (path.isEmpty() || path.startsWith(':example-plugins')) {
project(projectName).projectDir = dir
}
for (File subdir : dir.listFiles()) {
addSubProjects(projectName, subdir)
}
}
// include example plugins first, so adding plugin dirs below won't muck with :example-plugins
File examplePluginsDir = new File(rootProject.projectDir, 'plugins/examples')
for (File example : examplePluginsDir.listFiles()) {
if (example.isDirectory() == false) continue;
if (example.name.startsWith('build') || example.name.startsWith('.')) continue;
addSubProjects(':example-plugins', example)
}
project(':example-plugins').projectDir = new File(rootProject.projectDir, 'plugins/examples')
addSubProjects('', new File(rootProject.projectDir, 'libs'))
addSubProjects('', new File(rootProject.projectDir, 'modules'))
addSubProjects('', new File(rootProject.projectDir, 'plugins'))
addSubProjects('', new File(rootProject.projectDir, 'qa'))
addSubProjects('test', new File(rootProject.projectDir, 'test/external-modules'))
addSubProjects('', new File(rootProject.projectDir, 'x-pack'))
List startTasks = gradle.startParameter.taskNames
include projects.toArray(new String[0])
project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc')
project(':build-tools:reaper').projectDir = new File(rootProject.projectDir, 'buildSrc/reaper')
project(":libs").children.each { libsProject ->
libsProject.name = "elasticsearch-${libsProject.name}"
}
project(":test:external-modules").children.each { testProject ->
testProject.name = "test-${testProject.name}"
}
// look for extra plugins for elasticsearch
File extraProjects = new File(rootProject.projectDir.parentFile, "${rootProject.projectDir.name}-extra")
if (extraProjects.exists()) {
for (File extraProjectDir : extraProjects.listFiles()) {
addSubProjects('', extraProjectDir)
}
}
Computing file changes ...