Skip to content

Commit

Permalink
Merge branch 'upstream/master' into rollup-improve-logger
Browse files Browse the repository at this point in the history 
* upstream/master:
  Pass IndexMetadata to AllocationDecider.can_remain (elastic#88453)
  [TSDB] Cache rollup bucket timestamp to reduce rounding cost (elastic#88420)
  Correct some typos/mistakes in comments/docs (elastic#88446)
  Make ClusterInfo use immutable maps in all cases (elastic#88447)
  Reduce map lookups (elastic#88418)
  Don't index geo_shape field in AbstractBuilderTestCase (elastic#88437)
  Remove usages of TestGeoShapeFieldMapperPlugin from enrich module (elastic#88440)
  Fix test memory leak (elastic#88362)
  Improve error when sorting on incompatible types (elastic#88399)
  Remove usages of BucketCollector#getLeafCollector(LeafReaderContext) (elastic#88414)
  Mute ReactiveStorageIT::testScaleWhileShrinking (elastic#88431)
  Clarify snapshot docs on archive indices (elastic#88417)
  [Stack Monitoring] Switch cgroup memory fields to keyword (elastic#88260)
  Fix RealmIdentifier XContent parser (elastic#88410)
  Make LoggedExec gradle task configuration cache compatible (elastic#87621)
  Update CorruptedFileIT so that it passes with new allocation strategy (elastic#88314)
  Update RareClusterStateIT to work with the new shards allocator (elastic#87922)
  Ensure CreateApiKey always creates a new document (elastic#88413)

# Conflicts:
#	x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java
  • Loading branch information
@weizijun
weizijun committed Jul 12, 2022
2 parents f6bd4fb + 4889650 commit b9e97db
Show file tree
Hide file tree
Showing 102 changed files with 1,139 additions and 537 deletions.
2 changes: 0 additions & 2 deletions ...rc/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ import org.gradle.testkit.runner.TaskOutcome
class InternalBwcGitPluginFuncTest extends AbstractGitAwareGradleFuncTest {

def setup() {
// using LoggedExec is not cc compatible
configurationCacheCompatible = false
internalBuild()
buildFile << """
import org.elasticsearch.gradle.Version;
Expand Down
4 changes: 2 additions & 2 deletions ...roovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,11 @@ import spock.lang.Unroll
/*
* Test is ignored on ARM since this test case tests the ability to build certain older BWC branches that we don't support on ARM
*/

@IgnoreIf({ Architecture.current() == Architecture.AARCH64 })
class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest {

def setup() {
// used LoggedExec task is not configuration cache compatible and
// Cannot serialize BwcSetupExtension containing project object
configurationCacheCompatible = false
internalBuild()
buildFile << """
Expand Down Expand Up @@ -119,4 +118,5 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.0/" +
"distribution/archives/darwin-tar/build/install/elasticsearch-8.0.0-SNAPSHOT")
}

}
16 changes: 7 additions & 9 deletions build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,20 @@ import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.internal.test.AntFixture
import org.gradle.api.file.FileSystemOperations
import org.gradle.api.file.ProjectLayout
import org.gradle.api.tasks.Internal
import org.gradle.process.ExecOperations

import javax.inject.Inject

class AntFixtureStop extends LoggedExec implements FixtureStop {
abstract class AntFixtureStop extends LoggedExec implements FixtureStop {

@Internal
AntFixture fixture

@Internal
FileSystemOperations fileSystemOperations

@Inject
AntFixtureStop(FileSystemOperations fileSystemOperations) {
super(fileSystemOperations)
this.fileSystemOperations = fileSystemOperations
AntFixtureStop(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) {
super(projectLayout, execOperations, fileSystemOperations)
}

void setFixture(AntFixture fixture) {
Expand All @@ -40,10 +38,10 @@ class AntFixtureStop extends LoggedExec implements FixtureStop {
}

if (Os.isFamily(Os.FAMILY_WINDOWS)) {
executable = 'Taskkill'
getExecutable().set('Taskkill')
args('/PID', pid, '/F')
} else {
executable = 'kill'
getExecutable().set('kill')
args('-9', pid)
}
doLast {
Expand Down
58 changes: 11 additions & 47 deletions build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,12 @@
import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.logging.LogLevel;
import org.gradle.api.provider.Provider;
import org.gradle.api.tasks.TaskProvider;

import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -64,26 +61,21 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String
return project.getTasks().register(name, LoggedExec.class, loggedExec -> {
loggedExec.dependsOn("checkoutBwcBranch");
loggedExec.usesService(bwcTaskThrottleProvider);
loggedExec.setSpoolOutput(true);
loggedExec.setWorkingDir(checkoutDir.get());
loggedExec.doFirst(new Action<Task>() {
@Override
public void execute(Task t) {
// Execution time so that the checkouts are available
String compilerVersionInfoPath = minimumCompilerVersionPath(unreleasedVersionInfo.get().version());
String minimumCompilerVersion = readFromFile(new File(checkoutDir.get(), compilerVersionInfoPath));
loggedExec.environment("JAVA_HOME", getJavaHome(Integer.parseInt(minimumCompilerVersion)));
}
});
loggedExec.getWorkingDir().set(checkoutDir.get());

loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> {
String minimumCompilerVersion = readFromFile(new File(checkoutDir, minimumCompilerVersionPath(version.version())));
return getJavaHome(Integer.parseInt(minimumCompilerVersion));
}));

if (Os.isFamily(Os.FAMILY_WINDOWS)) {
loggedExec.executable("cmd");
loggedExec.getExecutable().set("cmd");
loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString());
} else {
loggedExec.executable(new File(checkoutDir.get(), "gradlew").toString());
loggedExec.getExecutable().set(new File(checkoutDir.get(), "gradlew").toString());
}

loggedExec.args("-g", project.getGradle().getGradleUserHomeDir());
loggedExec.args("-g", project.getGradle().getGradleUserHomeDir().toString());
if (project.getGradle().getStartParameter().isOffline()) {
loggedExec.args("--offline");
}
Expand All @@ -93,8 +85,7 @@ public void execute(Task t) {
loggedExec.args("-Dorg.elasticsearch.build.cache.url=" + buildCacheUrl);
}

loggedExec.args("-Dbuild.snapshot=true");
loggedExec.args("-Dscan.tag.NESTED");
loggedExec.args("-Dbuild.snapshot=true", "-Dscan.tag.NESTED");
final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel();
List<LogLevel> nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG);
if (nonDefaultLogLevels.contains(logLevel)) {
Expand All @@ -110,8 +101,7 @@ public void execute(Task t) {
if (project.getGradle().getStartParameter().isParallelProjectExecutionEnabled()) {
loggedExec.args("--parallel");
}
loggedExec.setStandardOutput(new IndentingOutputStream(System.out, unreleasedVersionInfo.get().version()));
loggedExec.setErrorOutput(new IndentingOutputStream(System.err, unreleasedVersionInfo.get().version()));
loggedExec.getIndentingConsoleOutput().set(unreleasedVersionInfo.map(v -> v.version().toString()));
configAction.execute(loggedExec);
});
}
Expand All @@ -122,32 +112,6 @@ private String minimumCompilerVersionPath(Version bwcVersion) {
: "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
}

private static class IndentingOutputStream extends OutputStream {

public final byte[] indent;
private final OutputStream delegate;

IndentingOutputStream(OutputStream delegate, Object version) {
this.delegate = delegate;
indent = (" [" + version + "] ").getBytes(StandardCharsets.UTF_8);
}

@Override
public void write(int b) throws IOException {
int[] arr = { b };
write(arr, 0, 1);
}

public void write(int[] bytes, int offset, int length) throws IOException {
for (int i = 0; i < bytes.length; i++) {
delegate.write(bytes[i]);
if (bytes[i] == '\n') {
delegate.write(indent);
}
}
}
}

private static String readFromFile(File file) {
try {
return FileUtils.readFileToString(file).trim();
Expand Down
24 changes: 11 additions & 13 deletions ...-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,29 +69,27 @@ public void apply(Project project) {
TaskContainer tasks = project.getTasks();
TaskProvider<LoggedExec> createCloneTaskProvider = tasks.register("createClone", LoggedExec.class, createClone -> {
createClone.onlyIf(task -> this.gitExtension.getCheckoutDir().get().exists() == false);
createClone.setCommandLine(asList("git", "clone", buildLayout.getRootDirectory(), gitExtension.getCheckoutDir().get()));
createClone.commandLine("git", "clone", buildLayout.getRootDirectory(), gitExtension.getCheckoutDir().get());
});

ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
TaskProvider<LoggedExec> findRemoteTaskProvider = tasks.register("findRemote", LoggedExec.class, findRemote -> {
findRemote.dependsOn(createCloneTaskProvider);
// TODO Gradle should provide property based configuration here
findRemote.setWorkingDir(gitExtension.getCheckoutDir().get());
findRemote.setCommandLine(asList("git", "remote", "-v"));
ByteArrayOutputStream output = new ByteArrayOutputStream();
findRemote.setStandardOutput(output);
findRemote.doLast(t -> { extraProperties.set("remoteExists", isRemoteAvailable(remote, output)); });
findRemote.getWorkingDir().set(gitExtension.getCheckoutDir());
findRemote.commandLine("git", "remote", "-v");
findRemote.getCaptureOutput().set(true);
findRemote.doLast(t -> { extraProperties.set("remoteExists", isRemoteAvailable(remote, findRemote.getOutput())); });
});

TaskProvider<LoggedExec> addRemoteTaskProvider = tasks.register("addRemote", LoggedExec.class, addRemote -> {
addRemote.dependsOn(findRemoteTaskProvider);
addRemote.onlyIf(task -> ((boolean) extraProperties.get("remoteExists")) == false);
addRemote.setWorkingDir(gitExtension.getCheckoutDir().get());
addRemote.getWorkingDir().set(gitExtension.getCheckoutDir().get());
String remoteRepo = remote.get();
// for testing only we can override the base remote url
String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo")
.getOrElse("https://github.com/" + remoteRepo + "/elasticsearch.git");
addRemote.setCommandLine(asList("git", "remote", "add", remoteRepo, remoteRepoUrl));
addRemote.commandLine("git", "remote", "add", remoteRepo, remoteRepoUrl);
});

boolean isOffline = project.getGradle().getStartParameter().isOffline();
Expand All @@ -107,8 +105,8 @@ public void apply(Project project) {
});
fetchLatest.onlyIf(t -> isOffline == false && gitFetchLatest.get());
fetchLatest.dependsOn(addRemoteTaskProvider);
fetchLatest.setWorkingDir(gitExtension.getCheckoutDir().get());
fetchLatest.setCommandLine(asList("git", "fetch", "--all"));
fetchLatest.getWorkingDir().set(gitExtension.getCheckoutDir().get());
fetchLatest.commandLine("git", "fetch", "--all");
});

String projectPath = project.getPath();
Expand Down Expand Up @@ -210,7 +208,7 @@ private String execInCheckoutDir(Action<ExecSpec> execSpecConfig) {
return os.toString().trim();
}

private static boolean isRemoteAvailable(Provider<String> remote, ByteArrayOutputStream output) {
return new String(output.toByteArray()).lines().anyMatch(l -> l.contains(remote.get() + "\t"));
private static boolean isRemoteAvailable(Provider<String> remote, String output) {
return output.lines().anyMatch(l -> l.contains(remote.get() + "\t"));
}
}
4 changes: 2 additions & 2 deletions ...l/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,9 +241,9 @@ static void createBuildBwcTask(
return BuildParams.isCi()
&& (gitBranch == null || gitBranch.endsWith("master") == false || gitBranch.endsWith("main") == false);
});
c.args(projectPath.replace('/', ':') + ":" + assembleTaskName);
c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName);
if (project.getGradle().getStartParameter().isBuildCacheEnabled()) {
c.args("--build-cache");
c.getArgs().add("--build-cache");
}
c.doLast(new Action<Task>() {
@Override
Expand Down
Loading

0 comments on commit b9e97db

Please sign in to comment.