Skip to content

Commit

Permalink
Merge branch 'upstream/master' into rollup-improve-logger
Browse files Browse the repository at this point in the history
* upstream/master:
  Pass IndexMetadata to AllocationDecider.can_remain (elastic#88453)
  [TSDB] Cache rollup bucket timestamp to reduce rounding cost (elastic#88420)
  Correct some typos/mistakes in comments/docs (elastic#88446)
  Make ClusterInfo use immutable maps in all cases (elastic#88447)
  Reduce map lookups (elastic#88418)
  Don't index geo_shape field in AbstractBuilderTestCase (elastic#88437)
  Remove usages of TestGeoShapeFieldMapperPlugin from enrich module (elastic#88440)
  Fix test memory leak (elastic#88362)
  Improve error when sorting on incompatible types (elastic#88399)
  Remove usages of BucketCollector#getLeafCollector(LeafReaderContext) (elastic#88414)
  Mute ReactiveStorageIT::testScaleWhileShrinking (elastic#88431)
  Clarify snapshot docs on archive indices (elastic#88417)
  [Stack Monitoring] Switch cgroup memory fields to keyword (elastic#88260)
  Fix RealmIdentifier XContent parser (elastic#88410)
  Make LoggedExec gradle task configuration cache compatible (elastic#87621)
  Update CorruptedFileIT so that it passes with new allocation strategy (elastic#88314)
  Update RareClusterStateIT to work with the new shards allocator (elastic#87922)
  Ensure CreateApiKey always creates a new document (elastic#88413)

# Conflicts:
#	x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java
  • Loading branch information
weizijun committed Jul 12, 2022
2 parents f6bd4fb + 4889650 commit b9e97db
Show file tree
Hide file tree
Showing 102 changed files with 1,139 additions and 537 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ import org.gradle.testkit.runner.TaskOutcome
class InternalBwcGitPluginFuncTest extends AbstractGitAwareGradleFuncTest {

def setup() {
// using LoggedExec is not cc compatible
configurationCacheCompatible = false
internalBuild()
buildFile << """
import org.elasticsearch.gradle.Version;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,11 @@ import spock.lang.Unroll
/*
* Test is ignored on ARM since this test case tests the ability to build certain older BWC branches that we don't support on ARM
*/

@IgnoreIf({ Architecture.current() == Architecture.AARCH64 })
class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest {

def setup() {
// used LoggedExec task is not configuration cache compatible and
// Cannot serialize BwcSetupExtension containing project object
configurationCacheCompatible = false
internalBuild()
buildFile << """
Expand Down Expand Up @@ -119,4 +118,5 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.0/" +
"distribution/archives/darwin-tar/build/install/elasticsearch-8.0.0-SNAPSHOT")
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,20 @@ import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.internal.test.AntFixture
import org.gradle.api.file.FileSystemOperations
import org.gradle.api.file.ProjectLayout
import org.gradle.api.tasks.Internal
import org.gradle.process.ExecOperations

import javax.inject.Inject

class AntFixtureStop extends LoggedExec implements FixtureStop {
abstract class AntFixtureStop extends LoggedExec implements FixtureStop {

@Internal
AntFixture fixture

@Internal
FileSystemOperations fileSystemOperations

@Inject
AntFixtureStop(FileSystemOperations fileSystemOperations) {
super(fileSystemOperations)
this.fileSystemOperations = fileSystemOperations
AntFixtureStop(ProjectLayout projectLayout, ExecOperations execOperations, FileSystemOperations fileSystemOperations) {
super(projectLayout, execOperations, fileSystemOperations)
}

void setFixture(AntFixture fixture) {
Expand All @@ -40,10 +38,10 @@ class AntFixtureStop extends LoggedExec implements FixtureStop {
}

if (Os.isFamily(Os.FAMILY_WINDOWS)) {
executable = 'Taskkill'
getExecutable().set('Taskkill')
args('/PID', pid, '/F')
} else {
executable = 'kill'
getExecutable().set('kill')
args('-9', pid)
}
doLast {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,12 @@
import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.logging.LogLevel;
import org.gradle.api.provider.Provider;
import org.gradle.api.tasks.TaskProvider;

import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -64,26 +61,21 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String
return project.getTasks().register(name, LoggedExec.class, loggedExec -> {
loggedExec.dependsOn("checkoutBwcBranch");
loggedExec.usesService(bwcTaskThrottleProvider);
loggedExec.setSpoolOutput(true);
loggedExec.setWorkingDir(checkoutDir.get());
loggedExec.doFirst(new Action<Task>() {
@Override
public void execute(Task t) {
// Execution time so that the checkouts are available
String compilerVersionInfoPath = minimumCompilerVersionPath(unreleasedVersionInfo.get().version());
String minimumCompilerVersion = readFromFile(new File(checkoutDir.get(), compilerVersionInfoPath));
loggedExec.environment("JAVA_HOME", getJavaHome(Integer.parseInt(minimumCompilerVersion)));
}
});
loggedExec.getWorkingDir().set(checkoutDir.get());

loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> {
String minimumCompilerVersion = readFromFile(new File(checkoutDir, minimumCompilerVersionPath(version.version())));
return getJavaHome(Integer.parseInt(minimumCompilerVersion));
}));

if (Os.isFamily(Os.FAMILY_WINDOWS)) {
loggedExec.executable("cmd");
loggedExec.getExecutable().set("cmd");
loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString());
} else {
loggedExec.executable(new File(checkoutDir.get(), "gradlew").toString());
loggedExec.getExecutable().set(new File(checkoutDir.get(), "gradlew").toString());
}

loggedExec.args("-g", project.getGradle().getGradleUserHomeDir());
loggedExec.args("-g", project.getGradle().getGradleUserHomeDir().toString());
if (project.getGradle().getStartParameter().isOffline()) {
loggedExec.args("--offline");
}
Expand All @@ -93,8 +85,7 @@ public void execute(Task t) {
loggedExec.args("-Dorg.elasticsearch.build.cache.url=" + buildCacheUrl);
}

loggedExec.args("-Dbuild.snapshot=true");
loggedExec.args("-Dscan.tag.NESTED");
loggedExec.args("-Dbuild.snapshot=true", "-Dscan.tag.NESTED");
final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel();
List<LogLevel> nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG);
if (nonDefaultLogLevels.contains(logLevel)) {
Expand All @@ -110,8 +101,7 @@ public void execute(Task t) {
if (project.getGradle().getStartParameter().isParallelProjectExecutionEnabled()) {
loggedExec.args("--parallel");
}
loggedExec.setStandardOutput(new IndentingOutputStream(System.out, unreleasedVersionInfo.get().version()));
loggedExec.setErrorOutput(new IndentingOutputStream(System.err, unreleasedVersionInfo.get().version()));
loggedExec.getIndentingConsoleOutput().set(unreleasedVersionInfo.map(v -> v.version().toString()));
configAction.execute(loggedExec);
});
}
Expand All @@ -122,32 +112,6 @@ private String minimumCompilerVersionPath(Version bwcVersion) {
: "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
}

private static class IndentingOutputStream extends OutputStream {

public final byte[] indent;
private final OutputStream delegate;

IndentingOutputStream(OutputStream delegate, Object version) {
this.delegate = delegate;
indent = (" [" + version + "] ").getBytes(StandardCharsets.UTF_8);
}

@Override
public void write(int b) throws IOException {
int[] arr = { b };
write(arr, 0, 1);
}

public void write(int[] bytes, int offset, int length) throws IOException {
for (int i = 0; i < bytes.length; i++) {
delegate.write(bytes[i]);
if (bytes[i] == '\n') {
delegate.write(indent);
}
}
}
}

private static String readFromFile(File file) {
try {
return FileUtils.readFileToString(file).trim();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,29 +69,27 @@ public void apply(Project project) {
TaskContainer tasks = project.getTasks();
TaskProvider<LoggedExec> createCloneTaskProvider = tasks.register("createClone", LoggedExec.class, createClone -> {
createClone.onlyIf(task -> this.gitExtension.getCheckoutDir().get().exists() == false);
createClone.setCommandLine(asList("git", "clone", buildLayout.getRootDirectory(), gitExtension.getCheckoutDir().get()));
createClone.commandLine("git", "clone", buildLayout.getRootDirectory(), gitExtension.getCheckoutDir().get());
});

ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
TaskProvider<LoggedExec> findRemoteTaskProvider = tasks.register("findRemote", LoggedExec.class, findRemote -> {
findRemote.dependsOn(createCloneTaskProvider);
// TODO Gradle should provide property based configuration here
findRemote.setWorkingDir(gitExtension.getCheckoutDir().get());
findRemote.setCommandLine(asList("git", "remote", "-v"));
ByteArrayOutputStream output = new ByteArrayOutputStream();
findRemote.setStandardOutput(output);
findRemote.doLast(t -> { extraProperties.set("remoteExists", isRemoteAvailable(remote, output)); });
findRemote.getWorkingDir().set(gitExtension.getCheckoutDir());
findRemote.commandLine("git", "remote", "-v");
findRemote.getCaptureOutput().set(true);
findRemote.doLast(t -> { extraProperties.set("remoteExists", isRemoteAvailable(remote, findRemote.getOutput())); });
});

TaskProvider<LoggedExec> addRemoteTaskProvider = tasks.register("addRemote", LoggedExec.class, addRemote -> {
addRemote.dependsOn(findRemoteTaskProvider);
addRemote.onlyIf(task -> ((boolean) extraProperties.get("remoteExists")) == false);
addRemote.setWorkingDir(gitExtension.getCheckoutDir().get());
addRemote.getWorkingDir().set(gitExtension.getCheckoutDir().get());
String remoteRepo = remote.get();
// for testing only we can override the base remote url
String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo")
.getOrElse("https://github.com/" + remoteRepo + "/elasticsearch.git");
addRemote.setCommandLine(asList("git", "remote", "add", remoteRepo, remoteRepoUrl));
addRemote.commandLine("git", "remote", "add", remoteRepo, remoteRepoUrl);
});

boolean isOffline = project.getGradle().getStartParameter().isOffline();
Expand All @@ -107,8 +105,8 @@ public void apply(Project project) {
});
fetchLatest.onlyIf(t -> isOffline == false && gitFetchLatest.get());
fetchLatest.dependsOn(addRemoteTaskProvider);
fetchLatest.setWorkingDir(gitExtension.getCheckoutDir().get());
fetchLatest.setCommandLine(asList("git", "fetch", "--all"));
fetchLatest.getWorkingDir().set(gitExtension.getCheckoutDir().get());
fetchLatest.commandLine("git", "fetch", "--all");
});

String projectPath = project.getPath();
Expand Down Expand Up @@ -210,7 +208,7 @@ private String execInCheckoutDir(Action<ExecSpec> execSpecConfig) {
return os.toString().trim();
}

private static boolean isRemoteAvailable(Provider<String> remote, ByteArrayOutputStream output) {
return new String(output.toByteArray()).lines().anyMatch(l -> l.contains(remote.get() + "\t"));
private static boolean isRemoteAvailable(Provider<String> remote, String output) {
return output.lines().anyMatch(l -> l.contains(remote.get() + "\t"));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -241,9 +241,9 @@ static void createBuildBwcTask(
return BuildParams.isCi()
&& (gitBranch == null || gitBranch.endsWith("master") == false || gitBranch.endsWith("main") == false);
});
c.args(projectPath.replace('/', ':') + ":" + assembleTaskName);
c.getArgs().add(projectPath.replace('/', ':') + ":" + assembleTaskName);
if (project.getGradle().getStartParameter().isBuildCacheEnabled()) {
c.args("--build-cache");
c.getArgs().add("--build-cache");
}
c.doLast(new Action<Task>() {
@Override
Expand Down
Loading

0 comments on commit b9e97db

Please sign in to comment.