Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bump anomaly-detection version to 1.2 #286

Merged
merged 2 commits into from
Nov 5, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 11 additions & 64 deletions .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,77 +27,24 @@ jobs:
- name: Checkout AD
uses: actions/checkout@v2

# dependencies: OpenSearch
- name: Checkout OpenSearch
uses: actions/checkout@v2
with:
repository: 'opensearch-project/OpenSearch'
path: OpenSearch
ref: '1.1'
- name: Build OpenSearch
working-directory: ./OpenSearch
run: ./gradlew publishToMavenLocal

# dependencies: common-utils
- name: Checkout common-utils
uses: actions/checkout@v2
with:
ref: 'main'
repository: 'opensearch-project/common-utils'
path: common-utils
- name: Build common-utils
working-directory: ./common-utils
run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT

# dependencies: job-scheduler
- name: Checkout job-scheduler
uses: actions/checkout@v2
with:
ref: 'main'
repository: 'opensearch-project/job-scheduler'
path: job-scheduler

- name: Build job-scheduler
working-directory: ./job-scheduler
run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT
- name: Assemble job-scheduler
working-directory: ./job-scheduler
run: |
./gradlew assemble -Dopensearch.version=1.1.0-SNAPSHOT
echo "Creating ../src/test/resources/job-scheduler ..."
mkdir -p ../src/test/resources/job-scheduler
pwd
echo "Copying ./build/distributions/*.zip to ../src/test/resources/job-scheduler ..."
ls ./build/distributions/
cp ./build/distributions/*.zip ../src/test/resources/job-scheduler
echo "Copied ./build/distributions/*.zip to ../src/test/resources/job-scheduler ..."
ls ../src/test/resources/job-scheduler
echo "Creating ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT ..."
mkdir -p ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT
echo "Copying ./build/distributions/*.zip to ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT ..."
ls ./build/distributions/
cp ./build/distributions/*.zip ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT
echo "Copied ./build/distributions/*.zip to ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT ..."
ls ../src/test/resources/org/opensearch/ad/bwc/job-scheduler/1.1.0.0-SNAPSHOT

- name: Assemble anomaly-detection
run: |
./gradlew assemble -Dopensearch.version=1.1.0-SNAPSHOT
echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT ..."
mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT
echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT ..."
./gradlew assemble -Dopensearch.version=1.2.0-SNAPSHOT
echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT ..."
mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT
echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT ..."
ls ./build/distributions/
cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT
echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT ..."
ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.1.0.0-SNAPSHOT
cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT
echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT ..."
ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT

- name: Build and Run Tests
run: |
./gradlew build -Dopensearch.version=1.1.0-SNAPSHOT
./gradlew build -Dopensearch.version=1.2.0-SNAPSHOT

- name: Publish to Maven Local
run: |
./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT
./gradlew publishToMavenLocal -Dopensearch.version=1.2.0-SNAPSHOT

- name: Multi Nodes Integration Testing
run: |
Expand All @@ -106,8 +53,8 @@ jobs:
- name: Pull and Run Docker
run: |
plugin=`ls build/distributions/*.zip`
version=1.1.0-SNAPSHOT
plugin_version=1.1.0.0-SNAPSHOT
version=1.2.0-SNAPSHOT
plugin_version=1.2.0.0-SNAPSHOT
echo Using OpenSearch $version with AD $plugin_version
cd ..
if docker pull opensearchstaging/opensearch:$version
Expand Down
37 changes: 12 additions & 25 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,16 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
buildscript {
ext {
opensearch_group = "org.opensearch"
opensearch_version = System.getProperty("opensearch.version", "1.1.0-SNAPSHOT")
// 1.1.0 -> 1.1.0.0, and 1.1.0-SNAPSHOT -> 1.1.0.0-SNAPSHOT
opensearch_version = System.getProperty("opensearch.version", "1.2.0-SNAPSHOT")
// 1.2.0 -> 1.2.0.0, and 1.2.0-SNAPSHOT -> 1.2.0.0-SNAPSHOT
opensearch_build = opensearch_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2')
common_utils_version = System.getProperty("common_utils.version", opensearch_build)
job_scheduler_version = System.getProperty("job_scheduler.version", opensearch_build)
}

repositories {
mavenLocal()
maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
mavenCentral()
maven { url "https://plugins.gradle.org/m2/" }
jcenter()
Expand All @@ -39,7 +40,6 @@ plugins {
id 'nebula.ospackage' version "8.3.0" apply false
id "com.diffplug.gradle.spotless" version "3.26.1"
id 'java-library'
id 'checkstyle'
}

tasks.withType(JavaCompile) {
Expand All @@ -54,6 +54,7 @@ tasks.withType(Javadoc) {

repositories {
mavenLocal()
maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
mavenCentral()
maven { url "https://plugins.gradle.org/m2/" }
jcenter()
Expand Down Expand Up @@ -102,11 +103,12 @@ configurations.all {
if (it.state != Configuration.State.UNRESOLVED) return
resolutionStrategy {
force "joda-time:joda-time:${versions.joda}"
force "com.fasterxml.jackson.core:jackson-core:2.11.4"
force "commons-logging:commons-logging:${versions.commonslogging}"
force "org.apache.httpcomponents:httpcore:${versions.httpcore}"
force "commons-codec:commons-codec:${versions.commonscodec}"

force "org.mockito:mockito-core:3.0.0"
force "org.mockito:mockito-core:2.25.0"
force "org.objenesis:objenesis:3.0.1"
force "net.bytebuddy:byte-buddy:1.9.15"
force "net.bytebuddy:byte-buddy-agent:1.9.15"
Expand Down Expand Up @@ -274,7 +276,7 @@ String bwcFilePath = "src/test/resources/org/opensearch/ad/bwc/"
testClusters {
"${baseName}$i" {
testDistribution = "ARCHIVE"
versions = ["7.10.2","1.1.0-SNAPSHOT"]
versions = ["7.10.2","1.2.0-SNAPSHOT"]
numberOfNodes = 3
plugin(provider(new Callable<RegularFile>(){
@Override
Expand Down Expand Up @@ -538,10 +540,6 @@ jacocoTestReport {
check.dependsOn jacocoTestCoverageVerification
jacocoTestCoverageVerification.dependsOn jacocoTestReport

checkstyle {
toolVersion = '8.29'
}

dependencies {
compile "org.opensearch:opensearch:${opensearch_version}"
compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${opensearch_version}"
Expand All @@ -556,20 +554,11 @@ dependencies {
compile group: 'commons-lang', name: 'commons-lang', version: '2.6'
compile group: 'org.apache.commons', name: 'commons-pool2', version: '2.10.0'

// randomcutforest-serialization uses jackson 2.12, but opensearch-scripting-painless-spi uses jackson 2.11.
// compile scope won't work due to conflict.
// resolutionStrategy using 2.11 won't work as
// com.fasterxml.jackson.databind.ObjectMapper depends on com/fasterxml/jackson/core/util/JacksonFeature
// that is created since 2.12. Compile won't fail but there is a runtime ClassNotFoundException
// due to absent JacksonFeature.
// The fix is to put jackson in direct dependency and use implementation scope.
// implementation scope let the dependency in both compiling and running classpath, but
// not leaked through to clients (Opensearch). Here we force the jackson version to whatever
// opensearch uses.
// force Jackson version to avoid version conflict issue
implementation 'software.amazon.randomcutforest:randomcutforest-serialization:2.0.1'
implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
implementation "com.fasterxml.jackson.core:jackson-core:2.11.4"
implementation "com.fasterxml.jackson.core:jackson-databind:2.11.4"
implementation "com.fasterxml.jackson.core:jackson-annotations:2.11.4"
compile files('lib/randomcutforest-parkservices-2.0.1.jar')
compile files('lib/randomcutforest-core-2.0.1.jar')

Expand All @@ -585,7 +574,7 @@ dependencies {
}

testCompile group: 'pl.pragmatists', name: 'JUnitParams', version: '1.1.1'
testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.0.0'
testImplementation group: 'org.mockito', name: 'mockito-core', version: '2.25.0'
testImplementation group: 'org.powermock', name: 'powermock-api-mockito2', version: '2.0.2'
testImplementation group: 'org.powermock', name: 'powermock-module-junit4', version: '2.0.2'
testImplementation group: 'org.powermock', name: 'powermock-module-junit4-common', version: '2.0.2'
Expand All @@ -602,8 +591,6 @@ dependencies {
testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.7.2'
testRuntimeOnly 'org.junit.vintage:junit-vintage-engine:5.7.2'
testCompileOnly 'junit:junit:4.12'

checkstyle "com.puppycrawl.tools:checkstyle:${project.checkstyle.toolVersion}"
}

compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
Expand Down
11 changes: 11 additions & 0 deletions src/test/java/org/opensearch/ad/ADIntegTestCase.java
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.opensearch.ad.common.exception.AnomalyDetectionException;
import org.opensearch.ad.constant.CommonName;
import org.opensearch.ad.indices.AnomalyDetectionIndices;
import org.opensearch.ad.mock.plugin.MockReindexPlugin;
import org.opensearch.ad.model.ADTask;
import org.opensearch.ad.model.AnomalyDetector;
import org.opensearch.ad.model.AnomalyResult;
Expand All @@ -62,6 +63,7 @@
import org.opensearch.search.aggregations.AggregationBuilder;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.test.OpenSearchIntegTestCase;
import org.opensearch.test.transport.MockTransportService;

import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.collect.ImmutableMap;
Expand All @@ -86,6 +88,15 @@ protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return Collections.singletonList(AnomalyDetectorPlugin.class);
}

@Override
protected Collection<Class<? extends Plugin>> getMockPlugins() {
final ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>();
plugins.add(MockReindexPlugin.class);
plugins.addAll(super.getMockPlugins());
plugins.remove(MockTransportService.TestPlugin.class);
return Collections.unmodifiableList(plugins);
}

@Override
@Before
public void setUp() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import org.opensearch.test.OpenSearchIntegTestCase;

@OpenSearchIntegTestCase.ClusterScope(transportClientRatio = 0.9)
public class ADStatsIT extends OpenSearchIntegTestCase {
public class ADStatsITTests extends OpenSearchIntegTestCase {

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,13 @@

import org.opensearch.action.ActionFuture;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.ad.ADIntegTestCase;
import org.opensearch.ad.AnomalyDetectorPlugin;
import org.opensearch.plugins.Plugin;
import org.opensearch.test.OpenSearchIntegTestCase;

@OpenSearchIntegTestCase.ClusterScope(transportClientRatio = 0.9)
public class DeleteIT extends OpenSearchIntegTestCase {
public class DeleteITTests extends ADIntegTestCase {

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.opensearch.test.OpenSearchIntegTestCase;

@OpenSearchIntegTestCase.ClusterScope(transportClientRatio = 0.9)
public class ProfileIT extends OpenSearchIntegTestCase {
public class ProfileITTests extends OpenSearchIntegTestCase {

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.opensearch.test.OpenSearchIntegTestCase;

@OpenSearchIntegTestCase.ClusterScope(transportClientRatio = 0.9)
public class RCFResultIT extends OpenSearchIntegTestCase {
public class RCFResultITTests extends OpenSearchIntegTestCase {

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.opensearch.test.OpenSearchIntegTestCase;

@OpenSearchIntegTestCase.ClusterScope(transportClientRatio = 0.9)
public class ThresholdResultIT extends OpenSearchIntegTestCase {
public class ThresholdResultITTests extends OpenSearchIntegTestCase {

@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
Expand Down
Binary file not shown.
Binary file not shown.