Merge branch 'release/v1.1.0' into feature/ning-http-client-1.0

Changelog.md

@@ -4,8 +4,52 @@ Noteworthy changes to the agent are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.0.6-public-preview] - TO BE DISCLOSED
+
+## [1.0.8-public-preview] TO BE DECIDED
+### Changes
+- Support for stored procedure call detection in SQL events
+- Support for extracting environment variables in case of Remote Code Execution events
+- Support for executing script file analysis in case of Remote Code Execution events
+- Enabled the transformation of the low-priority instrumentation module by default in case of IAST
+- SecureCookie schema check has been removed
+### Fixes
+- Incorrect user file details in the vulnerability details
+- Low severity hook event was not generated when the same url can process multiple request methods
+- Detection of server app directory to mitigate false positives for File Access vulnerability
+
+## [1.0.7-public-preview] - 2023-12-6
 ### Changes
+- Async HttpClient v2+ Support: The security agent now also supports Async HTTP client version 2 and above
+- Sun Net HTTP Server support: The security agent now supports Sun Net HTTP Server
+- Add APM trace information population in the event
+- WS headers added : NR-CSEC-ENTITY-GUID & NR-CSEC-ENTITY-NAME
+- JSON version bump to 1.1.1
+- Add critical error logging via LogMessage event
+### Fixes
+- Insecure cookie attack vulnerability was flagged in secure communication, accounting communication type to mitigate the issue
+- DynamoDB v2 issue: missing attribute values for conditionCheck method in case of transactWriteItems operation on DynamoDB
+- Never print LicenseKey 
+### Misc
+- Updated unit test cases for all the outbound request instrumentation modules to include test cases for csec parent id header
+- Unit test cases for Async HttpClient v2+
+- Unit test cases for Jetty v12+
+- Unit test cases for Sun Net HTTP Server
+- Unit test cases for Netty Server
+
+## [1.0.6-public-preview] - 2023-10-17
+### Changes
+- Cassandra DB v3.0+ Support: The Security agent now supports Cassandra DB version 3.0 and above
+- HttpClient v5.0+ Support: The Security agent now also supports HttpClient version 5.0 and above
+- Support for std-out logging
+- Added feature for Daily log rollover
+- Support for logger config: log_file_count and log_limit_in_kbytes
+- Relocating all our instrumentation packages under the package com.newrelic.agent.security.instrumentation.*
+- Package Refactoring for Unit Tests: Move packaging for all UTs to com.nr.agent.security.instrumentation.*
+- Set default value for low severity instrumentation to false
+
+### Fixes
+- Fixed ClassNotFoundException for IOStreamHelper class with Glassfish
+- Updated PostgreSQL UTs with Embedded Server instead of test container
 
 ## [1.0.5-public-preview] - 2023-08-29
 ### Changes

README.md

@@ -28,6 +28,7 @@ The agent automatically instruments the following frameworks.
 - Log4j from 2.0 to 2.20.0
 - Servlet from 2.4 to latest
 - Spring from 0 to latest
+- Sun Net HTTP Server
 
 ### Java Native Operations
 
@@ -55,6 +56,7 @@ The agent automatically instruments the following HTTP clients and messaging ser
 - Jaxen XPATH from 1.1 to latest
 - Saxpath 1.0
 - Xalan XPATH 2.1.0 to latest
+- Async Http Client from 2.0 to latest
 
 ### Datastores
 

gradle.properties

@@ -1,13 +1,13 @@
 # The agent version.
-agentVersion=1.0.6
-jsonVersion=1.1.0
+agentVersion=1.0.8
+jsonVersion=1.1.1
 # Updated exposed NR APM API version.
-nrAPIVersion=8.3.0
+nrAPIVersion=8.4.0
 
 # Actual NR APM Agent version
 # This is intentionally kept to an older NR agent version since it is only used as dependency for unit test framework &
 # verify instrumentation plugin. This will only be updated when either of these functions require the update.
-nrAgentVersion=8.3.0
+nrAgentVersion=8.4.0
 #org.gradle.jvmargs=-Xmx2048m
 org.gradle.jvmargs=-Xmx4g
 org.gradle.caching=true

instrumentation-security-test/src/main/java/com/newrelic/agent/security/introspec/internal/HttpServerRule.java

@@ -30,6 +30,12 @@ protected void after() {
 
     @Override
     public void shutdown() {
+        try {
+            // to prevent socket.io: broken pipe error for async calls
+            Thread.sleep(1000);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
         server.shutdown();
     }
 

instrumentation-security-test/src/main/java/com/newrelic/agent/security/introspec/internal/HttpTestServerImpl.java

@@ -32,7 +32,7 @@
 class HttpTestServerImpl extends NanoHTTPD implements HttpTestServer {
     private final int port;
 
-    private Map<String, String> headers = new HashMap<>();
+    private static Map<String, String> headers = new HashMap<>();
 
     public HttpTestServerImpl() throws IOException {
         this(getRandomPort());
@@ -83,7 +83,7 @@ private Response serveNonDispatcher(IHTTPSession session) {
     private Response serveInternal(IHTTPSession session) {
         NewRelic.addCustomParameter("server.port", this.port);
         final Map<String, String> incomingHeaders = session.getHeaders();
-        headers = incomingHeaders;
+        headers.putAll(incomingHeaders);
 
         if (incomingHeaders.containsKey(SLEEP_MS_HEADER_KEY)) {
             try {

instrumentation-security/async-http-client-2.0.0/build.gradle

@@ -0,0 +1,27 @@
+dependencies {
+    implementation(project(":newrelic-security-api"))
+    implementation("com.newrelic.agent.java:newrelic-weaver-api:${nrAPIVersion}")
+    implementation("com.newrelic.agent.java:newrelic-api:${nrAPIVersion}")
+    implementation("org.asynchttpclient:async-http-client:2.0.0")
+}
+
+jar {
+    manifest { attributes 'Implementation-Title': 'com.newrelic.instrumentation.security.async-http-client-2.0.0' }
+}
+
+verifyInstrumentation {
+    passesOnly 'org.asynchttpclient:async-http-client:[2.0.0-RC1,)'
+    excludeRegex ".*(alpha|RC).*"
+}
+
+test {
+    // These instrumentation tests only run on Java 8 regardless of the -PtestN gradle property that is set.
+    onlyIf {
+        project.hasProperty('test8')
+    }
+}
+
+site {
+    title 'Async Http Client'
+    type 'Messaging'
+}

instrumentation-security/async-http-client-2.0.0/src/main/java/com/newrelic/agent/security/instrumentation/org/asynchttpclient/AsynchttpHelper.java

@@ -0,0 +1,122 @@
+package com.newrelic.agent.security.instrumentation.org.asynchttpclient;
+
+import com.newrelic.api.agent.security.NewRelicSecurity;
+import com.newrelic.api.agent.security.instrumentation.helpers.GenericHelper;
+import com.newrelic.api.agent.security.instrumentation.helpers.ServletHelper;
+import com.newrelic.api.agent.security.schema.AbstractOperation;
+import com.newrelic.api.agent.security.schema.StringUtils;
+import com.newrelic.api.agent.security.schema.exceptions.NewRelicSecurityException;
+import com.newrelic.api.agent.security.schema.operation.SSRFOperation;
+import com.newrelic.api.agent.security.utils.SSRFUtils;
+import org.asynchttpclient.Request;
+
+public class AsynchttpHelper {
+
+
+    public static final String NR_SEC_CUSTOM_ATTRIB_NAME = "ASYNCHTTP_OPERATION_LOCK-";
+
+    public static final String METHOD_EXECUTE = "executeRequest";
+
+    public static boolean skipExistsEvent() {
+        if (!(NewRelicSecurity.getAgent().getCurrentPolicy().getVulnerabilityScan().getEnabled() &&
+                NewRelicSecurity.getAgent().getCurrentPolicy().getVulnerabilityScan().getIastScan().getEnabled())) {
+            return true;
+        }
+
+        return false;
+    }
+
+    public static boolean isLockAcquired() {
+        try {
+            return NewRelicSecurity.isHookProcessingActive() &&
+                    Boolean.TRUE.equals(NewRelicSecurity.getAgent().getSecurityMetaData().getCustomAttribute(getNrSecCustomAttribName(), Boolean.class));
+        } catch (Throwable ignored) {}
+        return false;
+    }
+
+    public static boolean acquireLockIfPossible() {
+        try {
+            if (NewRelicSecurity.isHookProcessingActive() &&
+                    !isLockAcquired()) {
+                NewRelicSecurity.getAgent().getSecurityMetaData().addCustomAttribute(getNrSecCustomAttribName(), true);
+                return true;
+            }
+        } catch (Throwable ignored){}
+        return false;
+    }
+
+    public static void releaseLock() {
+        try {
+            if (NewRelicSecurity.isHookProcessingActive()) {
+                NewRelicSecurity.getAgent().getSecurityMetaData().addCustomAttribute(getNrSecCustomAttribName(), null);
+            }
+        } catch (Throwable ignored) {
+        }
+    }
+
+    private static String getNrSecCustomAttribName() {
+        return NR_SEC_CUSTOM_ATTRIB_NAME + Thread.currentThread().getId();
+    }
+
+    public static AbstractOperation preprocessSecurityHook(String url, String className, String methodName) {
+        try {
+            if (!NewRelicSecurity.isHookProcessingActive() ||
+                    NewRelicSecurity.getAgent().getSecurityMetaData().getRequest().isEmpty() ||
+                    url == null || url.trim().isEmpty()) {
+                return null;
+            }
+
+            SSRFOperation operation = new SSRFOperation(url,
+                    className, methodName);
+            NewRelicSecurity.getAgent().registerOperation(operation);
+            return operation;
+        } catch (Throwable e) {
+            if (e instanceof NewRelicSecurityException) {
+                e.printStackTrace();
+                throw e;
+            }
+        }
+        return null;
+    }
+
+    public static void registerExitOperation(boolean isProcessingAllowed, AbstractOperation operation) {
+        try {
+            if (operation == null || !isProcessingAllowed || !NewRelicSecurity.isHookProcessingActive() ||
+                    NewRelicSecurity.getAgent().getSecurityMetaData().getRequest().isEmpty() || skipExistsEvent()
+            ) {
+                return;
+            }
+            NewRelicSecurity.getAgent().registerExitEvent(operation);
+        } catch (Throwable ignored) {
+        }
+    }
+
+    public static Request addSecurityHeaders(Request request, AbstractOperation operation) {
+        if (operation == null || request == null) {
+            return null;
+        }
+
+        // Add Security IAST header
+        String iastHeader = NewRelicSecurity.getAgent().getSecurityMetaData().getFuzzRequestIdentifier().getRaw();
+        if (iastHeader != null && !iastHeader.trim().isEmpty()) {
+            request.getHeaders().add(ServletHelper.CSEC_IAST_FUZZ_REQUEST_ID, iastHeader);
+        }
+
+        String csecParaentId = NewRelicSecurity.getAgent().getSecurityMetaData().getCustomAttribute(GenericHelper.CSEC_PARENT_ID, String.class);
+        if(StringUtils.isNotBlank(csecParaentId)){
+            request.getHeaders().add(GenericHelper.CSEC_PARENT_ID, csecParaentId);
+        }
+
+        if (operation.getApiID() != null && !operation.getApiID().trim().isEmpty() &&
+                operation.getExecutionId() != null && !operation.getExecutionId().trim().isEmpty()) {
+            // Add Security distributed tracing header
+            request.getHeaders().remove(ServletHelper.CSEC_DISTRIBUTED_TRACING_HEADER);
+            request.getHeaders().add(ServletHelper.CSEC_DISTRIBUTED_TRACING_HEADER,
+                    SSRFUtils.generateTracingHeaderValue(NewRelicSecurity.getAgent().getSecurityMetaData()
+                                    .getTracingHeaderValue(),
+                            operation.getApiID(), operation.getExecutionId(),
+                            NewRelicSecurity.getAgent().getAgentUUID()));
+        }
+        return request;
+    }
+}

instrumentation-security/async-http-client-2.0.0/src/main/java/org/asynchttpclient/AsyncHttpClient_Instrumentation.java

@@ -0,0 +1,60 @@
+/*
+ *
+ *  * Copyright 2020 New Relic Corporation. All rights reserved.
+ *  * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.asynchttpclient;
+import com.newrelic.api.agent.security.schema.AbstractOperation;
+import com.newrelic.api.agent.weaver.MatchType;
+import com.newrelic.api.agent.weaver.Weave;
+import com.newrelic.api.agent.weaver.Weaver;
+import com.newrelic.agent.security.instrumentation.org.asynchttpclient.AsynchttpHelper;
+
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * Instrumentation for the provider interface.
+ */
+@Weave(type = MatchType.Interface, originalName = "org.asynchttpclient.AsyncHttpClient")
+public abstract class AsyncHttpClient_Instrumentation {
+
+    public <T> ListenableFuture<T> executeRequest(Request request, AsyncHandler<T> handler) {
+        URI uri = null;
+        boolean isLockAcquired = AsynchttpHelper.acquireLockIfPossible();
+        AbstractOperation operation = null;
+        if(isLockAcquired) {
+            try {
+                uri = new URI(request.getUrl());
+                String scheme = uri.getScheme().toLowerCase();
+
+                // only instrument HTTP or HTTPS calls
+                if (("http".equals(scheme) || "https".equals(scheme))) {
+                    operation = AsynchttpHelper.preprocessSecurityHook(uri.toURL().toString(), this.getClass().getName(),
+                            AsynchttpHelper.METHOD_EXECUTE);
+                    Request updatedRequest = AsynchttpHelper.addSecurityHeaders(request, operation);
+                    if (updatedRequest != null) {
+                        request = updatedRequest;
+                    }
+                }
+
+            } catch (URISyntaxException | MalformedURLException uriSyntaxException) {
+                // if Java can't parse the URI, asynchttpclient won't be able to either
+                // let's just proceed without instrumentation
+            }
+        }
+        ListenableFuture<T> returnVal = null;
+        try {
+            returnVal = Weaver.callOriginal();
+        } finally {
+            if(isLockAcquired){
+                AsynchttpHelper.releaseLock();
+            }
+        }
+        AsynchttpHelper.registerExitOperation(isLockAcquired, operation);
+        return returnVal;
+    }
+}

instrumentation-security/async-http-client-2.0.0/src/main/java/play/CorePlugin.java

@@ -0,0 +1,18 @@
+/*
+ *
+ *  * Copyright 2020 New Relic Corporation. All rights reserved.
+ *  * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package play;
+
+import com.newrelic.api.agent.weaver.SkipIfPresent;
+
+/**
+ * Play v1 instrumentation is implemented using its own set of pointcuts that don't work well with our async APIs. This
+ * class is present in Play v1 but not v2, and will cause this module NOT to load if the customer is using Play v1.
+ */
+@SkipIfPresent
+public class CorePlugin {
+}