Support async profiler feature

.github/workflows/skywalking.yaml

@@ -697,6 +697,15 @@ jobs:
 
           - name: Cilium Service
             config: test/e2e-v2/cases/cilium/e2e.yaml
+
+          - name: Async Profiler ES
+            config: test/e2e-v2/cases/profiling/async-profiler/es/e2e.yaml
+          - name: Async Profiler BanyanDB
+            config: test/e2e-v2/cases/profiling/async-profiler/banyandb/e2e.yaml
+          - name: Async Profiler MySQL
+            config: test/e2e-v2/cases/profiling/async-profiler/mysql/e2e.yaml
+          - name: Async Profiler h2
+            config: test/e2e-v2/cases/profiling/async-profiler/h2/e2e.yaml
     steps:
       - uses: actions/checkout@v3
         with:

.licenserc.yaml

@@ -68,6 +68,7 @@ header:
     - '**/src/main/proto/opentelemetry/**'
     - 'oap-server/server-starter/src/main/resources/version.properties'
     - '**/mockito-extensions/**'
+    - 'oap-server/server-library/library-async-profiler-jfr-parser'
 
   comment: on-failure
 

LICENSE

@@ -225,3 +225,4 @@ The text of each license is the standard Apache 2.0 license.
    mvnw files from https://github.com/apache/maven-wrapper Apache 2.0
    svg files from skywalking-ui/src/assets/icons: https://github.com/google/material-design-icons Apache 2.0
    ZipkinQueryHandler.java reference from zipkin2.server.internal.ZipkinQueryApiV2: https://github.com/openzipkin/zipkin  Apache 2.0
+   async-profiler parser files in server-library/library-async-profiler-jfr-parser from https://github.com/async-profiler/async-profiler/tree/v3.0 Apache 2.0

apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/AsyncProfilerTaskCommand.java

@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.skywalking.oap.server.network.trace.component.command;
+
+import lombok.Getter;
+import org.apache.skywalking.apm.network.common.v3.Command;
+import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
+
+import java.util.List;
+import java.util.Objects;
+
+@Getter
+public class AsyncProfilerTaskCommand extends BaseCommand implements Serializable, Deserializable<AsyncProfilerTaskCommand> {
+    public static final Deserializable<AsyncProfilerTaskCommand> DESERIALIZER = new AsyncProfilerTaskCommand("", "", 0, null, "", 0);
+    public static final String NAME = "AsyncProfilerTaskQuery";
+
+    /**
+     * async-profiler taskId
+     */
+    private final String taskId;
+    /**
+     * run profiling for duration (second)
+     */
+    private final int duration;
+    /**
+     * async profiler extended parameters. Here is a table of optional parameters.
+     *
+     * <p>lock[=DURATION]  - profile contended locks overflowing the DURATION ns bucket (default: 10us)</p>
+     * <p>alloc[=BYTES]    - profile allocations with BYTES interval</p>
+     * <p>interval=N       - sampling interval in ns (default: 10'000'000, i.e. 10 ms)</p>
+     * <p>jstackdepth=N    - maximum Java stack depth (default: 2048)</p>
+     * <p>chunksize=N      - approximate size of JFR chunk in bytes (default: 100 MB) </p>
+     * <p>chunktime=N      - duration of JFR chunk in seconds (default: 1 hour) </p>
+     * details @see <a href="https://github.com/async-profiler/async-profiler/blob/master/src/arguments.cpp#L44">async-profiler argument</a>
+     */
+    private final String execArgs;
+    /**
+     * task create time
+     */
+    private final long createTime;
+
+    public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration,
+                                    List<String> events, String execArgs, long createTime) {
+        super(NAME, serialNumber);
+        this.taskId = taskId;
+        this.duration = duration;
+        this.createTime = createTime;
+        String comma = ",";
+        StringBuilder sb = new StringBuilder();
+        if (Objects.nonNull(events) && !events.isEmpty()) {
+            sb.append("event=").append(String.join(comma, events));
+        }
+        if (execArgs != null && !execArgs.isEmpty()) {
+            sb.append(comma).append(execArgs);
+        }
+        this.execArgs = sb.toString();
+    }
+
+    public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration,
+                                    String execArgs, long createTime) {
+        super(NAME, serialNumber);
+        this.taskId = taskId;
+        this.duration = duration;
+        this.execArgs = execArgs;
+        this.createTime = createTime;
+    }
+
+    @Override
+    public AsyncProfilerTaskCommand deserialize(Command command) {
+        final List<KeyStringValuePair> argsList = command.getArgsList();
+        String taskId = null;
+        int duration = 0;
+        String execArgs = null;
+        long createTime = 0;
+        String serialNumber = null;
+        for (final KeyStringValuePair pair : argsList) {
+            if ("SerialNumber".equals(pair.getKey())) {
+                serialNumber = pair.getValue();
+            } else if ("TaskId".equals(pair.getKey())) {
+                taskId = pair.getValue();
+            } else if ("Duration".equals(pair.getKey())) {
+                duration = Integer.parseInt(pair.getValue());
+            } else if ("ExecArgs".equals(pair.getKey())) {
+                execArgs = pair.getValue();
+            } else if ("CreateTime".equals(pair.getKey())) {
+                createTime = Long.parseLong(pair.getValue());
+            }
+        }
+        return new AsyncProfilerTaskCommand(serialNumber, taskId, duration, execArgs, createTime);
+    }
+
+    @Override
+    public Command.Builder serialize() {
+        final Command.Builder builder = commandBuilder();
+        builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId))
+                .addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration)))
+                .addArgs(KeyStringValuePair.newBuilder().setKey("ExecArgs").setValue(execArgs))
+                .addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime)));
+        return builder;
+    }
+}

apm-protocol/apm-network/src/main/java/org/apache/skywalking/oap/server/network/trace/component/command/CommandDeserializer.java

@@ -27,6 +27,8 @@ public static BaseCommand deserialize(final Command command) {
             return ProfileTaskCommand.DESERIALIZER.deserialize(command);
         } else if (ConfigurationDiscoveryCommand.NAME.equals(commandName)) {
             return ConfigurationDiscoveryCommand.DESERIALIZER.deserialize(command);
+        } else if (AsyncProfilerTaskCommand.NAME.equals(commandName)) {
+            return AsyncProfilerTaskCommand.DESERIALIZER.deserialize(command);
         }
         throw new UnsupportedCommandException(command);
     }

docs/en/api/query-protocol.md

@@ -189,10 +189,12 @@ extend type Query {
 Event query fetches the event list based on given sources and time range conditions.
 
 ### Profiling
-SkyWalking offers two types of [profiling](../concepts-and-designs/profiling.md), in-process and out-process, allowing users to create tasks and check their execution status.
+SkyWalking offers two types of [profiling](../concepts-and-designs/profiling.md), in-process(tracing profiling and async-profiler) and out-process(ebpf profiling), allowing users to create tasks and check their execution status.
 
 #### In-process profiling
 
+##### tracing profiling 
+
 ```graphql
 extend type Mutation {
     # crate new profile task
@@ -210,6 +212,24 @@ extend type Query {
 }
 ```
 
+##### async-profiler
+
+```graphql
+extend type Mutation {
+    # Create a new async-profiler task
+    createAsyncProfilerTask(asyncProfilerTaskCreationRequest: AsyncProfilerTaskCreationRequest!): AsyncProfilerTaskCreationResult!
+}
+
+extend type Query {
+    # Query all task lists and sort them in descending order by start time
+    queryAsyncProfilerTaskList(request: AsyncProfilerTaskListRequest!): AsyncProfilerTaskListResult!
+    # Query task progress, including task logs
+    queryAsyncProfilerTaskProgress(taskId: String!): AsyncProfilerTaskProgress!
+    # Query the flame graph produced by async-profiler
+    queryAsyncProfilerAnalyze(request: AsyncProfilerAnalyzationRequest!): AsyncProfilerAnalyzation!
+}
+```
+
 #### Out-process profiling
 
 ```graphql

docs/en/changes/changes.md

@@ -12,6 +12,7 @@
 * Add Component ID(5023) for the GoZero framework.
 * Support Kong monitoring.
 * Support adding additional attr[0-4] for service level metrics.
+* Support async-profiler feature for performance analysis.
 
 #### UI
 

docs/en/concepts-and-designs/profiling.md

@@ -17,6 +17,11 @@ In the SkyWalking landscape, we provided three ways to support profiling within
 ## In-process profiling
 
 In-process profiling is primarily provided by auto-instrument agents in the VM-based runtime.
+
+### Tracing Profiling
+Learn more tech details from the post, [**Use Profiling to Fix the Blind Spot of Distributed
+Tracing**](sdk-profiling.md).
+
 This feature resolves the issue <1> through capture the snapshot of the thread stacks periodically.
 The OAP would aggregate the thread stack per RPC request, and provide a hierarchy graph to indicate the slow methods
 based
@@ -25,11 +30,21 @@ on continuous snapshot.
 The period is usually every 10-100 milliseconds, which is not recommended to be less, due to this capture would usually
 cause classical stop-the-world for the VM, which would impact the whole process performance.
 
-Learn more tech details from the post, [**Use Profiling to Fix the Blind Spot of Distributed
-Tracing**](sdk-profiling.md).
-
 For now, Java and Python agents support this.
 
+### Async Profiler
+
+Async Profiler is a low overhead sampling profiler for Java that does not suffer from Safepoint bias problem. It features HotSpot-specific APIs to collect stack traces and to track memory allocations. The profiler works with OpenJDK and other Java runtimes based on the HotSpot JVM.
+
+async-profiler can trace the following kinds of events:
+
+CPU cycles
+Hardware and Software performance counters like cache misses, branch misses, page faults, context switches etc.
+Allocations in Java Heap
+Contented lock attempts, including both Java object monitors and ReentrantLocks
+
+For now, Java agent support this.
+
 ## Out-of-process profiling
 
 Out-of-process profiling leverage [eBPF](https://ebpf.io/) technology with origins in the Linux kernel.

docs/en/setup/backend/backend-async-profiler.md

@@ -0,0 +1,85 @@
+# Async Profiler
+
+Async Profiler is bound within the auto-instrument agent and corresponds to [In-Process Profiling](../../concepts-and-designs/profiling.md#in-process-profiling).
+
+It is passed to the proxy in the form of a task, allowing it to be enabled or disabled dynamically.
+When service encounters performance issues (cpu usage, memory allocation, locks), async-profiler task can be created.
+When the proxy receives a task, it enables Async Profiler for sampling.
+After sampling is completed, a flame graph will be generated for performance analysis to determine the specific business code line that caused the performance problem.
+
+## Activate async profiler in the OAP
+OAP and the agent use a brand-new protocol to exchange Async Profiler data, so it is necessary to start OAP with the following configuration:
+
+```yaml
+receiver-async-profiler:
+  selector: ${SW_RECEIVER_ASYNC_PROFILER:default}
+  default:
+```
+
+## Async Profiler Task with Analysis
+
+To use the Async Profiler feature, please follow these steps:
+
+1. **Create Async Profiler task**: Use the UI or CLI tool to create a task.
+2. **Wait agent collect data and upload**: Wait for Async Profiler to collect JFR data and report
+3. **Query task progress**: Query the progress of tasks, including analyzing successful and failed instances and task logs
+4. **Analyze the data**: Analyze the JFR data to determine where performance bottlenecks exist in the service.
+
+### Create an Async Profiler task
+
+Create an Async Profiler task to notify some java-agent instances in the execution service to start Async Profiler for data collection.
+
+When creating a task, the following configuration fields are required:
+
+1. **serviceId**: Define the service to execute the task.
+2. **serviceInstanceIds**: Define which instances need to execute tasks.
+3. **duration**: Define the duration of this task (second).
+4. **events**: Define which event types this task needs to collect.
+5. **execArgs**: Other Async Profiler execution options, e.g. alloc=2k,lock=2s.
+
+When the Agent receives a Async Profiler task from OAP, it automatically generates a log to notify that the task has been acknowledged. The log contains the following field information:
+
+1. **Instance**: The name of the instance where the Agent is located.
+2. **Type**: Supports "NOTIFIED" and "EXECUTION_FINISHED" and "JFR_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR", with the current log displaying "NOTIFIED".
+3. **Time**: The time when the Agent received the task.
+
+### Wait the agent to collect data and upload
+
+At this point, async-profiler will trace the following kinds of events:   
+
+1. CPU cycles
+2. Hardware and Software performance counters like cache misses, branch misses, page faults, context switches etc.
+3. Allocations in Java Heap
+4. Contented lock attempts, including both Java object monitors and ReentrantLocks
+
+Finally, java agent will upload the jfr file produced by async-profiler to the oap server for online performance analysis.
+
+### Query the profiling task progresses
+
+Wait for async-profiler to complete data collection and upload successfully，We can query the execution log of the async-profiler task and the successful and failed instances,which includes the following information:
+
+1. **successInstanceIds**: SuccessInstanceIds gives instances that have executed the task successfully.
+2. **errorInstanceIds**: ErrorInstanceIds gives instances that failed to execute the task.
+3. **logs**: All task execution logs of the current task.
+    1. **id**: The task id.
+    2. **instanceId**: InstanceId is the id of the instance which reported this task log.
+    3. **instanceName**: InstanceName is the name of the instance which reported this task log.
+    4. **operationType**: Contains "NOTIFIED" and "EXECUTION_FINISHED" and "JFR_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR".
+    5. **operationTime**: operationTime is the time when the operation occurs.
+
+### Analyze the profiling data
+
+Once some agents completed the task, we can analyze the data through the following query:
+
+1. **taskId**: The task id.
+2. **instanceIds**: InstanceIds defines the instances to be included for analysis
+3. **eventType**: EventType is the specific JFR Event type to be selected for analysis even if multiple events are included in the JFR file.
+
+After the query, the following data would be returned to render a flame graph:
+1. **type**: eventType in query parameters.
+2. **elements**: Combined with "id" to determine the hierarchical relationship.
+   1. **Id**: Id is the identity of the stack element.
+   2. **parentId**: Parent element ID. The dependency relationship between elements can be determined using the element ID and parent element ID.
+   3. **codeSignature**: Method signatures in tree nodes.
+   4. **total**:The total number of samples of the current tree node, including child nodes.
+   5. **self**: The sampling number of the current tree node, excluding samples of the children.