Skip to content

Commit

Permalink
Support async profiler feature (#12671)
Browse files Browse the repository at this point in the history
Co-authored-by: zhengziyi <[email protected]>
Co-authored-by: Jiajing LU <[email protected]>
Co-authored-by: 吴晟 Wu Sheng <[email protected]>
Co-authored-by: kezhenxu94 <[email protected]>
  • Loading branch information
5 people authored Oct 31, 2024
1 parent fd98b0f commit 9e5fb8f
Show file tree
Hide file tree
Showing 348 changed files with 6,811 additions and 21 deletions.
9 changes: 9 additions & 0 deletions .github/workflows/skywalking.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,15 @@ jobs:

- name: Cilium Service
config: test/e2e-v2/cases/cilium/e2e.yaml

- name: Async Profiler ES
config: test/e2e-v2/cases/profiling/async-profiler/es/e2e.yaml
- name: Async Profiler BanyanDB
config: test/e2e-v2/cases/profiling/async-profiler/banyandb/e2e.yaml
- name: Async Profiler MySQL
config: test/e2e-v2/cases/profiling/async-profiler/mysql/e2e.yaml
- name: Async Profiler h2
config: test/e2e-v2/cases/profiling/async-profiler/h2/e2e.yaml
steps:
- uses: actions/checkout@v3
with:
Expand Down
1 change: 1 addition & 0 deletions .licenserc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ header:
- '**/src/main/proto/opentelemetry/**'
- 'oap-server/server-starter/src/main/resources/version.properties'
- '**/mockito-extensions/**'
- 'oap-server/server-library/library-async-profiler-jfr-parser'

comment: on-failure

Expand Down
1 change: 1 addition & 0 deletions LICENSE
Original file line number Diff line number Diff line change
Expand Up @@ -225,3 +225,4 @@ The text of each license is the standard Apache 2.0 license.
mvnw files from https://github.com/apache/maven-wrapper Apache 2.0
svg files from skywalking-ui/src/assets/icons: https://github.com/google/material-design-icons Apache 2.0
ZipkinQueryHandler.java reference from zipkin2.server.internal.ZipkinQueryApiV2: https://github.com/openzipkin/zipkin Apache 2.0
async-profiler parser files in server-library/library-async-profiler-jfr-parser from https://github.com/async-profiler/async-profiler/tree/v3.0 Apache 2.0
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.skywalking.oap.server.network.trace.component.command;

import lombok.Getter;
import org.apache.skywalking.apm.network.common.v3.Command;
import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;

import java.util.List;
import java.util.Objects;

@Getter
public class AsyncProfilerTaskCommand extends BaseCommand implements Serializable, Deserializable<AsyncProfilerTaskCommand> {
public static final Deserializable<AsyncProfilerTaskCommand> DESERIALIZER = new AsyncProfilerTaskCommand("", "", 0, null, "", 0);
public static final String NAME = "AsyncProfilerTaskQuery";

/**
* async-profiler taskId
*/
private final String taskId;
/**
* run profiling for duration (second)
*/
private final int duration;
/**
* async profiler extended parameters. Here is a table of optional parameters.
*
* <p>lock[=DURATION] - profile contended locks overflowing the DURATION ns bucket (default: 10us)</p>
* <p>alloc[=BYTES] - profile allocations with BYTES interval</p>
* <p>interval=N - sampling interval in ns (default: 10'000'000, i.e. 10 ms)</p>
* <p>jstackdepth=N - maximum Java stack depth (default: 2048)</p>
* <p>chunksize=N - approximate size of JFR chunk in bytes (default: 100 MB) </p>
* <p>chunktime=N - duration of JFR chunk in seconds (default: 1 hour) </p>
* details @see <a href="https://github.com/async-profiler/async-profiler/blob/master/src/arguments.cpp#L44">async-profiler argument</a>
*/
private final String execArgs;
/**
* task create time
*/
private final long createTime;

public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration,
List<String> events, String execArgs, long createTime) {
super(NAME, serialNumber);
this.taskId = taskId;
this.duration = duration;
this.createTime = createTime;
String comma = ",";
StringBuilder sb = new StringBuilder();
if (Objects.nonNull(events) && !events.isEmpty()) {
sb.append("event=").append(String.join(comma, events));
}
if (execArgs != null && !execArgs.isEmpty()) {
sb.append(comma).append(execArgs);
}
this.execArgs = sb.toString();
}

public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration,
String execArgs, long createTime) {
super(NAME, serialNumber);
this.taskId = taskId;
this.duration = duration;
this.execArgs = execArgs;
this.createTime = createTime;
}

@Override
public AsyncProfilerTaskCommand deserialize(Command command) {
final List<KeyStringValuePair> argsList = command.getArgsList();
String taskId = null;
int duration = 0;
String execArgs = null;
long createTime = 0;
String serialNumber = null;
for (final KeyStringValuePair pair : argsList) {
if ("SerialNumber".equals(pair.getKey())) {
serialNumber = pair.getValue();
} else if ("TaskId".equals(pair.getKey())) {
taskId = pair.getValue();
} else if ("Duration".equals(pair.getKey())) {
duration = Integer.parseInt(pair.getValue());
} else if ("ExecArgs".equals(pair.getKey())) {
execArgs = pair.getValue();
} else if ("CreateTime".equals(pair.getKey())) {
createTime = Long.parseLong(pair.getValue());
}
}
return new AsyncProfilerTaskCommand(serialNumber, taskId, duration, execArgs, createTime);
}

@Override
public Command.Builder serialize() {
final Command.Builder builder = commandBuilder();
builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId))
.addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration)))
.addArgs(KeyStringValuePair.newBuilder().setKey("ExecArgs").setValue(execArgs))
.addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime)));
return builder;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ public static BaseCommand deserialize(final Command command) {
return ProfileTaskCommand.DESERIALIZER.deserialize(command);
} else if (ConfigurationDiscoveryCommand.NAME.equals(commandName)) {
return ConfigurationDiscoveryCommand.DESERIALIZER.deserialize(command);
} else if (AsyncProfilerTaskCommand.NAME.equals(commandName)) {
return AsyncProfilerTaskCommand.DESERIALIZER.deserialize(command);
}
throw new UnsupportedCommandException(command);
}
Expand Down
2 changes: 1 addition & 1 deletion apm-protocol/apm-network/src/main/proto
22 changes: 21 additions & 1 deletion docs/en/api/query-protocol.md
Original file line number Diff line number Diff line change
Expand Up @@ -189,10 +189,12 @@ extend type Query {
Event query fetches the event list based on given sources and time range conditions.

### Profiling
SkyWalking offers two types of [profiling](../concepts-and-designs/profiling.md), in-process and out-process, allowing users to create tasks and check their execution status.
SkyWalking offers two types of [profiling](../concepts-and-designs/profiling.md), in-process(tracing profiling and async-profiler) and out-process(ebpf profiling), allowing users to create tasks and check their execution status.

#### In-process profiling

##### tracing profiling

```graphql
extend type Mutation {
# crate new profile task
Expand All @@ -210,6 +212,24 @@ extend type Query {
}
```

##### async-profiler

```graphql
extend type Mutation {
# Create a new async-profiler task
createAsyncProfilerTask(asyncProfilerTaskCreationRequest: AsyncProfilerTaskCreationRequest!): AsyncProfilerTaskCreationResult!
}

extend type Query {
# Query all task lists and sort them in descending order by start time
queryAsyncProfilerTaskList(request: AsyncProfilerTaskListRequest!): AsyncProfilerTaskListResult!
# Query task progress, including task logs
queryAsyncProfilerTaskProgress(taskId: String!): AsyncProfilerTaskProgress!
# Query the flame graph produced by async-profiler
queryAsyncProfilerAnalyze(request: AsyncProfilerAnalyzationRequest!): AsyncProfilerAnalyzation!
}
```

#### Out-process profiling

```graphql
Expand Down
1 change: 1 addition & 0 deletions docs/en/changes/changes.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
* Add Component ID(5023) for the GoZero framework.
* Support Kong monitoring.
* Support adding additional attr[0-4] for service level metrics.
* Support async-profiler feature for performance analysis.

#### UI

Expand Down
21 changes: 18 additions & 3 deletions docs/en/concepts-and-designs/profiling.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ In the SkyWalking landscape, we provided three ways to support profiling within
## In-process profiling

In-process profiling is primarily provided by auto-instrument agents in the VM-based runtime.

### Tracing Profiling
Learn more tech details from the post, [**Use Profiling to Fix the Blind Spot of Distributed
Tracing**](sdk-profiling.md).

This feature resolves the issue <1> through capture the snapshot of the thread stacks periodically.
The OAP would aggregate the thread stack per RPC request, and provide a hierarchy graph to indicate the slow methods
based
Expand All @@ -25,11 +30,21 @@ on continuous snapshot.
The period is usually every 10-100 milliseconds, which is not recommended to be less, due to this capture would usually
cause classical stop-the-world for the VM, which would impact the whole process performance.

Learn more tech details from the post, [**Use Profiling to Fix the Blind Spot of Distributed
Tracing**](sdk-profiling.md).

For now, Java and Python agents support this.

### Async Profiler

Async Profiler is a low overhead sampling profiler for Java that does not suffer from Safepoint bias problem. It features HotSpot-specific APIs to collect stack traces and to track memory allocations. The profiler works with OpenJDK and other Java runtimes based on the HotSpot JVM.

async-profiler can trace the following kinds of events:

CPU cycles
Hardware and Software performance counters like cache misses, branch misses, page faults, context switches etc.
Allocations in Java Heap
Contented lock attempts, including both Java object monitors and ReentrantLocks

For now, Java agent support this.

## Out-of-process profiling

Out-of-process profiling leverage [eBPF](https://ebpf.io/) technology with origins in the Linux kernel.
Expand Down
85 changes: 85 additions & 0 deletions docs/en/setup/backend/backend-async-profiler.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
# Async Profiler

Async Profiler is bound within the auto-instrument agent and corresponds to [In-Process Profiling](../../concepts-and-designs/profiling.md#in-process-profiling).

It is passed to the proxy in the form of a task, allowing it to be enabled or disabled dynamically.
When service encounters performance issues (cpu usage, memory allocation, locks), async-profiler task can be created.
When the proxy receives a task, it enables Async Profiler for sampling.
After sampling is completed, a flame graph will be generated for performance analysis to determine the specific business code line that caused the performance problem.

## Activate async profiler in the OAP
OAP and the agent use a brand-new protocol to exchange Async Profiler data, so it is necessary to start OAP with the following configuration:

```yaml
receiver-async-profiler:
selector: ${SW_RECEIVER_ASYNC_PROFILER:default}
default:
```
## Async Profiler Task with Analysis
To use the Async Profiler feature, please follow these steps:
1. **Create Async Profiler task**: Use the UI or CLI tool to create a task.
2. **Wait agent collect data and upload**: Wait for Async Profiler to collect JFR data and report
3. **Query task progress**: Query the progress of tasks, including analyzing successful and failed instances and task logs
4. **Analyze the data**: Analyze the JFR data to determine where performance bottlenecks exist in the service.
### Create an Async Profiler task
Create an Async Profiler task to notify some java-agent instances in the execution service to start Async Profiler for data collection.
When creating a task, the following configuration fields are required:
1. **serviceId**: Define the service to execute the task.
2. **serviceInstanceIds**: Define which instances need to execute tasks.
3. **duration**: Define the duration of this task (second).
4. **events**: Define which event types this task needs to collect.
5. **execArgs**: Other Async Profiler execution options, e.g. alloc=2k,lock=2s.
When the Agent receives a Async Profiler task from OAP, it automatically generates a log to notify that the task has been acknowledged. The log contains the following field information:
1. **Instance**: The name of the instance where the Agent is located.
2. **Type**: Supports "NOTIFIED" and "EXECUTION_FINISHED" and "JFR_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR", with the current log displaying "NOTIFIED".
3. **Time**: The time when the Agent received the task.
### Wait the agent to collect data and upload
At this point, async-profiler will trace the following kinds of events:
1. CPU cycles
2. Hardware and Software performance counters like cache misses, branch misses, page faults, context switches etc.
3. Allocations in Java Heap
4. Contented lock attempts, including both Java object monitors and ReentrantLocks
Finally, java agent will upload the jfr file produced by async-profiler to the oap server for online performance analysis.
### Query the profiling task progresses
Wait for async-profiler to complete data collection and upload successfully,We can query the execution log of the async-profiler task and the successful and failed instances,which includes the following information:
1. **successInstanceIds**: SuccessInstanceIds gives instances that have executed the task successfully.
2. **errorInstanceIds**: ErrorInstanceIds gives instances that failed to execute the task.
3. **logs**: All task execution logs of the current task.
1. **id**: The task id.
2. **instanceId**: InstanceId is the id of the instance which reported this task log.
3. **instanceName**: InstanceName is the name of the instance which reported this task log.
4. **operationType**: Contains "NOTIFIED" and "EXECUTION_FINISHED" and "JFR_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR".
5. **operationTime**: operationTime is the time when the operation occurs.
### Analyze the profiling data
Once some agents completed the task, we can analyze the data through the following query:
1. **taskId**: The task id.
2. **instanceIds**: InstanceIds defines the instances to be included for analysis
3. **eventType**: EventType is the specific JFR Event type to be selected for analysis even if multiple events are included in the JFR file.
After the query, the following data would be returned to render a flame graph:
1. **type**: eventType in query parameters.
2. **elements**: Combined with "id" to determine the hierarchical relationship.
1. **Id**: Id is the identity of the stack element.
2. **parentId**: Parent element ID. The dependency relationship between elements can be determined using the element ID and parent element ID.
3. **codeSignature**: Method signatures in tree nodes.
4. **total**:The total number of samples of the current tree node, including child nodes.
5. **self**: The sampling number of the current tree node, excluding samples of the children.
Loading

0 comments on commit 9e5fb8f

Please sign in to comment.