-
Notifications
You must be signed in to change notification settings - Fork 6.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support async profiler feature (#12671)
Co-authored-by: zhengziyi <[email protected]> Co-authored-by: Jiajing LU <[email protected]> Co-authored-by: 吴晟 Wu Sheng <[email protected]> Co-authored-by: kezhenxu94 <[email protected]>
- Loading branch information
1 parent
fd98b0f
commit 9e5fb8f
Showing
348 changed files
with
6,811 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
118 changes: 118 additions & 0 deletions
118
...pache/skywalking/oap/server/network/trace/component/command/AsyncProfilerTaskCommand.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,118 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one | ||
* or more contributor license agreements. See the NOTICE file | ||
* distributed with this work for additional information | ||
* regarding copyright ownership. The ASF licenses this file | ||
* to you under the Apache License, Version 2.0 (the | ||
* "License"); you may not use this file except in compliance | ||
* with the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
|
||
package org.apache.skywalking.oap.server.network.trace.component.command; | ||
|
||
import lombok.Getter; | ||
import org.apache.skywalking.apm.network.common.v3.Command; | ||
import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; | ||
|
||
import java.util.List; | ||
import java.util.Objects; | ||
|
||
@Getter | ||
public class AsyncProfilerTaskCommand extends BaseCommand implements Serializable, Deserializable<AsyncProfilerTaskCommand> { | ||
public static final Deserializable<AsyncProfilerTaskCommand> DESERIALIZER = new AsyncProfilerTaskCommand("", "", 0, null, "", 0); | ||
public static final String NAME = "AsyncProfilerTaskQuery"; | ||
|
||
/** | ||
* async-profiler taskId | ||
*/ | ||
private final String taskId; | ||
/** | ||
* run profiling for duration (second) | ||
*/ | ||
private final int duration; | ||
/** | ||
* async profiler extended parameters. Here is a table of optional parameters. | ||
* | ||
* <p>lock[=DURATION] - profile contended locks overflowing the DURATION ns bucket (default: 10us)</p> | ||
* <p>alloc[=BYTES] - profile allocations with BYTES interval</p> | ||
* <p>interval=N - sampling interval in ns (default: 10'000'000, i.e. 10 ms)</p> | ||
* <p>jstackdepth=N - maximum Java stack depth (default: 2048)</p> | ||
* <p>chunksize=N - approximate size of JFR chunk in bytes (default: 100 MB) </p> | ||
* <p>chunktime=N - duration of JFR chunk in seconds (default: 1 hour) </p> | ||
* details @see <a href="https://github.com/async-profiler/async-profiler/blob/master/src/arguments.cpp#L44">async-profiler argument</a> | ||
*/ | ||
private final String execArgs; | ||
/** | ||
* task create time | ||
*/ | ||
private final long createTime; | ||
|
||
public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration, | ||
List<String> events, String execArgs, long createTime) { | ||
super(NAME, serialNumber); | ||
this.taskId = taskId; | ||
this.duration = duration; | ||
this.createTime = createTime; | ||
String comma = ","; | ||
StringBuilder sb = new StringBuilder(); | ||
if (Objects.nonNull(events) && !events.isEmpty()) { | ||
sb.append("event=").append(String.join(comma, events)); | ||
} | ||
if (execArgs != null && !execArgs.isEmpty()) { | ||
sb.append(comma).append(execArgs); | ||
} | ||
this.execArgs = sb.toString(); | ||
} | ||
|
||
public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration, | ||
String execArgs, long createTime) { | ||
super(NAME, serialNumber); | ||
this.taskId = taskId; | ||
this.duration = duration; | ||
this.execArgs = execArgs; | ||
this.createTime = createTime; | ||
} | ||
|
||
@Override | ||
public AsyncProfilerTaskCommand deserialize(Command command) { | ||
final List<KeyStringValuePair> argsList = command.getArgsList(); | ||
String taskId = null; | ||
int duration = 0; | ||
String execArgs = null; | ||
long createTime = 0; | ||
String serialNumber = null; | ||
for (final KeyStringValuePair pair : argsList) { | ||
if ("SerialNumber".equals(pair.getKey())) { | ||
serialNumber = pair.getValue(); | ||
} else if ("TaskId".equals(pair.getKey())) { | ||
taskId = pair.getValue(); | ||
} else if ("Duration".equals(pair.getKey())) { | ||
duration = Integer.parseInt(pair.getValue()); | ||
} else if ("ExecArgs".equals(pair.getKey())) { | ||
execArgs = pair.getValue(); | ||
} else if ("CreateTime".equals(pair.getKey())) { | ||
createTime = Long.parseLong(pair.getValue()); | ||
} | ||
} | ||
return new AsyncProfilerTaskCommand(serialNumber, taskId, duration, execArgs, createTime); | ||
} | ||
|
||
@Override | ||
public Command.Builder serialize() { | ||
final Command.Builder builder = commandBuilder(); | ||
builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId)) | ||
.addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration))) | ||
.addArgs(KeyStringValuePair.newBuilder().setKey("ExecArgs").setValue(execArgs)) | ||
.addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime))); | ||
return builder; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,85 @@ | ||
# Async Profiler | ||
|
||
Async Profiler is bound within the auto-instrument agent and corresponds to [In-Process Profiling](../../concepts-and-designs/profiling.md#in-process-profiling). | ||
|
||
It is passed to the proxy in the form of a task, allowing it to be enabled or disabled dynamically. | ||
When service encounters performance issues (cpu usage, memory allocation, locks), async-profiler task can be created. | ||
When the proxy receives a task, it enables Async Profiler for sampling. | ||
After sampling is completed, a flame graph will be generated for performance analysis to determine the specific business code line that caused the performance problem. | ||
|
||
## Activate async profiler in the OAP | ||
OAP and the agent use a brand-new protocol to exchange Async Profiler data, so it is necessary to start OAP with the following configuration: | ||
|
||
```yaml | ||
receiver-async-profiler: | ||
selector: ${SW_RECEIVER_ASYNC_PROFILER:default} | ||
default: | ||
``` | ||
## Async Profiler Task with Analysis | ||
To use the Async Profiler feature, please follow these steps: | ||
1. **Create Async Profiler task**: Use the UI or CLI tool to create a task. | ||
2. **Wait agent collect data and upload**: Wait for Async Profiler to collect JFR data and report | ||
3. **Query task progress**: Query the progress of tasks, including analyzing successful and failed instances and task logs | ||
4. **Analyze the data**: Analyze the JFR data to determine where performance bottlenecks exist in the service. | ||
### Create an Async Profiler task | ||
Create an Async Profiler task to notify some java-agent instances in the execution service to start Async Profiler for data collection. | ||
When creating a task, the following configuration fields are required: | ||
1. **serviceId**: Define the service to execute the task. | ||
2. **serviceInstanceIds**: Define which instances need to execute tasks. | ||
3. **duration**: Define the duration of this task (second). | ||
4. **events**: Define which event types this task needs to collect. | ||
5. **execArgs**: Other Async Profiler execution options, e.g. alloc=2k,lock=2s. | ||
When the Agent receives a Async Profiler task from OAP, it automatically generates a log to notify that the task has been acknowledged. The log contains the following field information: | ||
1. **Instance**: The name of the instance where the Agent is located. | ||
2. **Type**: Supports "NOTIFIED" and "EXECUTION_FINISHED" and "JFR_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR", with the current log displaying "NOTIFIED". | ||
3. **Time**: The time when the Agent received the task. | ||
### Wait the agent to collect data and upload | ||
At this point, async-profiler will trace the following kinds of events: | ||
1. CPU cycles | ||
2. Hardware and Software performance counters like cache misses, branch misses, page faults, context switches etc. | ||
3. Allocations in Java Heap | ||
4. Contented lock attempts, including both Java object monitors and ReentrantLocks | ||
Finally, java agent will upload the jfr file produced by async-profiler to the oap server for online performance analysis. | ||
### Query the profiling task progresses | ||
Wait for async-profiler to complete data collection and upload successfully,We can query the execution log of the async-profiler task and the successful and failed instances,which includes the following information: | ||
1. **successInstanceIds**: SuccessInstanceIds gives instances that have executed the task successfully. | ||
2. **errorInstanceIds**: ErrorInstanceIds gives instances that failed to execute the task. | ||
3. **logs**: All task execution logs of the current task. | ||
1. **id**: The task id. | ||
2. **instanceId**: InstanceId is the id of the instance which reported this task log. | ||
3. **instanceName**: InstanceName is the name of the instance which reported this task log. | ||
4. **operationType**: Contains "NOTIFIED" and "EXECUTION_FINISHED" and "JFR_UPLOAD_FILE_TOO_LARGE_ERROR", "EXECUTION_TASK_ERROR". | ||
5. **operationTime**: operationTime is the time when the operation occurs. | ||
### Analyze the profiling data | ||
Once some agents completed the task, we can analyze the data through the following query: | ||
1. **taskId**: The task id. | ||
2. **instanceIds**: InstanceIds defines the instances to be included for analysis | ||
3. **eventType**: EventType is the specific JFR Event type to be selected for analysis even if multiple events are included in the JFR file. | ||
After the query, the following data would be returned to render a flame graph: | ||
1. **type**: eventType in query parameters. | ||
2. **elements**: Combined with "id" to determine the hierarchical relationship. | ||
1. **Id**: Id is the identity of the stack element. | ||
2. **parentId**: Parent element ID. The dependency relationship between elements can be determined using the element ID and parent element ID. | ||
3. **codeSignature**: Method signatures in tree nodes. | ||
4. **total**:The total number of samples of the current tree node, including child nodes. | ||
5. **self**: The sampling number of the current tree node, excluding samples of the children. |
Oops, something went wrong.