From a73d03b10be7303636b9a2a236b9cde42e003144 Mon Sep 17 00:00:00 2001 From: zhengziyi Date: Sun, 26 May 2024 16:20:18 +0800 Subject: [PATCH 01/14] feat: add async-profiler start/stop demo --- apm-protocol/apm-network/pom.xml | 4 + .../command/AsyncProfilerTaskCommand.java | 510 ++++++++++++++ .../command/CommandDeserializer.java | 3 + .../core/asyncprofiler/AsyncProfilerTask.java | 644 ++++++++++++++++++ .../AsyncProfilerTaskChannelService.java | 116 ++++ .../AsyncProfilerTaskExecutionService.java | 76 +++ .../core/commands/CommandExecutorService.java | 10 +- .../AsyncProfilerCommandExecutor.java | 42 ++ .../profile/ProfileTaskChannelService.java | 15 +- ...skywalking.apm.agent.core.boot.BootService | 2 + pom.xml | 7 + 11 files changed, 1420 insertions(+), 9 deletions(-) create mode 100644 apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java create mode 100644 apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java create mode 100644 apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java create mode 100644 apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java create mode 100644 apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java diff --git a/apm-protocol/apm-network/pom.xml b/apm-protocol/apm-network/pom.xml index f7d68fbcbe..eb56675efe 100644 --- a/apm-protocol/apm-network/pom.xml +++ b/apm-protocol/apm-network/pom.xml @@ -78,6 +78,10 @@ ${org.apache.tomcat.annotations-api.version} provided + + io.pyroscope + async-profiler-context + diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java new file mode 100644 index 0000000000..d89fca7c14 --- /dev/null +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java @@ -0,0 +1,510 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.network.trace.component.command; + +import org.apache.skywalking.apm.network.common.v3.Command; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; + +import java.util.List; + +/** + * TODO add full arg + */ +public class AsyncProfilerTaskCommand extends BaseCommand implements Serializable, Deserializable { + public static final Deserializable DESERIALIZER = new AsyncProfilerTaskCommand("", ""); + public static final String NAME = "AsyncProfilerQuery"; + + private String action; + private String actionArg; + + /** + * which event to trace (cpu, wall, cache-misses, etc.) + */ + private String event; + + /** + * profile allocations with BYTES interval + * according to async-profiler README, alloc may contains non-numeric characters + */ + private String alloc; + + /** + * build allocation profile from live objects only + */ + private boolean live; + + /** + * profile contended locks longer than DURATION ns + * according to async-profiler README, alloc may contains non-numeric characters + */ + private String lock; + + /** + * start Java Flight Recording with the given config along with the profiler + */ + private String jfrsync; + + /** + * output file name for dumping + */ + private String file; + + /** + * output file format, default value is html. + */ + private String format; + + /** + * sampling interval in ns (default: 10'000'000, i.e. 10 ms) + */ + private Long interval; + + /** + * maximum Java stack depth (default: 2048) + */ + private Integer jstackdepth; + + /** + * profile different threads separately + */ + private boolean threads; + + /** + * group threads by scheduling policy + */ + private boolean sched; + + /** + * how to collect C stack frames in addition to Java stack + * MODE is 'fp' (Frame Pointer), 'dwarf', 'lbr' (Last Branch Record) or 'no' + */ + private String cstack; + + /** + * use simple class names instead of FQN + */ + private boolean simple; + + /** + * print method signatures + */ + private boolean sig; + + /** + * annotate Java methods + */ + private boolean ann; + + /** + * prepend library names + */ + private boolean lib; + + /** + * include only user-mode events + */ + private boolean alluser; + + /** + * run profiling for duration seconds + */ + private Long duration; + + /** + * include stack traces containing PATTERN + */ + private List includes; + + /** + * exclude stack traces containing PATTERN + */ + private List excludes; + + /** + * automatically start profiling when the specified native function is executed. + */ + private String begin; + + /** + * automatically stop profiling when the specified native function is executed. + */ + private String end; + + /** + * time-to-safepoint profiling. + * An alias for --begin SafepointSynchronize::begin --end RuntimeService::record_safepoint_synchronized + */ + private boolean ttsp; + + /** + * FlameGraph title + */ + private String title; + + /** + * FlameGraph minimum frame width in percent + */ + private String minwidth; + + /** + * generate stack-reversed FlameGraph / Call tree + */ + private boolean reverse; + + /** + * count the total value (time, bytes, etc.) instead of samples + */ + private boolean total; + + /** + * approximate size of JFR chunk in bytes (default: 100 MB) + */ + private String chunksize; + + /** + * duration of JFR chunk in seconds (default: 1 hour) + */ + private String chunktime; + + /** + * run profiler in a loop (continuous profiling) + */ + private String loop; + + /** + * automatically stop profiler at TIME (absolute or relative) + */ + private String timeout; + + public enum ProfilerAction { + // start, resume, stop, dump, check, status, meminfo, list, collect, + start, resume, stop, dump, check, status, meminfo, list, collect, + version, + } + + public AsyncProfilerTaskCommand(String command, String serialNumber) { + super(command, serialNumber); + } + + @Override + public Command.Builder serialize() { + final Command.Builder builder = commandBuilder(); + builder.addArgs(KeyStringValuePair.newBuilder().setKey("Action").setValue(action)) + .addArgs(KeyStringValuePair.newBuilder().setKey("ActionArg").setValue(actionArg)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Event").setValue(event)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Alloc").setValue(alloc)); + return builder; + } + + @Override + public AsyncProfilerTaskCommand deserialize(Command command) { + List args = command.getArgsList(); + + String serialNumber = null; + String action = null; + String actionArg = null; + String event = null; + String alloc = null; + + for (final KeyStringValuePair pair : args) { + if ("SerialNumber".equals(pair.getKey())) { + serialNumber = pair.getValue(); + } else if ("Action".equals(pair.getKey())) { + action = pair.getValue(); + } else if ("ActionArg".equals(pair.getKey())) { + actionArg = pair.getValue(); + } else if ("Event".equals(pair.getKey())) { + event = pair.getValue(); + } else if ("Alloc".equals(pair.getKey())) { + alloc = pair.getValue(); + } + } + + AsyncProfilerTaskCommand asyncProfilerTaskCommand = new AsyncProfilerTaskCommand(NAME, serialNumber); + asyncProfilerTaskCommand.setAction(action); + asyncProfilerTaskCommand.setActionArg(actionArg); + asyncProfilerTaskCommand.setEvent(event); + asyncProfilerTaskCommand.setAlloc(alloc); + return asyncProfilerTaskCommand; + } + + public String getAction() { + return action; + } + + public void setAction(String action) { + this.action = action; + } + + public String getActionArg() { + return actionArg; + } + + public void setActionArg(String actionArg) { + this.actionArg = actionArg; + } + + public String getEvent() { + return event; + } + + public void setEvent(String event) { + this.event = event; + } + + public String getAlloc() { + return alloc; + } + + public void setAlloc(String alloc) { + this.alloc = alloc; + } + + public boolean isLive() { + return live; + } + + public void setLive(boolean live) { + this.live = live; + } + + public String getLock() { + return lock; + } + + public void setLock(String lock) { + this.lock = lock; + } + + public String getJfrsync() { + return jfrsync; + } + + public void setJfrsync(String jfrsync) { + this.jfrsync = jfrsync; + } + + public String getFile() { + return file; + } + + public void setFile(String file) { + this.file = file; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public Long getInterval() { + return interval; + } + + public void setInterval(Long interval) { + this.interval = interval; + } + + public Integer getJstackdepth() { + return jstackdepth; + } + + public void setJstackdepth(Integer jstackdepth) { + this.jstackdepth = jstackdepth; + } + + public boolean isThreads() { + return threads; + } + + public void setThreads(boolean threads) { + this.threads = threads; + } + + public boolean isSched() { + return sched; + } + + public void setSched(boolean sched) { + this.sched = sched; + } + + public String getCstack() { + return cstack; + } + + public void setCstack(String cstack) { + this.cstack = cstack; + } + + public boolean isSimple() { + return simple; + } + + public void setSimple(boolean simple) { + this.simple = simple; + } + + public boolean isSig() { + return sig; + } + + public void setSig(boolean sig) { + this.sig = sig; + } + + public boolean isAnn() { + return ann; + } + + public void setAnn(boolean ann) { + this.ann = ann; + } + + public boolean isLib() { + return lib; + } + + public void setLib(boolean lib) { + this.lib = lib; + } + + public boolean isAlluser() { + return alluser; + } + + public void setAlluser(boolean alluser) { + this.alluser = alluser; + } + + public Long getDuration() { + return duration; + } + + public void setDuration(Long duration) { + this.duration = duration; + } + + public List getIncludes() { + return includes; + } + + public void setIncludes(List includes) { + this.includes = includes; + } + + public List getExcludes() { + return excludes; + } + + public void setExcludes(List excludes) { + this.excludes = excludes; + } + + public String getBegin() { + return begin; + } + + public void setBegin(String begin) { + this.begin = begin; + } + + public String getEnd() { + return end; + } + + public void setEnd(String end) { + this.end = end; + } + + public boolean isTtsp() { + return ttsp; + } + + public void setTtsp(boolean ttsp) { + this.ttsp = ttsp; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getMinwidth() { + return minwidth; + } + + public void setMinwidth(String minwidth) { + this.minwidth = minwidth; + } + + public boolean isReverse() { + return reverse; + } + + public void setReverse(boolean reverse) { + this.reverse = reverse; + } + + public boolean isTotal() { + return total; + } + + public void setTotal(boolean total) { + this.total = total; + } + + public String getChunksize() { + return chunksize; + } + + public void setChunksize(String chunksize) { + this.chunksize = chunksize; + } + + public String getChunktime() { + return chunktime; + } + + public void setChunktime(String chunktime) { + this.chunktime = chunktime; + } + + public String getLoop() { + return loop; + } + + public void setLoop(String loop) { + this.loop = loop; + } + + public String getTimeout() { + return timeout; + } + + public void setTimeout(String timeout) { + this.timeout = timeout; + } +} diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/CommandDeserializer.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/CommandDeserializer.java index ff8680bcb3..4fd737ff98 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/CommandDeserializer.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/CommandDeserializer.java @@ -27,7 +27,10 @@ public static BaseCommand deserialize(final Command command) { return ProfileTaskCommand.DESERIALIZER.deserialize(command); } else if (ConfigurationDiscoveryCommand.NAME.equals(commandName)) { return ConfigurationDiscoveryCommand.DESERIALIZER.deserialize(command); + } else if (AsyncProfilerTaskCommand.NAME.equals(commandName)) { + return AsyncProfilerTaskCommand.DESERIALIZER.deserialize(command); } + throw new UnsupportedCommandException(command); } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java new file mode 100644 index 0000000000..00065e8a01 --- /dev/null +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java @@ -0,0 +1,644 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.asyncprofiler; + +import io.pyroscope.one.profiler.AsyncProfiler; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand; +import org.apache.skywalking.apm.util.StringUtil; + +import java.io.File; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; + +public class AsyncProfilerTask { + private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTask.class); + private static final String OUTPUT_DIR = "skywalking-output"; + + private AsyncProfilerTaskCommand.ProfilerAction action; + private String actionArg; + + /** + * which event to trace (cpu, wall, cache-misses, etc.) + */ + private String event; + + /** + * profile allocations with BYTES interval + * according to async-profiler README, alloc may contains non-numeric characters + */ + private String alloc; + + /** + * build allocation profile from live objects only + */ + private boolean live; + + /** + * profile contended locks longer than DURATION ns + * according to async-profiler README, alloc may contains non-numeric characters + */ + private String lock; + + /** + * start Java Flight Recording with the given config along with the profiler + */ + private String jfrsync; + + /** + * output file name for dumping + */ + private String file; + + /** + * output file format, default value is html. + */ + private String format; + + /** + * sampling interval in ns (default: 10'000'000, i.e. 10 ms) + */ + private Long interval; + + /** + * maximum Java stack depth (default: 2048) + */ + private Integer jstackdepth; + + /** + * profile different threads separately + */ + private boolean threads; + + /** + * group threads by scheduling policy + */ + private boolean sched; + + /** + * how to collect C stack frames in addition to Java stack + * MODE is 'fp' (Frame Pointer), 'dwarf', 'lbr' (Last Branch Record) or 'no' + */ + private String cstack; + + /** + * use simple class names instead of FQN + */ + private boolean simple; + + /** + * print method signatures + */ + private boolean sig; + + /** + * annotate Java methods + */ + private boolean ann; + + /** + * prepend library names + */ + private boolean lib; + + /** + * include only user-mode events + */ + private boolean alluser; + + /** + * run profiling for duration seconds + */ + private Long duration; + + /** + * include stack traces containing PATTERN + */ + private List includes; + + /** + * exclude stack traces containing PATTERN + */ + private List excludes; + + /** + * automatically start profiling when the specified native function is executed. + */ + private String begin; + + /** + * automatically stop profiling when the specified native function is executed. + */ + private String end; + + /** + * time-to-safepoint profiling. + * An alias for --begin SafepointSynchronize::begin --end RuntimeService::record_safepoint_synchronized + */ + private boolean ttsp; + + /** + * FlameGraph title + */ + private String title; + + /** + * FlameGraph minimum frame width in percent + */ + private String minwidth; + + /** + * generate stack-reversed FlameGraph / Call tree + */ + private boolean reverse; + + /** + * count the total value (time, bytes, etc.) instead of samples + */ + private boolean total; + + /** + * approximate size of JFR chunk in bytes (default: 100 MB) + */ + private String chunksize; + + /** + * duration of JFR chunk in seconds (default: 1 hour) + */ + private String chunktime; + + /** + * run profiler in a loop (continuous profiling) + */ + private String loop; + + /** + * automatically stop profiler at TIME (absolute or relative) + */ + private String timeout; + + private String getExecuteArgs() { + StringBuilder sb = new StringBuilder(); + final char comma = ','; + + // start - start profiling + // resume - start or resume profiling without resetting collected data + // stop - stop profiling + sb.append(action).append(comma); + + if (this.event != null) { + sb.append("event=").append(this.event).append(comma); + } + if (this.alloc != null) { + sb.append("alloc=").append(this.alloc).append(comma); + } + if (this.live) { + sb.append("live").append(comma); + } + if (this.lock != null) { + sb.append("lock=").append(this.lock).append(comma); + } + if (this.jfrsync != null) { + this.format = "jfr"; + sb.append("jfrsync=").append(this.jfrsync).append(comma); + } + if (this.file != null) { + sb.append("file=").append(this.file).append(comma); + } + if (this.format != null) { + sb.append(this.format).append(comma); + } + if (this.interval != null) { + sb.append("interval=").append(this.interval).append(comma); + } + if (this.jstackdepth != null) { + sb.append("jstackdepth=").append(this.jstackdepth).append(comma); + } + if (this.threads) { + sb.append("threads").append(comma); + } + if (this.sched) { + sb.append("sched").append(comma); + } + if (this.cstack != null) { + sb.append("cstack=").append(this.cstack).append(comma); + } + if (this.simple) { + sb.append("simple").append(comma); + } + if (this.sig) { + sb.append("sig").append(comma); + } + if (this.ann) { + sb.append("ann").append(comma); + } + if (this.lib) { + sb.append("lib").append(comma); + } + if (this.alluser) { + sb.append("alluser").append(comma); + } + if (this.includes != null) { + for (String include : includes) { + sb.append("include=").append(include).append(comma); + } + } + if (this.excludes != null) { + for (String exclude : excludes) { + sb.append("exclude=").append(exclude).append(comma); + } + } + if (this.ttsp) { + this.begin = "SafepointSynchronize::begin"; + this.end = "RuntimeService::record_safepoint_synchronized"; + } + if (this.begin != null) { + sb.append("begin=").append(this.begin).append(comma); + } + if (this.end != null) { + sb.append("end=").append(this.end).append(comma); + } + + if (this.title != null) { + sb.append("title=").append(this.title).append(comma); + } + if (this.minwidth != null) { + sb.append("minwidth=").append(this.minwidth).append(comma); + } + if (this.reverse) { + sb.append("reverse").append(comma); + } + if (this.total) { + sb.append("total").append(comma); + } + if (this.chunksize != null) { + sb.append("chunksize=").append(this.chunksize).append(comma); + } + if (this.chunktime != null) { + sb.append("chunktime=").append(this.chunktime).append(comma); + } + if (this.loop != null) { + sb.append("loop=").append(this.loop).append(comma); + } + if (this.timeout != null) { + sb.append("timeout=").append(this.timeout).append(comma); + } + return sb.toString(); + } + + private static String execute(AsyncProfiler asyncProfiler, String arg) + throws IllegalArgumentException, IOException { + LOGGER.info("async profiler execute arg:{}", arg); + String result = asyncProfiler.execute(arg); + if (!result.endsWith("\n")) { + result += "\n"; + } + return result; + } + + public String process(AsyncProfiler asyncProfiler) throws IOException { + if (AsyncProfilerTaskCommand.ProfilerAction.start.equals(action)) { + String executeArgs = getExecuteArgs(); + return execute(asyncProfiler, executeArgs); + } else if (AsyncProfilerTaskCommand.ProfilerAction.stop.equals(action)) { + return processStop(asyncProfiler); + } + + return ""; + } + + private String processStop(AsyncProfiler asyncProfiler) throws IOException { + String outputFile = outputFile(); + String executeArgs = getExecuteArgs(); + LOGGER.info("dump async profiler out put file at:{}", outputFile); + return execute(asyncProfiler, executeArgs); + } + + /** + * This method should only be called when {@code this.file == null} is true. + */ + private String outputFileExt() { + String fileExt = ""; + if (this.format == null) { + fileExt = "html"; + } else if (this.format.startsWith("flat") || this.format.startsWith("traces") + || this.format.equals("collapsed")) { + fileExt = "txt"; + } else if (this.format.equals("flamegraph") || this.format.equals("tree")) { + fileExt = "html"; + } else if (this.format.equals("jfr")) { + fileExt = "jfr"; + } else { + // illegal -o option makes async-profiler use flat + fileExt = "txt"; + } + return fileExt; + } + + public String getOutputPath() { + String userDir = System.getProperty("user.dir"); + if (StringUtil.isBlank(userDir)) { + LOGGER.warn("get user directory from system properties failed"); + } + return userDir + File.separator + OUTPUT_DIR; + } + + // TODO create dir when dir is not exist + private String outputFile() throws IOException { + if (this.file == null) { + String fileExt = outputFileExt(); + String outputPath = getOutputPath(); + if (outputPath != null) { + file = new File(outputPath, + new SimpleDateFormat("yyyyMMdd-HHmmss").format(new Date()) + "." + fileExt) + .getAbsolutePath(); + } else { + this.file = File.createTempFile(OUTPUT_DIR, "." + fileExt).getAbsolutePath(); + } + } + return file; + } + + public AsyncProfilerTaskCommand.ProfilerAction getAction() { + return action; + } + + public void setAction(String action) { + this.action = AsyncProfilerTaskCommand.ProfilerAction.valueOf(action); + } + + public String getActionArg() { + return actionArg; + } + + public void setActionArg(String actionArg) { + this.actionArg = actionArg; + } + + public String getEvent() { + return event; + } + + public void setEvent(String event) { + this.event = event; + } + + public String getAlloc() { + return alloc; + } + + public void setAlloc(String alloc) { + this.alloc = alloc; + } + + public boolean isLive() { + return live; + } + + public void setLive(boolean live) { + this.live = live; + } + + public String getLock() { + return lock; + } + + public void setLock(String lock) { + this.lock = lock; + } + + public String getJfrsync() { + return jfrsync; + } + + public void setJfrsync(String jfrsync) { + this.jfrsync = jfrsync; + } + + public String getFile() { + return file; + } + + public void setFile(String file) { + this.file = file; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public Long getInterval() { + return interval; + } + + public void setInterval(Long interval) { + this.interval = interval; + } + + public Integer getJstackdepth() { + return jstackdepth; + } + + public void setJstackdepth(Integer jstackdepth) { + this.jstackdepth = jstackdepth; + } + + public boolean isThreads() { + return threads; + } + + public void setThreads(boolean threads) { + this.threads = threads; + } + + public boolean isSched() { + return sched; + } + + public void setSched(boolean sched) { + this.sched = sched; + } + + public String getCstack() { + return cstack; + } + + public void setCstack(String cstack) { + this.cstack = cstack; + } + + public boolean isSimple() { + return simple; + } + + public void setSimple(boolean simple) { + this.simple = simple; + } + + public boolean isSig() { + return sig; + } + + public void setSig(boolean sig) { + this.sig = sig; + } + + public boolean isAnn() { + return ann; + } + + public void setAnn(boolean ann) { + this.ann = ann; + } + + public boolean isLib() { + return lib; + } + + public void setLib(boolean lib) { + this.lib = lib; + } + + public boolean isAlluser() { + return alluser; + } + + public void setAlluser(boolean alluser) { + this.alluser = alluser; + } + + public Long getDuration() { + return duration; + } + + public void setDuration(Long duration) { + this.duration = duration; + } + + public List getIncludes() { + return includes; + } + + public void setIncludes(List includes) { + this.includes = includes; + } + + public List getExcludes() { + return excludes; + } + + public void setExcludes(List excludes) { + this.excludes = excludes; + } + + public String getBegin() { + return begin; + } + + public void setBegin(String begin) { + this.begin = begin; + } + + public String getEnd() { + return end; + } + + public void setEnd(String end) { + this.end = end; + } + + public boolean isTtsp() { + return ttsp; + } + + public void setTtsp(boolean ttsp) { + this.ttsp = ttsp; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getMinwidth() { + return minwidth; + } + + public void setMinwidth(String minwidth) { + this.minwidth = minwidth; + } + + public boolean isReverse() { + return reverse; + } + + public void setReverse(boolean reverse) { + this.reverse = reverse; + } + + public boolean isTotal() { + return total; + } + + public void setTotal(boolean total) { + this.total = total; + } + + public String getChunksize() { + return chunksize; + } + + public void setChunksize(String chunksize) { + this.chunksize = chunksize; + } + + public String getChunktime() { + return chunktime; + } + + public void setChunktime(String chunktime) { + this.chunktime = chunktime; + } + + public String getLoop() { + return loop; + } + + public void setLoop(String loop) { + this.loop = loop; + } + + public String getTimeout() { + return timeout; + } + + public void setTimeout(String timeout) { + this.timeout = timeout; + } +} diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java new file mode 100644 index 0000000000..1912b49509 --- /dev/null +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.asyncprofiler; + +import org.apache.skywalking.apm.agent.core.boot.BootService; +import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; +import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.commands.CommandService; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus; +import org.apache.skywalking.apm.network.common.v3.Command; +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand; +import org.apache.skywalking.apm.util.RunnableWithExceptionProtection; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * TODO

add receive grpc command

+ * now just debug AsyncProfilerTask + */ +@DefaultImplementor +public class AsyncProfilerTaskChannelService implements BootService, Runnable { + private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTaskChannelService.class); + + // channel status + private volatile GRPCChannelStatus status = GRPCChannelStatus.DISCONNECT; + + // query task list schedule + private volatile ScheduledFuture getTaskListFuture; + + @Override + public void run() { + if (status == GRPCChannelStatus.CONNECTED) { + // test start command and 10s after put stop command + Command startCommand = Command.newBuilder() + .addArgs(KeyStringValuePair.newBuilder().setKey("Action").setValue("start")) + .addArgs(KeyStringValuePair.newBuilder().setKey("SerialNumber").setValue("1")) + .setCommand(AsyncProfilerTaskCommand.NAME) + .build(); + Commands startCommands = Commands.newBuilder() + .addCommands(startCommand) + .build(); + ServiceManager.INSTANCE.findService(CommandService.class).receiveCommand(startCommands); + + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + Command stopCommand = Command.newBuilder() + .addArgs(KeyStringValuePair.newBuilder().setKey("Action").setValue("stop")) + .addArgs(KeyStringValuePair.newBuilder().setKey("SerialNumber").setValue("2")) + .setCommand(AsyncProfilerTaskCommand.NAME) + .build(); + Commands stopCommands = Commands.newBuilder() + .addCommands(stopCommand) + .build(); + ServiceManager.INSTANCE.findService(CommandService.class).receiveCommand(stopCommands); + + status = GRPCChannelStatus.DISCONNECT; + } + } + + @Override + public void prepare() throws Throwable { + status = GRPCChannelStatus.CONNECTED; + } + + @Override + public void boot() throws Throwable { + getTaskListFuture = Executors.newSingleThreadScheduledExecutor( + new DefaultNamedThreadFactory("AsyncProfileGetTaskService") + ).scheduleWithFixedDelay( + new RunnableWithExceptionProtection( + this, + t -> LOGGER.error("Query async profile task list failure.", t) + ), 0, Config.Collector.GET_PROFILE_TASK_INTERVAL, TimeUnit.SECONDS + ); + } + + @Override + public void onComplete() throws Throwable { + + } + + @Override + public void shutdown() throws Throwable { + if (getTaskListFuture != null) { + getTaskListFuture.cancel(true); + } + } +} diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java new file mode 100644 index 0000000000..79c988ec9b --- /dev/null +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.asyncprofiler; + +import io.pyroscope.one.profiler.AsyncProfiler; +import org.apache.skywalking.apm.agent.core.boot.BootService; +import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; +import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@DefaultImplementor +public class AsyncProfilerTaskExecutionService implements BootService { + + private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTaskChannelService.class); + + //TODO use local so file for debug + private static final AsyncProfiler ASYNC_PROFILER = AsyncProfiler.getInstance("/opt/async-profiler/lib/libasyncProfiler.so"); + + // profile executor thread pool, only running one thread + private final static ExecutorService ASYNC_PROFILE_EXECUTOR = Executors.newSingleThreadExecutor( + new DefaultNamedThreadFactory("ASYNC-PROFILING-TASK")); + + public void processAsyncProfilerTask(AsyncProfilerTask task) { + // add task to list + LOGGER.info("add async profiler task: {}", task); +// asyncProfileTaskList.add(task); + ASYNC_PROFILE_EXECUTOR.execute(() -> { + try { + task.process(ASYNC_PROFILER); + } catch (IOException e) { + LOGGER.error("AsyncProfilerTask executor error:" + e.getMessage(), e); + } + }); + } + + @Override + public void prepare() throws Throwable { + + } + + @Override + public void boot() throws Throwable { + + } + + @Override + public void onComplete() throws Throwable { + + } + + @Override + public void shutdown() throws Throwable { + ASYNC_PROFILE_EXECUTOR.shutdown(); + } +} diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/CommandExecutorService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/CommandExecutorService.java index 819b0b9ff1..8a4b071f6a 100755 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/CommandExecutorService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/CommandExecutorService.java @@ -17,17 +17,20 @@ package org.apache.skywalking.apm.agent.core.commands; -import java.util.HashMap; -import java.util.Map; import org.apache.skywalking.apm.agent.core.boot.BootService; import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; +import org.apache.skywalking.apm.agent.core.commands.executor.AsyncProfilerCommandExecutor; import org.apache.skywalking.apm.agent.core.commands.executor.ConfigurationDiscoveryCommandExecutor; import org.apache.skywalking.apm.agent.core.commands.executor.NoopCommandExecutor; import org.apache.skywalking.apm.agent.core.commands.executor.ProfileTaskCommandExecutor; +import org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand; import org.apache.skywalking.apm.network.trace.component.command.BaseCommand; import org.apache.skywalking.apm.network.trace.component.command.ConfigurationDiscoveryCommand; import org.apache.skywalking.apm.network.trace.component.command.ProfileTaskCommand; +import java.util.HashMap; +import java.util.Map; + /** * Command executor service, acts like a routing executor that controls all commands' execution, is responsible for * managing all the mappings between commands and their executors, one can simply invoke {@link #execute(BaseCommand)} @@ -48,6 +51,9 @@ public void prepare() throws Throwable { //Get ConfigurationDiscoveryCommand executor. commandExecutorMap.put(ConfigurationDiscoveryCommand.NAME, new ConfigurationDiscoveryCommandExecutor()); + + // AsyncProfiler task executor + commandExecutorMap.put(AsyncProfilerTaskCommand.NAME, new AsyncProfilerCommandExecutor()); } @Override diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java new file mode 100644 index 0000000000..8b92ca8d9f --- /dev/null +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.commands.executor; + +import org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTask; +import org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTaskExecutionService; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.commands.CommandExecutionException; +import org.apache.skywalking.apm.agent.core.commands.CommandExecutor; +import org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand; +import org.apache.skywalking.apm.network.trace.component.command.BaseCommand; + +public class AsyncProfilerCommandExecutor implements CommandExecutor { + @Override + public void execute(BaseCommand command) throws CommandExecutionException { + AsyncProfilerTaskCommand asyncProfilerTaskCommand = (AsyncProfilerTaskCommand) command; + + AsyncProfilerTask asyncProfilerTask = new AsyncProfilerTask(); + asyncProfilerTask.setAction(asyncProfilerTaskCommand.getAction()); + asyncProfilerTask.setActionArg(asyncProfilerTaskCommand.getActionArg()); + asyncProfilerTask.setEvent(asyncProfilerTaskCommand.getEvent()); + asyncProfilerTask.setAlloc(asyncProfilerTask.getAlloc()); + ServiceManager.INSTANCE.findService(AsyncProfilerTaskExecutionService.class) + .processAsyncProfilerTask(asyncProfilerTask); + } +} diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java index cfe1465ad3..f0d2e10b02 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java @@ -21,13 +21,6 @@ import io.grpc.Channel; import io.grpc.Status; import io.grpc.StatusRuntimeException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; import org.apache.skywalking.apm.agent.core.boot.BootService; import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory; @@ -45,6 +38,14 @@ import org.apache.skywalking.apm.network.language.profile.v3.ProfileTaskGrpc; import org.apache.skywalking.apm.util.RunnableWithExceptionProtection; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT; /** diff --git a/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService b/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService index cfda93521c..7b5e4864ef 100644 --- a/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService +++ b/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService @@ -36,3 +36,5 @@ org.apache.skywalking.apm.agent.core.remote.LogReportServiceClient org.apache.skywalking.apm.agent.core.conf.dynamic.ConfigurationDiscoveryService org.apache.skywalking.apm.agent.core.remote.EventReportServiceClient org.apache.skywalking.apm.agent.core.ServiceInstanceGenerator +org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTaskExecutionService +org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTaskChannelService \ No newline at end of file diff --git a/pom.xml b/pom.xml index c6dca564f9..ac4f1864b7 100755 --- a/pom.xml +++ b/pom.xml @@ -87,6 +87,7 @@ 1.14.9 + 0.14.0 1.53.0 4.1.100.Final 2.8.9 @@ -213,6 +214,12 @@ + + io.pyroscope + async-profiler-context + ${async-profiler.version} + + junit junit From 4c9995a329fdee83cbee97cecbb7afbc3caa1ad4 Mon Sep 17 00:00:00 2001 From: zhengziyi Date: Tue, 11 Jun 2024 22:59:24 +0800 Subject: [PATCH 02/14] temp commit --- .../command/AsyncProfilerTaskCommand.java | 17 +- .../core/asyncprofiler/AsyncProfilerTask.java | 543 ++---------------- .../AsyncProfilerTaskChannelService.java | 2 + .../AsyncProfilerTaskExecutionService.java | 40 +- .../AsyncProfilerCommandExecutor.java | 3 + 5 files changed, 100 insertions(+), 505 deletions(-) diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java index d89fca7c14..ba5f7699b3 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java @@ -208,7 +208,10 @@ public Command.Builder serialize() { builder.addArgs(KeyStringValuePair.newBuilder().setKey("Action").setValue(action)) .addArgs(KeyStringValuePair.newBuilder().setKey("ActionArg").setValue(actionArg)) .addArgs(KeyStringValuePair.newBuilder().setKey("Event").setValue(event)) - .addArgs(KeyStringValuePair.newBuilder().setKey("Alloc").setValue(alloc)); + .addArgs(KeyStringValuePair.newBuilder().setKey("Alloc").setValue(alloc)) + .addArgs(KeyStringValuePair.newBuilder().setKey("File").setValue(file)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Format").setValue(format)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration))); return builder; } @@ -221,6 +224,9 @@ public AsyncProfilerTaskCommand deserialize(Command command) { String actionArg = null; String event = null; String alloc = null; + String file = null; + String format = null; + Long duration = null; for (final KeyStringValuePair pair : args) { if ("SerialNumber".equals(pair.getKey())) { @@ -233,6 +239,12 @@ public AsyncProfilerTaskCommand deserialize(Command command) { event = pair.getValue(); } else if ("Alloc".equals(pair.getKey())) { alloc = pair.getValue(); + } else if ("File".equals(pair.getKey())) { + file = pair.getValue(); + } else if ("Format".equals(pair.getKey())) { + format = pair.getValue(); + } else if ("Duration".equals(pair.getKey())) { + duration = Long.parseLong(pair.getValue()); } } @@ -241,6 +253,9 @@ public AsyncProfilerTaskCommand deserialize(Command command) { asyncProfilerTaskCommand.setActionArg(actionArg); asyncProfilerTaskCommand.setEvent(event); asyncProfilerTaskCommand.setAlloc(alloc); + asyncProfilerTaskCommand.setFile(file); + asyncProfilerTaskCommand.setFormat(format); + asyncProfilerTaskCommand.setDuration(duration); return asyncProfilerTaskCommand; } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java index 00065e8a01..c371c68900 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java @@ -19,20 +19,23 @@ package org.apache.skywalking.apm.agent.core.asyncprofiler; import io.pyroscope.one.profiler.AsyncProfiler; +import io.pyroscope.one.profiler.Counter; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; import org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand; -import org.apache.skywalking.apm.util.StringUtil; +import java.io.DataInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; public class AsyncProfilerTask { private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTask.class); - private static final String OUTPUT_DIR = "skywalking-output"; + private static final String JFR = "jfr"; private AsyncProfilerTaskCommand.ProfilerAction action; private String actionArg; @@ -79,131 +82,18 @@ public class AsyncProfilerTask { */ private Long interval; - /** - * maximum Java stack depth (default: 2048) - */ - private Integer jstackdepth; - - /** - * profile different threads separately - */ - private boolean threads; - - /** - * group threads by scheduling policy - */ - private boolean sched; - - /** - * how to collect C stack frames in addition to Java stack - * MODE is 'fp' (Frame Pointer), 'dwarf', 'lbr' (Last Branch Record) or 'no' - */ - private String cstack; - - /** - * use simple class names instead of FQN - */ - private boolean simple; - - /** - * print method signatures - */ - private boolean sig; - - /** - * annotate Java methods - */ - private boolean ann; - - /** - * prepend library names - */ - private boolean lib; - - /** - * include only user-mode events - */ - private boolean alluser; - /** * run profiling for duration seconds */ private Long duration; - /** - * include stack traces containing PATTERN - */ - private List includes; - - /** - * exclude stack traces containing PATTERN - */ - private List excludes; - - /** - * automatically start profiling when the specified native function is executed. - */ - private String begin; - - /** - * automatically stop profiling when the specified native function is executed. - */ - private String end; - - /** - * time-to-safepoint profiling. - * An alias for --begin SafepointSynchronize::begin --end RuntimeService::record_safepoint_synchronized - */ - private boolean ttsp; - - /** - * FlameGraph title - */ - private String title; - - /** - * FlameGraph minimum frame width in percent - */ - private String minwidth; - - /** - * generate stack-reversed FlameGraph / Call tree - */ - private boolean reverse; - - /** - * count the total value (time, bytes, etc.) instead of samples - */ - private boolean total; - - /** - * approximate size of JFR chunk in bytes (default: 100 MB) - */ - private String chunksize; - - /** - * duration of JFR chunk in seconds (default: 1 hour) - */ - private String chunktime; - - /** - * run profiler in a loop (continuous profiling) - */ - private String loop; - - /** - * automatically stop profiler at TIME (absolute or relative) - */ - private String timeout; + private File tempJFRFile; - private String getExecuteArgs() { + private String getExecuteArgs() throws IOException { StringBuilder sb = new StringBuilder(); final char comma = ','; - // start - start profiling - // resume - start or resume profiling without resetting collected data - // stop - stop profiling - sb.append(action).append(comma); + sb.append("start"); if (this.event != null) { sb.append("event=").append(this.event).append(comma); @@ -217,90 +107,19 @@ private String getExecuteArgs() { if (this.lock != null) { sb.append("lock=").append(this.lock).append(comma); } - if (this.jfrsync != null) { - this.format = "jfr"; - sb.append("jfrsync=").append(this.jfrsync).append(comma); - } - if (this.file != null) { - sb.append("file=").append(this.file).append(comma); - } - if (this.format != null) { - sb.append(this.format).append(comma); - } if (this.interval != null) { sb.append("interval=").append(this.interval).append(comma); } - if (this.jstackdepth != null) { - sb.append("jstackdepth=").append(this.jstackdepth).append(comma); - } - if (this.threads) { - sb.append("threads").append(comma); - } - if (this.sched) { - sb.append("sched").append(comma); - } - if (this.cstack != null) { - sb.append("cstack=").append(this.cstack).append(comma); - } - if (this.simple) { - sb.append("simple").append(comma); - } - if (this.sig) { - sb.append("sig").append(comma); - } - if (this.ann) { - sb.append("ann").append(comma); - } - if (this.lib) { - sb.append("lib").append(comma); - } - if (this.alluser) { - sb.append("alluser").append(comma); - } - if (this.includes != null) { - for (String include : includes) { - sb.append("include=").append(include).append(comma); - } - } - if (this.excludes != null) { - for (String exclude : excludes) { - sb.append("exclude=").append(exclude).append(comma); + if (this.format != null) { + sb.append(this.format).append(comma); + if (JFR.equals(format)) { + // flight recorder is built on top of a file descriptor, so we need a file. + tempJFRFile = File.createTempFile("skywalking", ".jfr"); + tempJFRFile.deleteOnExit(); } } - if (this.ttsp) { - this.begin = "SafepointSynchronize::begin"; - this.end = "RuntimeService::record_safepoint_synchronized"; - } - if (this.begin != null) { - sb.append("begin=").append(this.begin).append(comma); - } - if (this.end != null) { - sb.append("end=").append(this.end).append(comma); - } - - if (this.title != null) { - sb.append("title=").append(this.title).append(comma); - } - if (this.minwidth != null) { - sb.append("minwidth=").append(this.minwidth).append(comma); - } - if (this.reverse) { - sb.append("reverse").append(comma); - } - if (this.total) { - sb.append("total").append(comma); - } - if (this.chunksize != null) { - sb.append("chunksize=").append(this.chunksize).append(comma); - } - if (this.chunktime != null) { - sb.append("chunktime=").append(this.chunktime).append(comma); - } - if (this.loop != null) { - sb.append("loop=").append(this.loop).append(comma); - } - if (this.timeout != null) { - sb.append("timeout=").append(this.timeout).append(comma); + if (this.file != null) { + sb.append("file=").append(this.file).append(comma); } return sb.toString(); } @@ -315,22 +134,35 @@ private static String execute(AsyncProfiler asyncProfiler, String arg) return result; } - public String process(AsyncProfiler asyncProfiler) throws IOException { - if (AsyncProfilerTaskCommand.ProfilerAction.start.equals(action)) { - String executeArgs = getExecuteArgs(); - return execute(asyncProfiler, executeArgs); - } else if (AsyncProfilerTaskCommand.ProfilerAction.stop.equals(action)) { - return processStop(asyncProfiler); - } + private String start(AsyncProfiler asyncProfiler) throws IOException { + String executeArgs = getExecuteArgs(); + return execute(asyncProfiler, executeArgs); + } - return ""; + /** + * stop async-profiler and dump profile data + */ + private byte[] processStop(AsyncProfiler asyncProfiler) throws IOException { + asyncProfiler.stop(); + final byte[] data; + if (format.equals(JFR)) { + data = dumpJFR(); + } else { + data = asyncProfiler.dumpCollapsed(Counter.SAMPLES).getBytes(StandardCharsets.UTF_8); + } + return data; } - private String processStop(AsyncProfiler asyncProfiler) throws IOException { - String outputFile = outputFile(); - String executeArgs = getExecuteArgs(); - LOGGER.info("dump async profiler out put file at:{}", outputFile); - return execute(asyncProfiler, executeArgs); + private byte[] dumpJFR() { + try { + byte[] bytes = new byte[(int) tempJFRFile.length()]; + try (DataInputStream ds = new DataInputStream(new FileInputStream(tempJFRFile))) { + ds.readFully(bytes); + } + return bytes; + } catch (IOException e) { + throw new IllegalStateException(e); + } } /** @@ -345,8 +177,8 @@ private String outputFileExt() { fileExt = "txt"; } else if (this.format.equals("flamegraph") || this.format.equals("tree")) { fileExt = "html"; - } else if (this.format.equals("jfr")) { - fileExt = "jfr"; + } else if (this.format.equals(JFR)) { + fileExt = JFR; } else { // illegal -o option makes async-profiler use flat fileExt = "txt"; @@ -354,291 +186,4 @@ private String outputFileExt() { return fileExt; } - public String getOutputPath() { - String userDir = System.getProperty("user.dir"); - if (StringUtil.isBlank(userDir)) { - LOGGER.warn("get user directory from system properties failed"); - } - return userDir + File.separator + OUTPUT_DIR; - } - - // TODO create dir when dir is not exist - private String outputFile() throws IOException { - if (this.file == null) { - String fileExt = outputFileExt(); - String outputPath = getOutputPath(); - if (outputPath != null) { - file = new File(outputPath, - new SimpleDateFormat("yyyyMMdd-HHmmss").format(new Date()) + "." + fileExt) - .getAbsolutePath(); - } else { - this.file = File.createTempFile(OUTPUT_DIR, "." + fileExt).getAbsolutePath(); - } - } - return file; - } - - public AsyncProfilerTaskCommand.ProfilerAction getAction() { - return action; - } - - public void setAction(String action) { - this.action = AsyncProfilerTaskCommand.ProfilerAction.valueOf(action); - } - - public String getActionArg() { - return actionArg; - } - - public void setActionArg(String actionArg) { - this.actionArg = actionArg; - } - - public String getEvent() { - return event; - } - - public void setEvent(String event) { - this.event = event; - } - - public String getAlloc() { - return alloc; - } - - public void setAlloc(String alloc) { - this.alloc = alloc; - } - - public boolean isLive() { - return live; - } - - public void setLive(boolean live) { - this.live = live; - } - - public String getLock() { - return lock; - } - - public void setLock(String lock) { - this.lock = lock; - } - - public String getJfrsync() { - return jfrsync; - } - - public void setJfrsync(String jfrsync) { - this.jfrsync = jfrsync; - } - - public String getFile() { - return file; - } - - public void setFile(String file) { - this.file = file; - } - - public String getFormat() { - return format; - } - - public void setFormat(String format) { - this.format = format; - } - - public Long getInterval() { - return interval; - } - - public void setInterval(Long interval) { - this.interval = interval; - } - - public Integer getJstackdepth() { - return jstackdepth; - } - - public void setJstackdepth(Integer jstackdepth) { - this.jstackdepth = jstackdepth; - } - - public boolean isThreads() { - return threads; - } - - public void setThreads(boolean threads) { - this.threads = threads; - } - - public boolean isSched() { - return sched; - } - - public void setSched(boolean sched) { - this.sched = sched; - } - - public String getCstack() { - return cstack; - } - - public void setCstack(String cstack) { - this.cstack = cstack; - } - - public boolean isSimple() { - return simple; - } - - public void setSimple(boolean simple) { - this.simple = simple; - } - - public boolean isSig() { - return sig; - } - - public void setSig(boolean sig) { - this.sig = sig; - } - - public boolean isAnn() { - return ann; - } - - public void setAnn(boolean ann) { - this.ann = ann; - } - - public boolean isLib() { - return lib; - } - - public void setLib(boolean lib) { - this.lib = lib; - } - - public boolean isAlluser() { - return alluser; - } - - public void setAlluser(boolean alluser) { - this.alluser = alluser; - } - - public Long getDuration() { - return duration; - } - - public void setDuration(Long duration) { - this.duration = duration; - } - - public List getIncludes() { - return includes; - } - - public void setIncludes(List includes) { - this.includes = includes; - } - - public List getExcludes() { - return excludes; - } - - public void setExcludes(List excludes) { - this.excludes = excludes; - } - - public String getBegin() { - return begin; - } - - public void setBegin(String begin) { - this.begin = begin; - } - - public String getEnd() { - return end; - } - - public void setEnd(String end) { - this.end = end; - } - - public boolean isTtsp() { - return ttsp; - } - - public void setTtsp(boolean ttsp) { - this.ttsp = ttsp; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getMinwidth() { - return minwidth; - } - - public void setMinwidth(String minwidth) { - this.minwidth = minwidth; - } - - public boolean isReverse() { - return reverse; - } - - public void setReverse(boolean reverse) { - this.reverse = reverse; - } - - public boolean isTotal() { - return total; - } - - public void setTotal(boolean total) { - this.total = total; - } - - public String getChunksize() { - return chunksize; - } - - public void setChunksize(String chunksize) { - this.chunksize = chunksize; - } - - public String getChunktime() { - return chunktime; - } - - public void setChunktime(String chunktime) { - this.chunktime = chunktime; - } - - public String getLoop() { - return loop; - } - - public void setLoop(String loop) { - this.loop = loop; - } - - public String getTimeout() { - return timeout; - } - - public void setTimeout(String timeout) { - this.timeout = timeout; - } } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java index 1912b49509..0b6bdacea6 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java @@ -58,6 +58,7 @@ public void run() { Command startCommand = Command.newBuilder() .addArgs(KeyStringValuePair.newBuilder().setKey("Action").setValue("start")) .addArgs(KeyStringValuePair.newBuilder().setKey("SerialNumber").setValue("1")) + .addArgs(KeyStringValuePair.newBuilder().setKey("Format").setValue("jfr")) .setCommand(AsyncProfilerTaskCommand.NAME) .build(); Commands startCommands = Commands.newBuilder() @@ -74,6 +75,7 @@ public void run() { Command stopCommand = Command.newBuilder() .addArgs(KeyStringValuePair.newBuilder().setKey("Action").setValue("stop")) .addArgs(KeyStringValuePair.newBuilder().setKey("SerialNumber").setValue("2")) + .addArgs(KeyStringValuePair.newBuilder().setKey("Format").setValue("jfr")) .setCommand(AsyncProfilerTaskCommand.NAME) .build(); Commands stopCommands = Commands.newBuilder() diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java index 79c988ec9b..80ae6ccac2 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java @@ -18,42 +18,72 @@ package org.apache.skywalking.apm.agent.core.asyncprofiler; +import io.pyroscope.labels.io.pyroscope.PyroscopeAsyncProfiler; import io.pyroscope.one.profiler.AsyncProfiler; import org.apache.skywalking.apm.agent.core.boot.BootService; import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.util.StringUtil; import java.io.IOException; -import java.util.concurrent.ExecutorService; +import java.util.Objects; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; @DefaultImplementor public class AsyncProfilerTaskExecutionService implements BootService { private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTaskChannelService.class); - //TODO use local so file for debug - private static final AsyncProfiler ASYNC_PROFILER = AsyncProfiler.getInstance("/opt/async-profiler/lib/libasyncProfiler.so"); + private static final AsyncProfiler ASYNC_PROFILER = PyroscopeAsyncProfiler.getAsyncProfiler(); // profile executor thread pool, only running one thread - private final static ExecutorService ASYNC_PROFILE_EXECUTOR = Executors.newSingleThreadExecutor( + private final static ScheduledExecutorService ASYNC_PROFILE_EXECUTOR = Executors.newSingleThreadScheduledExecutor( new DefaultNamedThreadFactory("ASYNC-PROFILING-TASK")); + private volatile ProfileState status = ProfileState.STOP; + + private enum ProfileState { + PROFILING, + STOP; + } + public void processAsyncProfilerTask(AsyncProfilerTask task) { // add task to list LOGGER.info("add async profiler task: {}", task); // asyncProfileTaskList.add(task); ASYNC_PROFILE_EXECUTOR.execute(() -> { try { - task.process(ASYNC_PROFILER); + if (status == ProfileState.PROFILING) { + // stop pre task + status = ProfileState.STOP; + // todo stop pre task + } + if (Objects.isNull(task.getDuration())) { + LOGGER.error("async profile task must need duration"); + } + String result = task.process(ASYNC_PROFILER); + + LOGGER.info("AsyncProfilerTask executor result:{}", result); +// PROFILE_TASK_SCHEDULE.schedule(() -> processProfileTask(task), timeToProcessMills, TimeUnit.MILLISECONDS); + ASYNC_PROFILE_EXECUTOR.schedule(() -> stopAsyncProfile(task), task.getDuration(), TimeUnit.MICROSECONDS); } catch (IOException e) { LOGGER.error("AsyncProfilerTask executor error:" + e.getMessage(), e); } }); } + private void stopAsyncProfile(AsyncProfilerTask task) { + // execute stop task + + status = ProfileState.STOP; + // upload file + + } + @Override public void prepare() throws Throwable { diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java index 8b92ca8d9f..832959f698 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java @@ -36,6 +36,9 @@ public void execute(BaseCommand command) throws CommandExecutionException { asyncProfilerTask.setActionArg(asyncProfilerTaskCommand.getActionArg()); asyncProfilerTask.setEvent(asyncProfilerTaskCommand.getEvent()); asyncProfilerTask.setAlloc(asyncProfilerTask.getAlloc()); + asyncProfilerTask.setFormat(asyncProfilerTaskCommand.getFormat()); + asyncProfilerTask.setFile(asyncProfilerTask.getFile()); + asyncProfilerTask.setDuration(asyncProfilerTask.getDuration()); ServiceManager.INSTANCE.findService(AsyncProfilerTaskExecutionService.class) .processAsyncProfilerTask(asyncProfilerTask); } From 0754556f38af41a9874ba06b4d9780c409f8d0fa Mon Sep 17 00:00:00 2001 From: "zhengziyi.0117" Date: Fri, 5 Jul 2024 10:51:00 +0800 Subject: [PATCH 03/14] feat: add async-profiler task receive and send data --- .../command/AsyncProfilerTaskCommand.java | 531 ++---------------- .../AsyncProfilerDataSender.java | 118 ++++ .../core/asyncprofiler/AsyncProfilerTask.java | 196 +++---- .../AsyncProfilerTaskChannelService.java | 67 +-- .../AsyncProfilerTaskExecutionService.java | 60 +- .../AsyncProfilerCommandExecutor.java | 11 +- ...skywalking.apm.agent.core.boot.BootService | 3 +- 7 files changed, 322 insertions(+), 664 deletions(-) create mode 100644 apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java index ba5f7699b3..902574441e 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java @@ -15,7 +15,6 @@ * limitations under the License. * */ - package org.apache.skywalking.apm.network.trace.component.command; import org.apache.skywalking.apm.network.common.v3.Command; @@ -23,503 +22,87 @@ import java.util.List; -/** - * TODO add full arg - */ public class AsyncProfilerTaskCommand extends BaseCommand implements Serializable, Deserializable { - public static final Deserializable DESERIALIZER = new AsyncProfilerTaskCommand("", ""); - public static final String NAME = "AsyncProfilerQuery"; - - private String action; - private String actionArg; - - /** - * which event to trace (cpu, wall, cache-misses, etc.) - */ - private String event; - - /** - * profile allocations with BYTES interval - * according to async-profiler README, alloc may contains non-numeric characters - */ - private String alloc; - - /** - * build allocation profile from live objects only - */ - private boolean live; - - /** - * profile contended locks longer than DURATION ns - * according to async-profiler README, alloc may contains non-numeric characters - */ - private String lock; - - /** - * start Java Flight Recording with the given config along with the profiler - */ - private String jfrsync; - - /** - * output file name for dumping - */ - private String file; - - /** - * output file format, default value is html. - */ - private String format; - - /** - * sampling interval in ns (default: 10'000'000, i.e. 10 ms) - */ - private Long interval; - - /** - * maximum Java stack depth (default: 2048) - */ - private Integer jstackdepth; - - /** - * profile different threads separately - */ - private boolean threads; - - /** - * group threads by scheduling policy - */ - private boolean sched; - - /** - * how to collect C stack frames in addition to Java stack - * MODE is 'fp' (Frame Pointer), 'dwarf', 'lbr' (Last Branch Record) or 'no' - */ - private String cstack; - - /** - * use simple class names instead of FQN - */ - private boolean simple; - - /** - * print method signatures - */ - private boolean sig; - - /** - * annotate Java methods - */ - private boolean ann; - - /** - * prepend library names - */ - private boolean lib; - - /** - * include only user-mode events - */ - private boolean alluser; - - /** - * run profiling for duration seconds - */ - private Long duration; - - /** - * include stack traces containing PATTERN - */ - private List includes; - - /** - * exclude stack traces containing PATTERN - */ - private List excludes; - - /** - * automatically start profiling when the specified native function is executed. - */ - private String begin; - - /** - * automatically stop profiling when the specified native function is executed. - */ - private String end; - - /** - * time-to-safepoint profiling. - * An alias for --begin SafepointSynchronize::begin --end RuntimeService::record_safepoint_synchronized - */ - private boolean ttsp; - - /** - * FlameGraph title - */ - private String title; - - /** - * FlameGraph minimum frame width in percent - */ - private String minwidth; - - /** - * generate stack-reversed FlameGraph / Call tree - */ - private boolean reverse; - - /** - * count the total value (time, bytes, etc.) instead of samples - */ - private boolean total; - - /** - * approximate size of JFR chunk in bytes (default: 100 MB) - */ - private String chunksize; - - /** - * duration of JFR chunk in seconds (default: 1 hour) - */ - private String chunktime; - - /** - * run profiler in a loop (continuous profiling) - */ - private String loop; - - /** - * automatically stop profiler at TIME (absolute or relative) - */ - private String timeout; - - public enum ProfilerAction { - // start, resume, stop, dump, check, status, meminfo, list, collect, - start, resume, stop, dump, check, status, meminfo, list, collect, - version, - } - - public AsyncProfilerTaskCommand(String command, String serialNumber) { - super(command, serialNumber); + public static final Deserializable DESERIALIZER = new AsyncProfilerTaskCommand("", "", 0, "", 0); + public static final String NAME = "AsyncProfileTaskQuery"; + + private final String taskId; + private final int duration; + private final String execArgs; + private final long createTime; + + public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration, String dataFormat, + String events, String execArgs, long createTime) { + super(NAME, serialNumber); + this.taskId = taskId; + this.duration = duration; + this.createTime = createTime; + String comma = ","; + StringBuilder sb = new StringBuilder(); + sb.append("event=").append(String.join(comma, events)).append(comma); + sb.append(dataFormat).append(comma); + if(execArgs != null && !execArgs.isEmpty()) { + sb.append(execArgs); + } + this.execArgs = sb.toString(); } - @Override - public Command.Builder serialize() { - final Command.Builder builder = commandBuilder(); - builder.addArgs(KeyStringValuePair.newBuilder().setKey("Action").setValue(action)) - .addArgs(KeyStringValuePair.newBuilder().setKey("ActionArg").setValue(actionArg)) - .addArgs(KeyStringValuePair.newBuilder().setKey("Event").setValue(event)) - .addArgs(KeyStringValuePair.newBuilder().setKey("Alloc").setValue(alloc)) - .addArgs(KeyStringValuePair.newBuilder().setKey("File").setValue(file)) - .addArgs(KeyStringValuePair.newBuilder().setKey("Format").setValue(format)) - .addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration))); - return builder; + public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration, String execArgs, long createTime) { + super(NAME, serialNumber); + this.taskId = taskId; + this.duration = duration; + this.execArgs = execArgs; + this.createTime = createTime; } @Override public AsyncProfilerTaskCommand deserialize(Command command) { - List args = command.getArgsList(); - + final List argsList = command.getArgsList(); + String taskId = null; + int duration = 0; + String execArgs = null; + long createTime = 0; String serialNumber = null; - String action = null; - String actionArg = null; - String event = null; - String alloc = null; - String file = null; - String format = null; - Long duration = null; - - for (final KeyStringValuePair pair : args) { + for (final KeyStringValuePair pair : argsList) { if ("SerialNumber".equals(pair.getKey())) { serialNumber = pair.getValue(); - } else if ("Action".equals(pair.getKey())) { - action = pair.getValue(); - } else if ("ActionArg".equals(pair.getKey())) { - actionArg = pair.getValue(); - } else if ("Event".equals(pair.getKey())) { - event = pair.getValue(); - } else if ("Alloc".equals(pair.getKey())) { - alloc = pair.getValue(); - } else if ("File".equals(pair.getKey())) { - file = pair.getValue(); - } else if ("Format".equals(pair.getKey())) { - format = pair.getValue(); + } else if ("TaskId".equals(pair.getKey())) { + taskId = pair.getValue(); } else if ("Duration".equals(pair.getKey())) { - duration = Long.parseLong(pair.getValue()); + duration = Integer.parseInt(pair.getValue()); + } else if ("ExecArgs".equals(pair.getKey())) { + execArgs = pair.getValue(); + } else if ("CreateTime".equals(pair.getKey())) { + createTime = Long.parseLong(pair.getValue()); } } - - AsyncProfilerTaskCommand asyncProfilerTaskCommand = new AsyncProfilerTaskCommand(NAME, serialNumber); - asyncProfilerTaskCommand.setAction(action); - asyncProfilerTaskCommand.setActionArg(actionArg); - asyncProfilerTaskCommand.setEvent(event); - asyncProfilerTaskCommand.setAlloc(alloc); - asyncProfilerTaskCommand.setFile(file); - asyncProfilerTaskCommand.setFormat(format); - asyncProfilerTaskCommand.setDuration(duration); - return asyncProfilerTaskCommand; - } - - public String getAction() { - return action; - } - - public void setAction(String action) { - this.action = action; - } - - public String getActionArg() { - return actionArg; - } - - public void setActionArg(String actionArg) { - this.actionArg = actionArg; - } - - public String getEvent() { - return event; - } - - public void setEvent(String event) { - this.event = event; - } - - public String getAlloc() { - return alloc; - } - - public void setAlloc(String alloc) { - this.alloc = alloc; - } - - public boolean isLive() { - return live; - } - - public void setLive(boolean live) { - this.live = live; - } - - public String getLock() { - return lock; - } - - public void setLock(String lock) { - this.lock = lock; - } - - public String getJfrsync() { - return jfrsync; - } - - public void setJfrsync(String jfrsync) { - this.jfrsync = jfrsync; - } - - public String getFile() { - return file; - } - - public void setFile(String file) { - this.file = file; - } - - public String getFormat() { - return format; - } - - public void setFormat(String format) { - this.format = format; - } - - public Long getInterval() { - return interval; - } - - public void setInterval(Long interval) { - this.interval = interval; + return new AsyncProfilerTaskCommand(serialNumber, taskId, duration, execArgs, createTime); } - public Integer getJstackdepth() { - return jstackdepth; - } - - public void setJstackdepth(Integer jstackdepth) { - this.jstackdepth = jstackdepth; - } - - public boolean isThreads() { - return threads; - } - - public void setThreads(boolean threads) { - this.threads = threads; - } - - public boolean isSched() { - return sched; - } - - public void setSched(boolean sched) { - this.sched = sched; - } - - public String getCstack() { - return cstack; - } - - public void setCstack(String cstack) { - this.cstack = cstack; - } - - public boolean isSimple() { - return simple; - } - - public void setSimple(boolean simple) { - this.simple = simple; - } - - public boolean isSig() { - return sig; - } - - public void setSig(boolean sig) { - this.sig = sig; - } - - public boolean isAnn() { - return ann; - } - - public void setAnn(boolean ann) { - this.ann = ann; - } - - public boolean isLib() { - return lib; - } - - public void setLib(boolean lib) { - this.lib = lib; - } - - public boolean isAlluser() { - return alluser; + @Override + public Command.Builder serialize() { + final Command.Builder builder = commandBuilder(); + builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId)) + .addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration))) + .addArgs(KeyStringValuePair.newBuilder().setKey("ExecArgs").setValue(execArgs)) + .addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime))); + return builder; } - public void setAlluser(boolean alluser) { - this.alluser = alluser; + public String getTaskId() { + return taskId; } - public Long getDuration() { + public int getDuration() { return duration; } - public void setDuration(Long duration) { - this.duration = duration; - } - - public List getIncludes() { - return includes; - } - - public void setIncludes(List includes) { - this.includes = includes; - } - - public List getExcludes() { - return excludes; + public String getExecArgs() { + return execArgs; } - public void setExcludes(List excludes) { - this.excludes = excludes; + public long getCreateTime() { + return createTime; } - public String getBegin() { - return begin; - } - - public void setBegin(String begin) { - this.begin = begin; - } - - public String getEnd() { - return end; - } - - public void setEnd(String end) { - this.end = end; - } - - public boolean isTtsp() { - return ttsp; - } - - public void setTtsp(boolean ttsp) { - this.ttsp = ttsp; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getMinwidth() { - return minwidth; - } - - public void setMinwidth(String minwidth) { - this.minwidth = minwidth; - } - - public boolean isReverse() { - return reverse; - } - - public void setReverse(boolean reverse) { - this.reverse = reverse; - } - - public boolean isTotal() { - return total; - } - - public void setTotal(boolean total) { - this.total = total; - } - - public String getChunksize() { - return chunksize; - } - - public void setChunksize(String chunksize) { - this.chunksize = chunksize; - } - - public String getChunktime() { - return chunktime; - } - - public void setChunktime(String chunktime) { - this.chunktime = chunktime; - } - - public String getLoop() { - return loop; - } - - public void setLoop(String loop) { - this.loop = loop; - } - - public String getTimeout() { - return timeout; - } - - public void setTimeout(String timeout) { - this.timeout = timeout; - } } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java new file mode 100644 index 0000000000..99eef1a2a5 --- /dev/null +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java @@ -0,0 +1,118 @@ +package org.apache.skywalking.apm.agent.core.asyncprofiler; + +import com.google.protobuf.ByteString; +import io.grpc.Channel; +import io.grpc.stub.StreamObserver; +import org.apache.skywalking.apm.agent.core.boot.BootService; +import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.profile.ProfileSnapshotSender; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus; +import org.apache.skywalking.apm.agent.core.remote.GRPCStreamServiceStatus; +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerData; +import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerMetaData; +import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerTaskGrpc; + +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT; + +@DefaultImplementor +public class AsyncProfilerDataSender implements BootService, GRPCChannelListener { + private static final ILog LOGGER = LogManager.getLogger(ProfileSnapshotSender.class); + private static final int DATA_CHUNK_SIZE = 1024 * 1024; + + private volatile GRPCChannelStatus status = GRPCChannelStatus.DISCONNECT; + + private volatile AsyncProfilerTaskGrpc.AsyncProfilerTaskStub asyncProfilerTaskStub; + + @Override + public void prepare() throws Throwable { + ServiceManager.INSTANCE.findService(GRPCChannelManager.class).addChannelListener(this); + } + + @Override + public void boot() throws Throwable { + + } + + @Override + public void onComplete() throws Throwable { + + } + + @Override + public void shutdown() throws Throwable { + + } + + @Override + public void statusChanged(GRPCChannelStatus status) { + if (GRPCChannelStatus.CONNECTED.equals(status)) { + Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel(); + asyncProfilerTaskStub = AsyncProfilerTaskGrpc.newStub(channel); + } else { + asyncProfilerTaskStub = null; + } + this.status = status; + } + + public void send(AsyncProfilerTask task, byte[] data) { + if (status != GRPCChannelStatus.CONNECTED || Objects.isNull(data) || data.length == 0) { + return; + } + final GRPCStreamServiceStatus status = new GRPCStreamServiceStatus(false); + StreamObserver dataStreamObserver = asyncProfilerTaskStub.withDeadlineAfter( + GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS + ).collect(new StreamObserver() { + @Override + public void onNext(Commands value) { + } + + @Override + public void onError(Throwable t) { + status.finished(); + if (LOGGER.isErrorEnable()) { + LOGGER.error( + t, "Send async profiler task data to collector fail with a grpc internal exception." + ); + } + ServiceManager.INSTANCE.findService(GRPCChannelManager.class).reportError(t); + } + + @Override + public void onCompleted() { + status.finished(); + } + }); + AsyncProfilerMetaData metaData = AsyncProfilerMetaData.newBuilder() + .setService(Config.Agent.SERVICE_NAME) + .setServiceInstance(Config.Agent.INSTANCE_NAME) + .setTaskId(task.getTaskId()) + .setExecutionArgs(task.getExecArgs()) + .build(); + AsyncProfilerData asyncProfilerData = AsyncProfilerData.newBuilder().setMetaData(metaData).build(); + dataStreamObserver.onNext(asyncProfilerData); + // send bin data + int idx = 0; + int len = data.length; + do { + int size = Math.min(DATA_CHUNK_SIZE, len - idx); + asyncProfilerData = AsyncProfilerData.newBuilder() + .setContent(ByteString.copyFrom(data, idx, size)) + .build(); + dataStreamObserver.onNext(asyncProfilerData); + idx += DATA_CHUNK_SIZE; + } while (idx < len); + + dataStreamObserver.onCompleted(); + status.wait4Finish(); + } +} diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java index c371c68900..6ad6e9a66e 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java @@ -19,110 +19,38 @@ package org.apache.skywalking.apm.agent.core.asyncprofiler; import io.pyroscope.one.profiler.AsyncProfiler; -import io.pyroscope.one.profiler.Counter; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; -import org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand; import java.io.DataInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; public class AsyncProfilerTask { private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTask.class); - private static final String JFR = "jfr"; - - private AsyncProfilerTaskCommand.ProfilerAction action; - private String actionArg; - - /** - * which event to trace (cpu, wall, cache-misses, etc.) - */ - private String event; - - /** - * profile allocations with BYTES interval - * according to async-profiler README, alloc may contains non-numeric characters - */ - private String alloc; - /** - * build allocation profile from live objects only + * task id */ - private boolean live; - + private String taskId; /** - * profile contended locks longer than DURATION ns - * according to async-profiler README, alloc may contains non-numeric characters + * execArgument from oap server */ - private String lock; - + private String execArgs; /** - * start Java Flight Recording with the given config along with the profiler - */ - private String jfrsync; - - /** - * output file name for dumping - */ - private String file; - - /** - * output file format, default value is html. + * run profiling for duration seconds */ - private String format; - + private int duration; /** - * sampling interval in ns (default: 10'000'000, i.e. 10 ms) + * run profiling for duration seconds */ - private Long interval; - + private long createTime; /** - * run profiling for duration seconds + * temp File */ - private Long duration; - - private File tempJFRFile; - - private String getExecuteArgs() throws IOException { - StringBuilder sb = new StringBuilder(); - final char comma = ','; - - sb.append("start"); - - if (this.event != null) { - sb.append("event=").append(this.event).append(comma); - } - if (this.alloc != null) { - sb.append("alloc=").append(this.alloc).append(comma); - } - if (this.live) { - sb.append("live").append(comma); - } - if (this.lock != null) { - sb.append("lock=").append(this.lock).append(comma); - } - if (this.interval != null) { - sb.append("interval=").append(this.interval).append(comma); - } - if (this.format != null) { - sb.append(this.format).append(comma); - if (JFR.equals(format)) { - // flight recorder is built on top of a file descriptor, so we need a file. - tempJFRFile = File.createTempFile("skywalking", ".jfr"); - tempJFRFile.deleteOnExit(); - } - } - if (this.file != null) { - sb.append("file=").append(this.file).append(comma); - } - return sb.toString(); - } + private Path tempFile; private static String execute(AsyncProfiler asyncProfiler, String arg) throws IllegalArgumentException, IOException { @@ -134,56 +62,76 @@ private static String execute(AsyncProfiler asyncProfiler, String arg) return result; } - private String start(AsyncProfiler asyncProfiler) throws IOException { - String executeArgs = getExecuteArgs(); - return execute(asyncProfiler, executeArgs); + public String start(AsyncProfiler asyncProfiler) throws IOException { + tempFile = Files.createFile(Paths.get("/Users/bytedance/IdeaProjects/skywalking-java/skywalking-output/" + taskId)); + execArgs = execArgs + "file=" + tempFile.toAbsolutePath(); + return execute(asyncProfiler, execArgs); } /** * stop async-profiler and dump profile data */ - private byte[] processStop(AsyncProfiler asyncProfiler) throws IOException { + public byte[] stop(AsyncProfiler asyncProfiler) throws IOException { + LOGGER.info("async profiler process stop and dump file"); asyncProfiler.stop(); - final byte[] data; - if (format.equals(JFR)) { - data = dumpJFR(); - } else { - data = asyncProfiler.dumpCollapsed(Counter.SAMPLES).getBytes(StandardCharsets.UTF_8); - } - return data; + // todo now only JFR + return dumpJFR(); +// final byte[] data; +// if (format.equals(JFR)) { +// data = dumpJFR(); +// } else { +// data = asyncProfiler.dumpCollapsed(Counter.SAMPLES).getBytes(StandardCharsets.UTF_8); +// } +// return data; } - private byte[] dumpJFR() { - try { - byte[] bytes = new byte[(int) tempJFRFile.length()]; - try (DataInputStream ds = new DataInputStream(new FileInputStream(tempJFRFile))) { - ds.readFully(bytes); - } - return bytes; - } catch (IOException e) { - throw new IllegalStateException(e); + private byte[] dumpJFR() throws IOException { + File file = tempFile.toFile(); + file.deleteOnExit(); + byte[] bytes = new byte[(int) file.length()]; + try (DataInputStream ds = new DataInputStream(Files.newInputStream(file.toPath()))) { + ds.readFully(bytes); } + return bytes; } - /** - * This method should only be called when {@code this.file == null} is true. - */ - private String outputFileExt() { - String fileExt = ""; - if (this.format == null) { - fileExt = "html"; - } else if (this.format.startsWith("flat") || this.format.startsWith("traces") - || this.format.equals("collapsed")) { - fileExt = "txt"; - } else if (this.format.equals("flamegraph") || this.format.equals("tree")) { - fileExt = "html"; - } else if (this.format.equals(JFR)) { - fileExt = JFR; - } else { - // illegal -o option makes async-profiler use flat - fileExt = "txt"; - } - return fileExt; + public void setExecArgs(String execArgs) { + this.execArgs = execArgs; + } + + public void setDuration(int duration) { + this.duration = duration; + } + + public void setTempFile(Path tempFile) { + this.tempFile = tempFile; } + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + public void setCreateTime(long createTime) { + this.createTime = createTime; + } + + public String getExecArgs() { + return execArgs; + } + + public int getDuration() { + return duration; + } + + public Path getTempFile() { + return tempFile; + } + + public String getTaskId() { + return taskId; + } + + public long getCreateTime() { + return createTime; + } } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java index 0b6bdacea6..fa045eeafa 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java @@ -18,6 +18,7 @@ package org.apache.skywalking.apm.agent.core.asyncprofiler; +import io.grpc.Channel; import org.apache.skywalking.apm.agent.core.boot.BootService; import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory; @@ -26,28 +27,30 @@ import org.apache.skywalking.apm.agent.core.conf.Config; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelListener; +import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager; import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus; -import org.apache.skywalking.apm.network.common.v3.Command; import org.apache.skywalking.apm.network.common.v3.Commands; -import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; -import org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand; +import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfileTaskCommandQuery; +import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerTaskGrpc; import org.apache.skywalking.apm.util.RunnableWithExceptionProtection; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -/** - * TODO

add receive grpc command

- * now just debug AsyncProfilerTask - */ +import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT; + + @DefaultImplementor -public class AsyncProfilerTaskChannelService implements BootService, Runnable { +public class AsyncProfilerTaskChannelService implements BootService, Runnable, GRPCChannelListener { private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTaskChannelService.class); // channel status private volatile GRPCChannelStatus status = GRPCChannelStatus.DISCONNECT; + private volatile AsyncProfilerTaskGrpc.AsyncProfilerTaskBlockingStub asyncProfilerTaskBlockingStub; + // query task list schedule private volatile ScheduledFuture getTaskListFuture; @@ -55,41 +58,33 @@ public class AsyncProfilerTaskChannelService implements BootService, Runnable { public void run() { if (status == GRPCChannelStatus.CONNECTED) { // test start command and 10s after put stop command - Command startCommand = Command.newBuilder() - .addArgs(KeyStringValuePair.newBuilder().setKey("Action").setValue("start")) - .addArgs(KeyStringValuePair.newBuilder().setKey("SerialNumber").setValue("1")) - .addArgs(KeyStringValuePair.newBuilder().setKey("Format").setValue("jfr")) - .setCommand(AsyncProfilerTaskCommand.NAME) - .build(); - Commands startCommands = Commands.newBuilder() - .addCommands(startCommand) + long lastCommandCreateTime = ServiceManager.INSTANCE + .findService(AsyncProfilerTaskExecutionService.class).getLastCommandCreateTime(); + AsyncProfileTaskCommandQuery query = AsyncProfileTaskCommandQuery.newBuilder() + .setServiceInstance(Config.Agent.INSTANCE_NAME) + .setService(Config.Agent.SERVICE_NAME) + .setLastCommandTime(lastCommandCreateTime) .build(); - ServiceManager.INSTANCE.findService(CommandService.class).receiveCommand(startCommands); - - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - Command stopCommand = Command.newBuilder() - .addArgs(KeyStringValuePair.newBuilder().setKey("Action").setValue("stop")) - .addArgs(KeyStringValuePair.newBuilder().setKey("SerialNumber").setValue("2")) - .addArgs(KeyStringValuePair.newBuilder().setKey("Format").setValue("jfr")) - .setCommand(AsyncProfilerTaskCommand.NAME) - .build(); - Commands stopCommands = Commands.newBuilder() - .addCommands(stopCommand) - .build(); - ServiceManager.INSTANCE.findService(CommandService.class).receiveCommand(stopCommands); + Commands commands = asyncProfilerTaskBlockingStub.withDeadlineAfter(GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS) + .getAsyncProfileTaskCommands(query); + ServiceManager.INSTANCE.findService(CommandService.class).receiveCommand(commands); + } + } - status = GRPCChannelStatus.DISCONNECT; + @Override + public void statusChanged(GRPCChannelStatus status) { + if (GRPCChannelStatus.CONNECTED.equals(status)) { + Channel channel = ServiceManager.INSTANCE.findService(GRPCChannelManager.class).getChannel(); + asyncProfilerTaskBlockingStub = AsyncProfilerTaskGrpc.newBlockingStub(channel); + } else { + asyncProfilerTaskBlockingStub = null; } + this.status = status; } @Override public void prepare() throws Throwable { - status = GRPCChannelStatus.CONNECTED; + ServiceManager.INSTANCE.findService(GRPCChannelManager.class).addChannelListener(this); } @Override diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java index 80ae6ccac2..2ad325075b 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java @@ -23,14 +23,15 @@ import org.apache.skywalking.apm.agent.core.boot.BootService; import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; import org.apache.skywalking.apm.agent.core.boot.DefaultNamedThreadFactory; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; -import org.apache.skywalking.apm.util.StringUtil; import java.io.IOException; import java.util.Objects; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @DefaultImplementor @@ -40,36 +41,42 @@ public class AsyncProfilerTaskExecutionService implements BootService { private static final AsyncProfiler ASYNC_PROFILER = PyroscopeAsyncProfiler.getAsyncProfiler(); + private static final String SUCCESS_RESULT = "Profiling started\n"; + // profile executor thread pool, only running one thread private final static ScheduledExecutorService ASYNC_PROFILE_EXECUTOR = Executors.newSingleThreadScheduledExecutor( new DefaultNamedThreadFactory("ASYNC-PROFILING-TASK")); - private volatile ProfileState status = ProfileState.STOP; + // last command create time, use to next query task list + private volatile long lastCommandCreateTime = -1; - private enum ProfileState { - PROFILING, - STOP; - } + // task schedule future + private volatile ScheduledFuture scheduledFuture; + private volatile AsyncProfilerTask preAsyncProfilerTask; public void processAsyncProfilerTask(AsyncProfilerTask task) { + if (task.getCreateTime() > lastCommandCreateTime) { + lastCommandCreateTime = task.getCreateTime(); + } + LOGGER.info("add async profiler task: {}", task.getTaskId()); // add task to list - LOGGER.info("add async profiler task: {}", task); -// asyncProfileTaskList.add(task); ASYNC_PROFILE_EXECUTOR.execute(() -> { try { - if (status == ProfileState.PROFILING) { + if (Objects.nonNull(scheduledFuture) && !scheduledFuture.isDone()) { + scheduledFuture.cancel(true); // stop pre task - status = ProfileState.STOP; - // todo stop pre task + stopAsyncProfile(preAsyncProfilerTask); + preAsyncProfilerTask = null; } - if (Objects.isNull(task.getDuration())) { - LOGGER.error("async profile task must need duration"); + String result = task.start(ASYNC_PROFILER); + if (!SUCCESS_RESULT.equals(result)) { + LOGGER.error("AsyncProfilerTask start fail result:" + result); + return; } - String result = task.process(ASYNC_PROFILER); - - LOGGER.info("AsyncProfilerTask executor result:{}", result); -// PROFILE_TASK_SCHEDULE.schedule(() -> processProfileTask(task), timeToProcessMills, TimeUnit.MILLISECONDS); - ASYNC_PROFILE_EXECUTOR.schedule(() -> stopAsyncProfile(task), task.getDuration(), TimeUnit.MICROSECONDS); + preAsyncProfilerTask = task; + scheduledFuture = ASYNC_PROFILE_EXECUTOR.schedule( + () -> stopAsyncProfile(task), task.getDuration(), TimeUnit.SECONDS + ); } catch (IOException e) { LOGGER.error("AsyncProfilerTask executor error:" + e.getMessage(), e); } @@ -77,11 +84,20 @@ public void processAsyncProfilerTask(AsyncProfilerTask task) { } private void stopAsyncProfile(AsyncProfilerTask task) { - // execute stop task - - status = ProfileState.STOP; - // upload file + try { + // execute stop task + byte[] data = task.stop(ASYNC_PROFILER); + // upload file + AsyncProfilerDataSender dataSender = ServiceManager.INSTANCE.findService(AsyncProfilerDataSender.class); + dataSender.send(task, data); + } catch (Exception e) { + LOGGER.error("stop async profiler task error", e); + return; + } + } + public long getLastCommandCreateTime() { + return lastCommandCreateTime; } @Override diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java index 832959f698..adb6336c98 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java @@ -32,13 +32,10 @@ public void execute(BaseCommand command) throws CommandExecutionException { AsyncProfilerTaskCommand asyncProfilerTaskCommand = (AsyncProfilerTaskCommand) command; AsyncProfilerTask asyncProfilerTask = new AsyncProfilerTask(); - asyncProfilerTask.setAction(asyncProfilerTaskCommand.getAction()); - asyncProfilerTask.setActionArg(asyncProfilerTaskCommand.getActionArg()); - asyncProfilerTask.setEvent(asyncProfilerTaskCommand.getEvent()); - asyncProfilerTask.setAlloc(asyncProfilerTask.getAlloc()); - asyncProfilerTask.setFormat(asyncProfilerTaskCommand.getFormat()); - asyncProfilerTask.setFile(asyncProfilerTask.getFile()); - asyncProfilerTask.setDuration(asyncProfilerTask.getDuration()); + asyncProfilerTask.setTaskId(asyncProfilerTaskCommand.getTaskId()); + asyncProfilerTask.setDuration(asyncProfilerTaskCommand.getDuration()); + asyncProfilerTask.setExecArgs(asyncProfilerTaskCommand.getExecArgs()); + asyncProfilerTask.setCreateTime(asyncProfilerTaskCommand.getCreateTime()); ServiceManager.INSTANCE.findService(AsyncProfilerTaskExecutionService.class) .processAsyncProfilerTask(asyncProfilerTask); } diff --git a/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService b/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService index 7b5e4864ef..f75d28cd78 100644 --- a/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService +++ b/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService @@ -37,4 +37,5 @@ org.apache.skywalking.apm.agent.core.conf.dynamic.ConfigurationDiscoveryService org.apache.skywalking.apm.agent.core.remote.EventReportServiceClient org.apache.skywalking.apm.agent.core.ServiceInstanceGenerator org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTaskExecutionService -org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTaskChannelService \ No newline at end of file +org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerTaskChannelService +org.apache.skywalking.apm.agent.core.asyncprofiler.AsyncProfilerDataSender \ No newline at end of file From 9adeb8254aa415676702dbd679c98fd7605109d4 Mon Sep 17 00:00:00 2001 From: "zhengziyi.0117" Date: Sun, 14 Jul 2024 19:59:04 +0800 Subject: [PATCH 04/14] feat: add async profiler dataformat enum --- .../command/AsyncProfilerTaskCommand.java | 29 +++++--- .../AsyncProfilerDataSender.java | 20 +++--- .../core/asyncprofiler/AsyncProfilerTask.java | 66 ++++++++++++------- .../AsyncProfilerTaskExecutionService.java | 5 +- .../AsyncProfilerCommandExecutor.java | 1 + 5 files changed, 76 insertions(+), 45 deletions(-) diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java index 902574441e..9c3dc992f4 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java @@ -19,39 +19,44 @@ import org.apache.skywalking.apm.network.common.v3.Command; import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerDataFormatType; import java.util.List; public class AsyncProfilerTaskCommand extends BaseCommand implements Serializable, Deserializable { - public static final Deserializable DESERIALIZER = new AsyncProfilerTaskCommand("", "", 0, "", 0); + public static final Deserializable DESERIALIZER = new AsyncProfilerTaskCommand("", "", 0, null, "", 0); public static final String NAME = "AsyncProfileTaskQuery"; private final String taskId; private final int duration; private final String execArgs; private final long createTime; + private final AsyncProfilerDataFormatType dataFormat; - public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration, String dataFormat, - String events, String execArgs, long createTime) { + public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration, + AsyncProfilerDataFormatType dataFormat, String events, + String execArgs, long createTime) { super(NAME, serialNumber); this.taskId = taskId; this.duration = duration; this.createTime = createTime; + this.dataFormat = dataFormat; String comma = ","; StringBuilder sb = new StringBuilder(); sb.append("event=").append(String.join(comma, events)).append(comma); - sb.append(dataFormat).append(comma); - if(execArgs != null && !execArgs.isEmpty()) { + if (execArgs != null && !execArgs.isEmpty()) { sb.append(execArgs); } this.execArgs = sb.toString(); } - public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration, String execArgs, long createTime) { + public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration, + AsyncProfilerDataFormatType dataFormat, String execArgs, long createTime) { super(NAME, serialNumber); this.taskId = taskId; this.duration = duration; this.execArgs = execArgs; + this.dataFormat = dataFormat; this.createTime = createTime; } @@ -63,6 +68,7 @@ public AsyncProfilerTaskCommand deserialize(Command command) { String execArgs = null; long createTime = 0; String serialNumber = null; + AsyncProfilerDataFormatType dataFormat = null; for (final KeyStringValuePair pair : argsList) { if ("SerialNumber".equals(pair.getKey())) { serialNumber = pair.getValue(); @@ -74,9 +80,11 @@ public AsyncProfilerTaskCommand deserialize(Command command) { execArgs = pair.getValue(); } else if ("CreateTime".equals(pair.getKey())) { createTime = Long.parseLong(pair.getValue()); + } else if ("AsyncProfilerDataFormatType".equals(pair.getKey())) { + dataFormat = AsyncProfilerDataFormatType.valueOf(pair.getValue()); } } - return new AsyncProfilerTaskCommand(serialNumber, taskId, duration, execArgs, createTime); + return new AsyncProfilerTaskCommand(serialNumber, taskId, duration, dataFormat, execArgs, createTime); } @Override @@ -85,7 +93,9 @@ public Command.Builder serialize() { builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId)) .addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration))) .addArgs(KeyStringValuePair.newBuilder().setKey("ExecArgs").setValue(execArgs)) - .addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime))); + .addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime))) + .addArgs(KeyStringValuePair.newBuilder().setKey("AsyncProfilerDataFormatType") + .setValue(String.valueOf(dataFormat.toString()))); return builder; } @@ -105,4 +115,7 @@ public long getCreateTime() { return createTime; } + public AsyncProfilerDataFormatType getDataFormat() { + return this.dataFormat; + } } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java index 99eef1a2a5..b768ea2df3 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java @@ -19,6 +19,8 @@ import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerMetaData; import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerTaskGrpc; +import java.io.IOException; +import java.io.InputStream; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -64,8 +66,8 @@ public void statusChanged(GRPCChannelStatus status) { this.status = status; } - public void send(AsyncProfilerTask task, byte[] data) { - if (status != GRPCChannelStatus.CONNECTED || Objects.isNull(data) || data.length == 0) { + public void send(AsyncProfilerTask task, InputStream fileDataInputStream) throws IOException { + if (status != GRPCChannelStatus.CONNECTED || Objects.isNull(fileDataInputStream)) { return; } final GRPCStreamServiceStatus status = new GRPCStreamServiceStatus(false); @@ -96,22 +98,18 @@ public void onCompleted() { .setService(Config.Agent.SERVICE_NAME) .setServiceInstance(Config.Agent.INSTANCE_NAME) .setTaskId(task.getTaskId()) - .setExecutionArgs(task.getExecArgs()) .build(); AsyncProfilerData asyncProfilerData = AsyncProfilerData.newBuilder().setMetaData(metaData).build(); dataStreamObserver.onNext(asyncProfilerData); // send bin data - int idx = 0; - int len = data.length; - do { - int size = Math.min(DATA_CHUNK_SIZE, len - idx); + byte[] data = new byte[DATA_CHUNK_SIZE]; + int byteRead; + while ((byteRead = fileDataInputStream.read()) != -1) { asyncProfilerData = AsyncProfilerData.newBuilder() - .setContent(ByteString.copyFrom(data, idx, size)) + .setContent(ByteString.copyFrom(data, 0, byteRead)) .build(); dataStreamObserver.onNext(asyncProfilerData); - idx += DATA_CHUNK_SIZE; - } while (idx < len); - + } dataStreamObserver.onCompleted(); status.wait4Finish(); } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java index 6ad6e9a66e..3cf17eef50 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java @@ -21,10 +21,11 @@ import io.pyroscope.one.profiler.AsyncProfiler; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerDataFormatType; -import java.io.DataInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -52,18 +53,18 @@ public class AsyncProfilerTask { */ private Path tempFile; + private AsyncProfilerDataFormatType dataFormat; + private static String execute(AsyncProfiler asyncProfiler, String arg) throws IllegalArgumentException, IOException { LOGGER.info("async profiler execute arg:{}", arg); String result = asyncProfiler.execute(arg); - if (!result.endsWith("\n")) { - result += "\n"; - } - return result; + return result.trim(); } public String start(AsyncProfiler asyncProfiler) throws IOException { - tempFile = Files.createFile(Paths.get("/Users/bytedance/IdeaProjects/skywalking-java/skywalking-output/" + taskId)); + + tempFile = Files.createFile(Paths.get("/Users/bytedance/IdeaProjects/skywalking-java/skywalking-output/" + taskId, "")); execArgs = execArgs + "file=" + tempFile.toAbsolutePath(); return execute(asyncProfiler, execArgs); } @@ -71,28 +72,37 @@ public String start(AsyncProfiler asyncProfiler) throws IOException { /** * stop async-profiler and dump profile data */ - public byte[] stop(AsyncProfiler asyncProfiler) throws IOException { + public InputStream stop(AsyncProfiler asyncProfiler) throws IOException { LOGGER.info("async profiler process stop and dump file"); asyncProfiler.stop(); - // todo now only JFR - return dumpJFR(); -// final byte[] data; -// if (format.equals(JFR)) { -// data = dumpJFR(); -// } else { -// data = asyncProfiler.dumpCollapsed(Counter.SAMPLES).getBytes(StandardCharsets.UTF_8); -// } -// return data; - } - - private byte[] dumpJFR() throws IOException { +// return dumpJFR(); + if (AsyncProfilerDataFormatType.JFR.equals(dataFormat)) { + return getJFRInputStream(); + } else if (AsyncProfilerDataFormatType.HTML.equals(dataFormat)) { + return null; + } else { + return null; + } + } + + private String getFileExtension() { + switch (dataFormat) { + case JFR: + return ".jfr"; + case HTML: + return ".html"; + default: + return ".html"; + } + } + + /** + * need manually close + */ + private InputStream getJFRInputStream() throws IOException { File file = tempFile.toFile(); file.deleteOnExit(); - byte[] bytes = new byte[(int) file.length()]; - try (DataInputStream ds = new DataInputStream(Files.newInputStream(file.toPath()))) { - ds.readFully(bytes); - } - return bytes; + return Files.newInputStream(file.toPath()); } public void setExecArgs(String execArgs) { @@ -115,6 +125,14 @@ public void setCreateTime(long createTime) { this.createTime = createTime; } + public void setDataFormat(AsyncProfilerDataFormatType dataFormat) { + this.dataFormat = dataFormat; + } + + public AsyncProfilerDataFormatType getDataFormat() { + return this.dataFormat; + } + public String getExecArgs() { return execArgs; } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java index 2ad325075b..262621678a 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java @@ -28,6 +28,7 @@ import org.apache.skywalking.apm.agent.core.logging.api.LogManager; import java.io.IOException; +import java.io.InputStream; import java.util.Objects; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -86,10 +87,10 @@ public void processAsyncProfilerTask(AsyncProfilerTask task) { private void stopAsyncProfile(AsyncProfilerTask task) { try { // execute stop task - byte[] data = task.stop(ASYNC_PROFILER); + InputStream fileDataInputStream = task.stop(ASYNC_PROFILER); // upload file AsyncProfilerDataSender dataSender = ServiceManager.INSTANCE.findService(AsyncProfilerDataSender.class); - dataSender.send(task, data); + dataSender.send(task, fileDataInputStream); } catch (Exception e) { LOGGER.error("stop async profiler task error", e); return; diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java index adb6336c98..3212b030a7 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java @@ -36,6 +36,7 @@ public void execute(BaseCommand command) throws CommandExecutionException { asyncProfilerTask.setDuration(asyncProfilerTaskCommand.getDuration()); asyncProfilerTask.setExecArgs(asyncProfilerTaskCommand.getExecArgs()); asyncProfilerTask.setCreateTime(asyncProfilerTaskCommand.getCreateTime()); + asyncProfilerTask.setDataFormat(asyncProfilerTaskCommand.getDataFormat()); ServiceManager.INSTANCE.findService(AsyncProfilerTaskExecutionService.class) .processAsyncProfilerTask(asyncProfilerTask); } From 8a0e11da25b71a0508bef0a43d385da223174d8e Mon Sep 17 00:00:00 2001 From: "zhengziyi.0117" Date: Tue, 16 Jul 2024 15:48:03 +0800 Subject: [PATCH 05/14] fix: inputStream upload profile data --- .../AsyncProfilerDataSender.java | 2 +- .../core/asyncprofiler/AsyncProfilerTask.java | 42 +++++++++++-------- .../AsyncProfilerTaskExecutionService.java | 4 +- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java index b768ea2df3..2fa2ee0892 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java @@ -104,7 +104,7 @@ public void onCompleted() { // send bin data byte[] data = new byte[DATA_CHUNK_SIZE]; int byteRead; - while ((byteRead = fileDataInputStream.read()) != -1) { + while ((byteRead = fileDataInputStream.read(data)) != -1) { asyncProfilerData = AsyncProfilerData.newBuilder() .setContent(ByteString.copyFrom(data, 0, byteRead)) .build(); diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java index 3cf17eef50..3bb7f713d9 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java @@ -22,6 +22,7 @@ import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerDataFormatType; +import org.apache.skywalking.apm.util.StringUtil; import java.io.File; import java.io.IOException; @@ -32,6 +33,7 @@ public class AsyncProfilerTask { private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTask.class); + private static final String COMMA = ","; /** * task id */ @@ -55,34 +57,38 @@ public class AsyncProfilerTask { private AsyncProfilerDataFormatType dataFormat; - private static String execute(AsyncProfiler asyncProfiler, String arg) + private static String execute(AsyncProfiler asyncProfiler, String args) throws IllegalArgumentException, IOException { - LOGGER.info("async profiler execute arg:{}", arg); - String result = asyncProfiler.execute(arg); + LOGGER.info("async profiler execute args:{}", args); + String result = asyncProfiler.execute(args); return result.trim(); } + /** + * start async profiler + * TODO file path + */ public String start(AsyncProfiler asyncProfiler) throws IOException { + Path tempFilePath = Paths.get("/Users/bytedance/IdeaProjects/skywalking-java/skywalking-output", taskId + getFileExtension()); + tempFile = Files.createFile(tempFilePath); + StringBuilder startArgs = new StringBuilder(); + startArgs.append("start").append(COMMA); + if (StringUtil.isEmpty(execArgs)) { + startArgs.append(execArgs).append(COMMA); + } + startArgs.append("file=").append(tempFile.toAbsolutePath()); - tempFile = Files.createFile(Paths.get("/Users/bytedance/IdeaProjects/skywalking-java/skywalking-output/" + taskId, "")); - execArgs = execArgs + "file=" + tempFile.toAbsolutePath(); - return execute(asyncProfiler, execArgs); + return execute(asyncProfiler, startArgs.toString()); } /** - * stop async-profiler and dump profile data + * stop async-profiler and get dump file inputStream */ public InputStream stop(AsyncProfiler asyncProfiler) throws IOException { LOGGER.info("async profiler process stop and dump file"); - asyncProfiler.stop(); -// return dumpJFR(); - if (AsyncProfilerDataFormatType.JFR.equals(dataFormat)) { - return getJFRInputStream(); - } else if (AsyncProfilerDataFormatType.HTML.equals(dataFormat)) { - return null; - } else { - return null; - } + String stopArgs = "stop" + COMMA + "file=" + tempFile.toAbsolutePath(); + execute(asyncProfiler, stopArgs); + return getProfilerFileInputStream(); } private String getFileExtension() { @@ -92,14 +98,14 @@ private String getFileExtension() { case HTML: return ".html"; default: - return ".html"; + return ".txt"; } } /** * need manually close */ - private InputStream getJFRInputStream() throws IOException { + private InputStream getProfilerFileInputStream() throws IOException { File file = tempFile.toFile(); file.deleteOnExit(); return Files.newInputStream(file.toPath()); diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java index 262621678a..6765c6233e 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java @@ -42,7 +42,7 @@ public class AsyncProfilerTaskExecutionService implements BootService { private static final AsyncProfiler ASYNC_PROFILER = PyroscopeAsyncProfiler.getAsyncProfiler(); - private static final String SUCCESS_RESULT = "Profiling started\n"; + private static final String SUCCESS_RESULT = "Profiling started"; // profile executor thread pool, only running one thread private final static ScheduledExecutorService ASYNC_PROFILE_EXECUTOR = Executors.newSingleThreadScheduledExecutor( @@ -91,6 +91,8 @@ private void stopAsyncProfile(AsyncProfilerTask task) { // upload file AsyncProfilerDataSender dataSender = ServiceManager.INSTANCE.findService(AsyncProfilerDataSender.class); dataSender.send(task, fileDataInputStream); + // close inputStream + fileDataInputStream.close(); } catch (Exception e) { LOGGER.error("stop async profiler task error", e); return; From 57ff1d75a7af031128945f608cefd93b1c7bc813 Mon Sep 17 00:00:00 2001 From: "zhengziyi.0117" Date: Tue, 16 Jul 2024 16:11:25 +0800 Subject: [PATCH 06/14] fix: code style --- .../command/AsyncProfilerTaskCommand.java | 1 + .../AsyncProfilerDataSender.java | 18 ++++++++ .../AsyncProfilerTaskChannelService.java | 1 - .../profile/ProfileTaskChannelService.java | 42 +++++++++---------- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java index 9c3dc992f4..3c02ad8df2 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java @@ -15,6 +15,7 @@ * limitations under the License. * */ + package org.apache.skywalking.apm.network.trace.component.command; import org.apache.skywalking.apm.network.common.v3.Command; diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java index 2fa2ee0892..dee6b2c59a 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.apache.skywalking.apm.agent.core.asyncprofiler; import com.google.protobuf.ByteString; diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java index fa045eeafa..07d3b3b7b1 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java @@ -41,7 +41,6 @@ import static org.apache.skywalking.apm.agent.core.conf.Config.Collector.GRPC_UPSTREAM_TIMEOUT; - @DefaultImplementor public class AsyncProfilerTaskChannelService implements BootService, Runnable, GRPCChannelListener { private static final ILog LOGGER = LogManager.getLogger(AsyncProfilerTaskChannelService.class); diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java index f0d2e10b02..4ef65f0eb4 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/profile/ProfileTaskChannelService.java @@ -66,7 +66,7 @@ public class ProfileTaskChannelService implements BootService, Runnable, GRPCCha // segment snapshot sender private final BlockingQueue snapshotQueue = new LinkedBlockingQueue<>( - Config.Profile.SNAPSHOT_TRANSPORT_BUFFER_SIZE); + Config.Profile.SNAPSHOT_TRANSPORT_BUFFER_SIZE); private volatile ScheduledFuture sendSnapshotFuture; // query task list schedule @@ -85,10 +85,10 @@ public void run() { // last command create time builder.setLastCommandTime(ServiceManager.INSTANCE.findService(ProfileTaskExecutionService.class) - .getLastCommandCreateTime()); + .getLastCommandCreateTime()); Commands commands = profileTaskBlockingStub.withDeadlineAfter(GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS) - .getProfileTaskCommands(builder.build()); + .getProfileTaskCommands(builder.build()); ServiceManager.INSTANCE.findService(CommandService.class).receiveCommand(commands); } catch (Throwable t) { @@ -124,27 +124,27 @@ public void boot() { if (Config.Profile.ACTIVE) { // query task list getTaskListFuture = Executors.newSingleThreadScheduledExecutor( - new DefaultNamedThreadFactory("ProfileGetTaskService") + new DefaultNamedThreadFactory("ProfileGetTaskService") ).scheduleWithFixedDelay( - new RunnableWithExceptionProtection( - this, - t -> LOGGER.error("Query profile task list failure.", t) - ), 0, Config.Collector.GET_PROFILE_TASK_INTERVAL, TimeUnit.SECONDS + new RunnableWithExceptionProtection( + this, + t -> LOGGER.error("Query profile task list failure.", t) + ), 0, Config.Collector.GET_PROFILE_TASK_INTERVAL, TimeUnit.SECONDS ); sendSnapshotFuture = Executors.newSingleThreadScheduledExecutor( - new DefaultNamedThreadFactory("ProfileSendSnapshotService") + new DefaultNamedThreadFactory("ProfileSendSnapshotService") ).scheduleWithFixedDelay( - new RunnableWithExceptionProtection( - () -> { - List buffer = new ArrayList<>(Config.Profile.SNAPSHOT_TRANSPORT_BUFFER_SIZE); - snapshotQueue.drainTo(buffer); - if (!buffer.isEmpty()) { - sender.send(buffer); - } - }, - t -> LOGGER.error("Profile segment snapshot upload failure.", t) - ), 0, 500, TimeUnit.MILLISECONDS + new RunnableWithExceptionProtection( + () -> { + List buffer = new ArrayList<>(Config.Profile.SNAPSHOT_TRANSPORT_BUFFER_SIZE); + snapshotQueue.drainTo(buffer); + if (!buffer.isEmpty()) { + sender.send(buffer); + } + }, + t -> LOGGER.error("Profile segment snapshot upload failure.", t) + ), 0, 500, TimeUnit.MILLISECONDS ); } } @@ -190,13 +190,13 @@ public void notifyProfileTaskFinish(ProfileTask task) { final ProfileTaskFinishReport.Builder reportBuilder = ProfileTaskFinishReport.newBuilder(); // sniffer info reportBuilder.setService(Config.Agent.SERVICE_NAME) - .setServiceInstance(Config.Agent.INSTANCE_NAME); + .setServiceInstance(Config.Agent.INSTANCE_NAME); // task info reportBuilder.setTaskId(task.getTaskId()); // send data profileTaskBlockingStub.withDeadlineAfter(GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS) - .reportTaskFinish(reportBuilder.build()); + .reportTaskFinish(reportBuilder.build()); } catch (Throwable e) { LOGGER.error(e, "Notify profile task finish to backend fail."); } From b88c26bcbdfc61d204079ce3de02a31df880bc9c Mon Sep 17 00:00:00 2001 From: "zhengziyi.0117" Date: Thu, 25 Jul 2024 15:29:00 +0800 Subject: [PATCH 07/14] fix: some bugs --- .../apm/agent/core/asyncprofiler/AsyncProfilerTask.java | 2 +- .../asyncprofiler/AsyncProfilerTaskExecutionService.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java index 3bb7f713d9..d22400920d 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java @@ -73,7 +73,7 @@ public String start(AsyncProfiler asyncProfiler) throws IOException { tempFile = Files.createFile(tempFilePath); StringBuilder startArgs = new StringBuilder(); startArgs.append("start").append(COMMA); - if (StringUtil.isEmpty(execArgs)) { + if (StringUtil.isNotEmpty(execArgs)) { startArgs.append(execArgs).append(COMMA); } startArgs.append("file=").append(tempFile.toAbsolutePath()); diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java index 6765c6233e..a231ab493f 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java @@ -56,9 +56,11 @@ public class AsyncProfilerTaskExecutionService implements BootService { private volatile AsyncProfilerTask preAsyncProfilerTask; public void processAsyncProfilerTask(AsyncProfilerTask task) { - if (task.getCreateTime() > lastCommandCreateTime) { - lastCommandCreateTime = task.getCreateTime(); + if (task.getCreateTime() <= lastCommandCreateTime) { + LOGGER.warn("get repeat task because createTime is less than lastCommandCreateTime"); + return; } + lastCommandCreateTime = task.getCreateTime(); LOGGER.info("add async profiler task: {}", task.getTaskId()); // add task to list ASYNC_PROFILE_EXECUTOR.execute(() -> { From c51abb34a3855a5fc43188165c151dc4bc63831b Mon Sep 17 00:00:00 2001 From: "zhengziyi.0117" Date: Mon, 19 Aug 2024 01:17:32 +0800 Subject: [PATCH 08/14] fix: code --- .../asyncprofiler/AsyncProfilerTaskExecutionService.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java index a231ab493f..2c36bfcb14 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java @@ -53,7 +53,7 @@ public class AsyncProfilerTaskExecutionService implements BootService { // task schedule future private volatile ScheduledFuture scheduledFuture; - private volatile AsyncProfilerTask preAsyncProfilerTask; + private volatile AsyncProfilerTask preTask; public void processAsyncProfilerTask(AsyncProfilerTask task) { if (task.getCreateTime() <= lastCommandCreateTime) { @@ -68,15 +68,14 @@ public void processAsyncProfilerTask(AsyncProfilerTask task) { if (Objects.nonNull(scheduledFuture) && !scheduledFuture.isDone()) { scheduledFuture.cancel(true); // stop pre task - stopAsyncProfile(preAsyncProfilerTask); - preAsyncProfilerTask = null; + stopAsyncProfile(preTask); } String result = task.start(ASYNC_PROFILER); if (!SUCCESS_RESULT.equals(result)) { LOGGER.error("AsyncProfilerTask start fail result:" + result); return; } - preAsyncProfilerTask = task; + preTask = task; scheduledFuture = ASYNC_PROFILE_EXECUTOR.schedule( () -> stopAsyncProfile(task), task.getDuration(), TimeUnit.SECONDS ); From 4412f2b5e69ee8878e7c218f116f4d282dfdc61e Mon Sep 17 00:00:00 2001 From: zhengziyi Date: Thu, 12 Sep 2024 21:27:25 +0800 Subject: [PATCH 09/14] fix: tasks run repeatedly --- .../core/asyncprofiler/AsyncProfilerTask.java | 2 +- .../AsyncProfilerTaskExecutionService.java | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java index d22400920d..71e927e91f 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java @@ -69,7 +69,7 @@ private static String execute(AsyncProfiler asyncProfiler, String args) * TODO file path */ public String start(AsyncProfiler asyncProfiler) throws IOException { - Path tempFilePath = Paths.get("/Users/bytedance/IdeaProjects/skywalking-java/skywalking-output", taskId + getFileExtension()); + Path tempFilePath = Paths.get("/home/zhengziyi/IdeaProjects/skywalking-java/skywalking-output", taskId + getFileExtension()); tempFile = Files.createFile(tempFilePath); StringBuilder startArgs = new StringBuilder(); startArgs.append("start").append(COMMA); diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java index 2c36bfcb14..420abe2326 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java @@ -45,7 +45,7 @@ public class AsyncProfilerTaskExecutionService implements BootService { private static final String SUCCESS_RESULT = "Profiling started"; // profile executor thread pool, only running one thread - private final static ScheduledExecutorService ASYNC_PROFILE_EXECUTOR = Executors.newSingleThreadScheduledExecutor( + private static final ScheduledExecutorService ASYNC_PROFILER_EXECUTOR = Executors.newSingleThreadScheduledExecutor( new DefaultNamedThreadFactory("ASYNC-PROFILING-TASK")); // last command create time, use to next query task list @@ -53,7 +53,6 @@ public class AsyncProfilerTaskExecutionService implements BootService { // task schedule future private volatile ScheduledFuture scheduledFuture; - private volatile AsyncProfilerTask preTask; public void processAsyncProfilerTask(AsyncProfilerTask task) { if (task.getCreateTime() <= lastCommandCreateTime) { @@ -63,20 +62,18 @@ public void processAsyncProfilerTask(AsyncProfilerTask task) { lastCommandCreateTime = task.getCreateTime(); LOGGER.info("add async profiler task: {}", task.getTaskId()); // add task to list - ASYNC_PROFILE_EXECUTOR.execute(() -> { + ASYNC_PROFILER_EXECUTOR.execute(() -> { try { if (Objects.nonNull(scheduledFuture) && !scheduledFuture.isDone()) { - scheduledFuture.cancel(true); - // stop pre task - stopAsyncProfile(preTask); + LOGGER.info("AsyncProfilerTask already running"); + return; } String result = task.start(ASYNC_PROFILER); if (!SUCCESS_RESULT.equals(result)) { LOGGER.error("AsyncProfilerTask start fail result:" + result); return; } - preTask = task; - scheduledFuture = ASYNC_PROFILE_EXECUTOR.schedule( + scheduledFuture = ASYNC_PROFILER_EXECUTOR.schedule( () -> stopAsyncProfile(task), task.getDuration(), TimeUnit.SECONDS ); } catch (IOException e) { @@ -121,6 +118,6 @@ public void onComplete() throws Throwable { @Override public void shutdown() throws Throwable { - ASYNC_PROFILE_EXECUTOR.shutdown(); + ASYNC_PROFILER_EXECUTOR.shutdown(); } } From 9fad5d4ef212f90312e96dc70ab566a8931f7488 Mon Sep 17 00:00:00 2001 From: zhengziyi Date: Thu, 12 Sep 2024 21:28:15 +0800 Subject: [PATCH 10/14] fix: only support jfr format --- apm-protocol/apm-network/src/main/proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index d4da569991..550aa6ae69 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit d4da5699915ee52288f8ff1c954decf6363485bc +Subproject commit 550aa6ae698733b7cb5277c9abc2f8f5fb95b9f8 From 7052660075813d43f83a67bc14df017d5cf95444 Mon Sep 17 00:00:00 2001 From: zhengziyi Date: Tue, 17 Sep 2024 12:44:10 +0800 Subject: [PATCH 11/14] feat: support service dimensions jfr profiling --- .../command/AsyncProfilerTaskCommand.java | 29 +++++++------------ apm-protocol/apm-network/src/main/proto | 2 +- .../AsyncProfilerDataSender.java | 1 + .../core/asyncprofiler/AsyncProfilerTask.java | 20 +------------ .../AsyncProfilerCommandExecutor.java | 1 - 5 files changed, 13 insertions(+), 40 deletions(-) diff --git a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java index 3c02ad8df2..57d87610fe 100644 --- a/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java +++ b/apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/command/AsyncProfilerTaskCommand.java @@ -20,9 +20,9 @@ import org.apache.skywalking.apm.network.common.v3.Command; import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; -import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerDataFormatType; import java.util.List; +import java.util.Objects; public class AsyncProfilerTaskCommand extends BaseCommand implements Serializable, Deserializable { public static final Deserializable DESERIALIZER = new AsyncProfilerTaskCommand("", "", 0, null, "", 0); @@ -32,19 +32,20 @@ public class AsyncProfilerTaskCommand extends BaseCommand implements Serializabl private final int duration; private final String execArgs; private final long createTime; - private final AsyncProfilerDataFormatType dataFormat; public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration, - AsyncProfilerDataFormatType dataFormat, String events, - String execArgs, long createTime) { + List events, String execArgs, long createTime) { super(NAME, serialNumber); this.taskId = taskId; this.duration = duration; this.createTime = createTime; - this.dataFormat = dataFormat; String comma = ","; StringBuilder sb = new StringBuilder(); - sb.append("event=").append(String.join(comma, events)).append(comma); + if (Objects.nonNull(events) && !events.isEmpty()) { + sb.append("event=") + .append(String.join(comma, events)) + .append(comma); + } if (execArgs != null && !execArgs.isEmpty()) { sb.append(execArgs); } @@ -52,12 +53,11 @@ public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration } public AsyncProfilerTaskCommand(String serialNumber, String taskId, int duration, - AsyncProfilerDataFormatType dataFormat, String execArgs, long createTime) { + String execArgs, long createTime) { super(NAME, serialNumber); this.taskId = taskId; this.duration = duration; this.execArgs = execArgs; - this.dataFormat = dataFormat; this.createTime = createTime; } @@ -69,7 +69,6 @@ public AsyncProfilerTaskCommand deserialize(Command command) { String execArgs = null; long createTime = 0; String serialNumber = null; - AsyncProfilerDataFormatType dataFormat = null; for (final KeyStringValuePair pair : argsList) { if ("SerialNumber".equals(pair.getKey())) { serialNumber = pair.getValue(); @@ -81,11 +80,9 @@ public AsyncProfilerTaskCommand deserialize(Command command) { execArgs = pair.getValue(); } else if ("CreateTime".equals(pair.getKey())) { createTime = Long.parseLong(pair.getValue()); - } else if ("AsyncProfilerDataFormatType".equals(pair.getKey())) { - dataFormat = AsyncProfilerDataFormatType.valueOf(pair.getValue()); } } - return new AsyncProfilerTaskCommand(serialNumber, taskId, duration, dataFormat, execArgs, createTime); + return new AsyncProfilerTaskCommand(serialNumber, taskId, duration, execArgs, createTime); } @Override @@ -94,9 +91,7 @@ public Command.Builder serialize() { builder.addArgs(KeyStringValuePair.newBuilder().setKey("TaskId").setValue(taskId)) .addArgs(KeyStringValuePair.newBuilder().setKey("Duration").setValue(String.valueOf(duration))) .addArgs(KeyStringValuePair.newBuilder().setKey("ExecArgs").setValue(execArgs)) - .addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime))) - .addArgs(KeyStringValuePair.newBuilder().setKey("AsyncProfilerDataFormatType") - .setValue(String.valueOf(dataFormat.toString()))); + .addArgs(KeyStringValuePair.newBuilder().setKey("CreateTime").setValue(String.valueOf(createTime))); return builder; } @@ -115,8 +110,4 @@ public String getExecArgs() { public long getCreateTime() { return createTime; } - - public AsyncProfilerDataFormatType getDataFormat() { - return this.dataFormat; - } } diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index 550aa6ae69..af38061136 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit 550aa6ae698733b7cb5277c9abc2f8f5fb95b9f8 +Subproject commit af38061136af67436429c0d9e5eb0735ec773086 diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java index dee6b2c59a..2f46f8688e 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerDataSender.java @@ -115,6 +115,7 @@ public void onCompleted() { AsyncProfilerMetaData metaData = AsyncProfilerMetaData.newBuilder() .setService(Config.Agent.SERVICE_NAME) .setServiceInstance(Config.Agent.INSTANCE_NAME) + .setUploadTime(System.currentTimeMillis()) .setTaskId(task.getTaskId()) .build(); AsyncProfilerData asyncProfilerData = AsyncProfilerData.newBuilder().setMetaData(metaData).build(); diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java index 71e927e91f..be04fcb76d 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java @@ -21,7 +21,6 @@ import io.pyroscope.one.profiler.AsyncProfiler; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; -import org.apache.skywalking.apm.network.language.asyncprofile.v3.AsyncProfilerDataFormatType; import org.apache.skywalking.apm.util.StringUtil; import java.io.File; @@ -55,8 +54,6 @@ public class AsyncProfilerTask { */ private Path tempFile; - private AsyncProfilerDataFormatType dataFormat; - private static String execute(AsyncProfiler asyncProfiler, String args) throws IllegalArgumentException, IOException { LOGGER.info("async profiler execute args:{}", args); @@ -92,14 +89,7 @@ public InputStream stop(AsyncProfiler asyncProfiler) throws IOException { } private String getFileExtension() { - switch (dataFormat) { - case JFR: - return ".jfr"; - case HTML: - return ".html"; - default: - return ".txt"; - } + return ".jfr"; } /** @@ -131,14 +121,6 @@ public void setCreateTime(long createTime) { this.createTime = createTime; } - public void setDataFormat(AsyncProfilerDataFormatType dataFormat) { - this.dataFormat = dataFormat; - } - - public AsyncProfilerDataFormatType getDataFormat() { - return this.dataFormat; - } - public String getExecArgs() { return execArgs; } diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java index 3212b030a7..adb6336c98 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java @@ -36,7 +36,6 @@ public void execute(BaseCommand command) throws CommandExecutionException { asyncProfilerTask.setDuration(asyncProfilerTaskCommand.getDuration()); asyncProfilerTask.setExecArgs(asyncProfilerTaskCommand.getExecArgs()); asyncProfilerTask.setCreateTime(asyncProfilerTaskCommand.getCreateTime()); - asyncProfilerTask.setDataFormat(asyncProfilerTaskCommand.getDataFormat()); ServiceManager.INSTANCE.findService(AsyncProfilerTaskExecutionService.class) .processAsyncProfilerTask(asyncProfilerTask); } From 030ebe685d309c862ce8d065309d5ffcfd5408e2 Mon Sep 17 00:00:00 2001 From: zhengziyi Date: Sat, 21 Sep 2024 13:49:11 +0800 Subject: [PATCH 12/14] feat: support async profiler agent config --- .../core/asyncprofiler/AsyncProfilerTask.java | 29 +++++++++---------- .../AsyncProfilerTaskChannelService.java | 19 +++++++----- .../AsyncProfilerTaskExecutionService.java | 6 +++- .../AsyncProfilerCommandExecutor.java | 4 ++- .../apm/agent/core/conf/Config.java | 19 ++++++++++++ apm-sniffer/config/agent.config | 6 ++++ 6 files changed, 58 insertions(+), 25 deletions(-) diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java index be04fcb76d..c04a08b08b 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java @@ -19,6 +19,7 @@ package org.apache.skywalking.apm.agent.core.asyncprofiler; import io.pyroscope.one.profiler.AsyncProfiler; +import org.apache.skywalking.apm.agent.core.conf.Config; import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; import org.apache.skywalking.apm.util.StringUtil; @@ -63,17 +64,15 @@ private static String execute(AsyncProfiler asyncProfiler, String args) /** * start async profiler - * TODO file path */ public String start(AsyncProfiler asyncProfiler) throws IOException { - Path tempFilePath = Paths.get("/home/zhengziyi/IdeaProjects/skywalking-java/skywalking-output", taskId + getFileExtension()); - tempFile = Files.createFile(tempFilePath); + tempFile = getProfilerFilePath(); StringBuilder startArgs = new StringBuilder(); startArgs.append("start").append(COMMA); if (StringUtil.isNotEmpty(execArgs)) { startArgs.append(execArgs).append(COMMA); } - startArgs.append("file=").append(tempFile.toAbsolutePath()); + startArgs.append("file=").append(tempFile.toString()); return execute(asyncProfiler, startArgs.toString()); } @@ -81,24 +80,24 @@ public String start(AsyncProfiler asyncProfiler) throws IOException { /** * stop async-profiler and get dump file inputStream */ - public InputStream stop(AsyncProfiler asyncProfiler) throws IOException { + public File stop(AsyncProfiler asyncProfiler) throws IOException { LOGGER.info("async profiler process stop and dump file"); String stopArgs = "stop" + COMMA + "file=" + tempFile.toAbsolutePath(); execute(asyncProfiler, stopArgs); - return getProfilerFileInputStream(); + return tempFile.toFile(); } - private String getFileExtension() { - return ".jfr"; + public Path getProfilerFilePath() throws IOException { + if (StringUtil.isNotEmpty(Config.AsyncProfiler.OUTPUT_PATH)) { + Path tempFilePath = Paths.get(Config.AsyncProfiler.OUTPUT_PATH, taskId + getFileExtension()); + return Files.createFile(tempFilePath.toAbsolutePath()); + } else { + return Files.createTempFile(taskId + getFileExtension(), taskId + getFileExtension()); + } } - /** - * need manually close - */ - private InputStream getProfilerFileInputStream() throws IOException { - File file = tempFile.toFile(); - file.deleteOnExit(); - return Files.newInputStream(file.toPath()); + private String getFileExtension() { + return ".jfr"; } public void setExecArgs(String execArgs) { diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java index 07d3b3b7b1..79bb4f016e 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskChannelService.java @@ -88,14 +88,17 @@ public void prepare() throws Throwable { @Override public void boot() throws Throwable { - getTaskListFuture = Executors.newSingleThreadScheduledExecutor( - new DefaultNamedThreadFactory("AsyncProfileGetTaskService") - ).scheduleWithFixedDelay( - new RunnableWithExceptionProtection( - this, - t -> LOGGER.error("Query async profile task list failure.", t) - ), 0, Config.Collector.GET_PROFILE_TASK_INTERVAL, TimeUnit.SECONDS - ); + + if (Config.AsyncProfiler.ACTIVE) { + getTaskListFuture = Executors.newSingleThreadScheduledExecutor( + new DefaultNamedThreadFactory("AsyncProfilerGetTaskService") + ).scheduleWithFixedDelay( + new RunnableWithExceptionProtection( + this, + t -> LOGGER.error("Query async profiler task list failure.", t) + ), 0, Config.Collector.GET_PROFILE_TASK_INTERVAL, TimeUnit.SECONDS + ); + } } @Override diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java index 420abe2326..d8f5f6a022 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java @@ -27,8 +27,10 @@ import org.apache.skywalking.apm.agent.core.logging.api.ILog; import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; import java.util.Objects; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -85,12 +87,14 @@ public void processAsyncProfilerTask(AsyncProfilerTask task) { private void stopAsyncProfile(AsyncProfilerTask task) { try { // execute stop task - InputStream fileDataInputStream = task.stop(ASYNC_PROFILER); + File dumpFile = task.stop(ASYNC_PROFILER); + InputStream fileDataInputStream = Files.newInputStream(dumpFile.toPath()); // upload file AsyncProfilerDataSender dataSender = ServiceManager.INSTANCE.findService(AsyncProfilerDataSender.class); dataSender.send(task, fileDataInputStream); // close inputStream fileDataInputStream.close(); + dumpFile.delete(); } catch (Exception e) { LOGGER.error("stop async profiler task error", e); return; diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java index adb6336c98..530b655f84 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/commands/executor/AsyncProfilerCommandExecutor.java @@ -23,6 +23,7 @@ import org.apache.skywalking.apm.agent.core.boot.ServiceManager; import org.apache.skywalking.apm.agent.core.commands.CommandExecutionException; import org.apache.skywalking.apm.agent.core.commands.CommandExecutor; +import org.apache.skywalking.apm.agent.core.conf.Config; import org.apache.skywalking.apm.network.trace.component.command.AsyncProfilerTaskCommand; import org.apache.skywalking.apm.network.trace.component.command.BaseCommand; @@ -33,7 +34,8 @@ public void execute(BaseCommand command) throws CommandExecutionException { AsyncProfilerTask asyncProfilerTask = new AsyncProfilerTask(); asyncProfilerTask.setTaskId(asyncProfilerTaskCommand.getTaskId()); - asyncProfilerTask.setDuration(asyncProfilerTaskCommand.getDuration()); + int duration = Math.min(Config.AsyncProfiler.MAX_DURATION, asyncProfilerTaskCommand.getDuration()); + asyncProfilerTask.setDuration(duration); asyncProfilerTask.setExecArgs(asyncProfilerTaskCommand.getExecArgs()); asyncProfilerTask.setCreateTime(asyncProfilerTaskCommand.getCreateTime()); ServiceManager.INSTANCE.findService(AsyncProfilerTaskExecutionService.class) diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java index be7d54a4e4..dc79dafc6e 100755 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java @@ -252,6 +252,25 @@ public static class Profile { public static int SNAPSHOT_TRANSPORT_BUFFER_SIZE = 500; } + public static class AsyncProfiler { + /** + * If true, skywalking agent will enable profile when user create a new async profiler task. + * Otherwise disable it. + */ + public static boolean ACTIVE = true; + + /** + * Max monitor time(second), if async profiler monitor time out of limit, then stop it. + * default 1h. + */ + public static int MAX_DURATION = 3600; + + /** + * jfr directory generated by async profiler + */ + public static String OUTPUT_PATH = ""; + } + public static class Meter { /** * If true, skywalking agent will enable sending meters. Otherwise disable meter report. diff --git a/apm-sniffer/config/agent.config b/apm-sniffer/config/agent.config index 06f5717de0..de09acc78e 100755 --- a/apm-sniffer/config/agent.config +++ b/apm-sniffer/config/agent.config @@ -164,6 +164,12 @@ profile.duration=${SW_AGENT_PROFILE_DURATION:10} profile.dump_max_stack_depth=${SW_AGENT_PROFILE_DUMP_MAX_STACK_DEPTH:500} # Snapshot transport to backend buffer size profile.snapshot_transport_buffer_size=${SW_AGENT_PROFILE_SNAPSHOT_TRANSPORT_BUFFER_SIZE:4500} +# If true, skywalking agent will enable profile when user create a new asyncprofile task. Otherwise disable it. +asyncprofiler.active=${SW_AGENT_ASYNC_PROFILER_ACTIVE:true} +# Max monitor time(second), if async profiler monitor time out of limit, then stop it. default 1h. +asyncprofiler.max_duration=${SW_AGENT_ASYNC_PROFILER_MAX_DURATION:3600} +# jfr directory generated by async profiler +asyncprofiler.output_path=${SW_AGENT_ASYNC_PROFILER_OUTPUT_PATH:} # If true, the agent collects and reports metrics to the backend. meter.active=${SW_METER_ACTIVE:true} # Report meters interval. The unit is second From 434dc01a8a97d7bd86b3bb5c0c4155ae78a61295 Mon Sep 17 00:00:00 2001 From: zhengziyi Date: Sat, 21 Sep 2024 13:57:38 +0800 Subject: [PATCH 13/14] fix: add log if jfr delete error --- .../core/asyncprofiler/AsyncProfilerTaskExecutionService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java index d8f5f6a022..4cffc13917 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTaskExecutionService.java @@ -94,7 +94,9 @@ private void stopAsyncProfile(AsyncProfilerTask task) { dataSender.send(task, fileDataInputStream); // close inputStream fileDataInputStream.close(); - dumpFile.delete(); + if (!dumpFile.delete()) { + LOGGER.warn("delete async profiler dump file failed"); + } } catch (Exception e) { LOGGER.error("stop async profiler task error", e); return; From 827819bccbf70148a61496c84ff70e57678f6117 Mon Sep 17 00:00:00 2001 From: zhengziyi Date: Thu, 26 Sep 2024 20:59:13 +0800 Subject: [PATCH 14/14] fix: code issue --- .../apm/agent/core/asyncprofiler/AsyncProfilerTask.java | 1 - 1 file changed, 1 deletion(-) diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java index c04a08b08b..2c82fb83fd 100644 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/asyncprofiler/AsyncProfilerTask.java @@ -26,7 +26,6 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths;