Skip to content

Commit 0e96fc6

Browse files
lauritlaurit
committed
Add alternative cpu profiler for jvms without jfr
1 parent 10ea6d8 commit 0e96fc6

File tree

15 files changed

+383
-253
lines changed

15 files changed

+383
-253
lines changed

‎profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ public class Configuration implements AutoConfigurationCustomizerProvider {
4343
private static final boolean DEFAULT_MEMORY_EVENT_RATE_LIMIT_ENABLED = true;
4444

4545
public static final String CONFIG_KEY_ENABLE_PROFILER = PROFILER_ENABLED_PROPERTY;
46+
public static final String CONFIG_KEY_PROFILER_JFR = "splunk.profiler.jfr";
4647
public static final String CONFIG_KEY_PROFILER_DIRECTORY = "splunk.profiler.directory";
4748
public static final String CONFIG_KEY_RECORDING_DURATION = "splunk.profiler.recording.duration";
4849
public static final String CONFIG_KEY_KEEP_FILES = "splunk.profiler.keep-files";
@@ -101,6 +102,10 @@ public static String getConfigUrl(ConfigProperties config) {
101102
return config.getString(CONFIG_KEY_INGEST_URL, ingestUrl);
102103
}
103104

105+
public static boolean getProfilerJfrEnabled(ConfigProperties config) {
106+
return config.getBoolean(CONFIG_KEY_PROFILER_JFR, true);
107+
}
108+
104109
public static boolean getTLABEnabled(ConfigProperties config) {
105110
boolean memoryEnabled = config.getBoolean(CONFIG_KEY_MEMORY_ENABLED, DEFAULT_MEMORY_ENABLED);
106111
return config.getBoolean(CONFIG_KEY_TLAB_ENABLED, memoryEnabled);

‎profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java

Lines changed: 79 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,12 @@
2727
import com.splunk.opentelemetry.profiler.allocation.exporter.AllocationEventExporter;
2828
import com.splunk.opentelemetry.profiler.allocation.exporter.PprofAllocationEventExporter;
2929
import com.splunk.opentelemetry.profiler.context.SpanContextualizer;
30-
import com.splunk.opentelemetry.profiler.events.EventPeriods;
30+
import com.splunk.opentelemetry.profiler.contextstorage.JavaContextStorage;
3131
import com.splunk.opentelemetry.profiler.exporter.CpuEventExporter;
3232
import com.splunk.opentelemetry.profiler.exporter.PprofCpuEventExporter;
3333
import com.splunk.opentelemetry.profiler.util.HelpfulExecutors;
3434
import io.opentelemetry.api.logs.Logger;
35+
import io.opentelemetry.api.trace.SpanContext;
3536
import io.opentelemetry.javaagent.extension.AgentListener;
3637
import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
3738
import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
@@ -45,43 +46,103 @@
4546
import java.nio.file.Path;
4647
import java.nio.file.Paths;
4748
import java.time.Duration;
49+
import java.time.Instant;
50+
import java.util.HashMap;
4851
import java.util.Map;
4952
import java.util.concurrent.ExecutorService;
53+
import java.util.concurrent.ScheduledExecutorService;
54+
import java.util.concurrent.TimeUnit;
5055

5156
@AutoService(AgentListener.class)
5257
public class JfrActivator implements AgentListener {
5358

5459
private static final java.util.logging.Logger logger =
5560
java.util.logging.Logger.getLogger(JfrActivator.class.getName());
56-
private final ExecutorService executor = HelpfulExecutors.newSingleThreadExecutor("JFR Profiler");
5761
private final ConfigurationLogger configurationLogger = new ConfigurationLogger();
5862

5963
@Override
6064
public void afterAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetrySdk) {
6165
ConfigProperties config = autoConfiguredOpenTelemetrySdk.getConfig();
62-
if (notClearForTakeoff(config)) {
66+
if (!config.getBoolean(CONFIG_KEY_ENABLE_PROFILER, false)) {
67+
logger.fine("Profiler is not enabled.");
6368
return;
6469
}
70+
boolean useJfr = Configuration.getProfilerJfrEnabled(config);
71+
if (useJfr && !JFR.instance.isAvailable()) {
72+
logger.fine(
73+
"JDK Flight Recorder (JFR) is not available in this JVM, switching to java profiler.");
74+
if (Configuration.getTLABEnabled(config)) {
75+
logger.warning(
76+
"JDK Flight Recorder (JFR) is not available in this JVM. Memory profiling is disabled.");
77+
}
78+
useJfr = false;
79+
}
6580

6681
configurationLogger.log(config);
6782
logger.info("Profiler is active.");
68-
executor.submit(
69-
logUncaught(
70-
() -> activateJfrAndRunForever(config, autoConfiguredOpenTelemetrySdk.getResource())));
71-
}
7283

73-
private boolean notClearForTakeoff(ConfigProperties config) {
74-
if (!config.getBoolean(CONFIG_KEY_ENABLE_PROFILER, false)) {
75-
logger.fine("Profiler is not enabled.");
76-
return true;
84+
if (useJfr) {
85+
JfrProfiler.run(this, config, autoConfiguredOpenTelemetrySdk.getResource());
86+
} else {
87+
JavaProfiler.run(this, config, autoConfiguredOpenTelemetrySdk.getResource());
7788
}
78-
if (!JFR.instance.isAvailable()) {
79-
logger.warning(
80-
"JDK Flight Recorder (JFR) is not available in this JVM. Profiling is disabled.");
81-
return true;
89+
}
90+
91+
private static class JfrProfiler {
92+
private static final ExecutorService executor =
93+
HelpfulExecutors.newSingleThreadExecutor("JFR Profiler");
94+
95+
static void run(JfrActivator activator, ConfigProperties config, Resource resource) {
96+
executor.submit(logUncaught(() -> activator.activateJfrAndRunForever(config, resource)));
8297
}
98+
}
8399

84-
return false;
100+
private static class JavaProfiler {
101+
private static final ScheduledExecutorService scheduler =
102+
HelpfulExecutors.newSingleThreadedScheduledExecutor("Profiler scheduler");
103+
104+
static void run(JfrActivator activator, ConfigProperties config, Resource resource) {
105+
int stackDepth = Configuration.getStackDepth(config);
106+
LogRecordExporter logsExporter = LogExporterBuilder.fromConfig(config);
107+
CpuEventExporter cpuEventExporter =
108+
PprofCpuEventExporter.builder()
109+
.otelLogger(
110+
activator.buildOtelLogger(
111+
SimpleLogRecordProcessor.create(logsExporter), resource))
112+
.period(Configuration.getCallStackInterval(config))
113+
.stackDepth(stackDepth)
114+
.build();
115+
116+
Runnable profiler =
117+
() -> {
118+
Instant now = Instant.now();
119+
Map<Thread, StackTraceElement[]> stackTracesMap;
120+
Map<Thread, SpanContext> contextMap = new HashMap<>();
121+
// disallow context changes while we are taking the thread dump
122+
JavaContextStorage.block();
123+
try {
124+
stackTracesMap = Thread.getAllStackTraces();
125+
// copy active context for each thread
126+
for (Thread thread : stackTracesMap.keySet()) {
127+
SpanContext spanContext = JavaContextStorage.activeContext.get(thread);
128+
if (spanContext != null) {
129+
contextMap.put(thread, spanContext);
130+
}
131+
}
132+
} finally {
133+
JavaContextStorage.unblock();
134+
}
135+
for (Map.Entry<Thread, StackTraceElement[]> entry : stackTracesMap.entrySet()) {
136+
Thread thread = entry.getKey();
137+
SpanContext spanContext = contextMap.get(thread);
138+
cpuEventExporter.export(thread, entry.getValue(), now, spanContext);
139+
}
140+
cpuEventExporter.flush();
141+
};
142+
long period = Configuration.getCallStackInterval(config).toMillis();
143+
scheduler.scheduleAtFixedRate(
144+
logUncaught(() -> profiler.run()), period, period, TimeUnit.MILLISECONDS);
145+
}
85146
}
86147

87148
private boolean checkOutputDir(Path outputDir) {
@@ -107,7 +168,7 @@ private boolean checkOutputDir(Path outputDir) {
107168
return true;
108169
}
109170

110-
private void outdirWarn(Path dir, String suffix) {
171+
private static void outdirWarn(Path dir, String suffix) {
111172
logger.log(WARNING, "The configured output directory {0} {1}.", new Object[] {dir, suffix});
112173
}
113174

@@ -128,13 +189,12 @@ private void activateJfrAndRunForever(ConfigProperties config, Resource resource
128189

129190
EventReader eventReader = new EventReader();
130191
SpanContextualizer spanContextualizer = new SpanContextualizer(eventReader);
131-
EventPeriods periods = new EventPeriods(jfrSettings::get);
132192
LogRecordExporter logsExporter = LogExporterBuilder.fromConfig(config);
133193

134194
CpuEventExporter cpuEventExporter =
135195
PprofCpuEventExporter.builder()
136196
.otelLogger(buildOtelLogger(SimpleLogRecordProcessor.create(logsExporter), resource))
137-
.eventPeriods(periods)
197+
.period(Configuration.getCallStackInterval(config))
138198
.stackDepth(stackDepth)
139199
.build();
140200

‎profiler/src/main/java/com/splunk/opentelemetry/profiler/SdkCustomizer.java

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,12 @@
1717
package com.splunk.opentelemetry.profiler;
1818

1919
import static com.splunk.opentelemetry.profiler.Configuration.CONFIG_KEY_ENABLE_PROFILER;
20+
import static com.splunk.opentelemetry.profiler.Configuration.CONFIG_KEY_PROFILER_JFR;
2021
import static java.util.Collections.emptyMap;
2122

2223
import com.google.auto.service.AutoService;
24+
import com.splunk.opentelemetry.profiler.contextstorage.JavaContextStorage;
25+
import com.splunk.opentelemetry.profiler.contextstorage.JfrContextStorage;
2326
import io.opentelemetry.context.ContextStorage;
2427
import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer;
2528
import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
@@ -32,8 +35,12 @@ public class SdkCustomizer implements AutoConfigurationCustomizerProvider {
3235
public void customize(AutoConfigurationCustomizer autoConfigurationCustomizer) {
3336
autoConfigurationCustomizer.addPropertiesCustomizer(
3437
config -> {
35-
if (jfrIsAvailable() && jfrIsEnabledInConfig(config)) {
36-
ContextStorage.addWrapper(JfrContextStorage::new);
38+
if (profilerIsEnabledInConfig(config)) {
39+
if (jfrIsAvailable() && jfrIsEnabledInConfig(config)) {
40+
ContextStorage.addWrapper(JfrContextStorage::new);
41+
} else {
42+
ContextStorage.addWrapper(JavaContextStorage::new);
43+
}
3744
}
3845
return emptyMap();
3946
});
@@ -43,7 +50,11 @@ private boolean jfrIsAvailable() {
4350
return JFR.instance.isAvailable();
4451
}
4552

46-
private boolean jfrIsEnabledInConfig(ConfigProperties config) {
53+
private boolean profilerIsEnabledInConfig(ConfigProperties config) {
4754
return config.getBoolean(CONFIG_KEY_ENABLE_PROFILER, false);
4855
}
56+
57+
private boolean jfrIsEnabledInConfig(ConfigProperties config) {
58+
return config.getBoolean(CONFIG_KEY_PROFILER_JFR, true);
59+
}
4960
}

‎profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/AbstractContextStorage.java

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright Splunk Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.splunk.opentelemetry.profiler.contextstorage;
18+
19+
import io.opentelemetry.api.trace.Span;
20+
import io.opentelemetry.context.Context;
21+
import io.opentelemetry.context.ContextStorage;
22+
import io.opentelemetry.context.Scope;
23+
import javax.annotation.Nullable;
24+
25+
abstract class AbstractContextStorage implements ContextStorage {
26+
27+
private final ContextStorage delegate;
28+
private final ThreadLocal<Span> activeSpan = ThreadLocal.withInitial(Span::getInvalid);
29+
30+
AbstractContextStorage(ContextStorage delegate) {
31+
this.delegate = delegate;
32+
}
33+
34+
@Override
35+
public Scope attach(Context toAttach) {
36+
Scope delegatedScope = delegate.attach(toAttach);
37+
Span span = Span.fromContext(toAttach);
38+
Span current = activeSpan.get();
39+
// do nothing when active span didn't change
40+
// do nothing if the span isn't sampled
41+
if (span == current || !span.getSpanContext().isSampled()) {
42+
return delegatedScope;
43+
}
44+
45+
// mark new span as active and generate event
46+
activeSpan.set(span);
47+
activateSpan(span);
48+
return () -> {
49+
// restore previous active span
50+
activeSpan.set(current);
51+
activateSpan(current);
52+
delegatedScope.close();
53+
};
54+
}
55+
56+
protected abstract void activateSpan(Span span);
57+
58+
@Nullable
59+
@Override
60+
public Context current() {
61+
return delegate.current();
62+
}
63+
}

‎profiler/src/main/java/com/splunk/opentelemetry/profiler/contextstorage/JavaContextStorage.java

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* Copyright Splunk Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.splunk.opentelemetry.profiler.contextstorage;
18+
19+
import io.opentelemetry.api.trace.Span;
20+
import io.opentelemetry.api.trace.SpanContext;
21+
import io.opentelemetry.context.ContextStorage;
22+
import io.opentelemetry.instrumentation.api.internal.cache.Cache;
23+
24+
// active context tracking for java profiler
25+
public class JavaContextStorage extends AbstractContextStorage {
26+
27+
public static final Cache<Thread, SpanContext> activeContext = Cache.weak();
28+
private static final Guard NOP = () -> {};
29+
private static final BlockingGuard GUARD = new BlockingGuard();
30+
private static volatile Guard guard = NOP;
31+
32+
public JavaContextStorage(ContextStorage delegate) {
33+
super(delegate);
34+
}
35+
36+
public static void block() {
37+
guard = GUARD;
38+
}
39+
40+
public static void unblock() {
41+
guard = NOP;
42+
GUARD.release();
43+
}
44+
45+
@Override
46+
protected void activateSpan(Span span) {
47+
// when taking thread dump we block all thread that attempt to modify the active contexts
48+
guard.stop();
49+
50+
SpanContext context = span.getSpanContext();
51+
if (context.isValid()) {
52+
activeContext.put(Thread.currentThread(), context);
53+
} else {
54+
activeContext.remove(Thread.currentThread());
55+
}
56+
}
57+
58+
private interface Guard {
59+
void stop();
60+
}
61+
62+
private static class BlockingGuard implements Guard {
63+
64+
@Override
65+
public synchronized void stop() {
66+
try {
67+
while (guard == GUARD) {
68+
wait();
69+
}
70+
} catch (InterruptedException exception) {
71+
Thread.currentThread().interrupt();
72+
}
73+
}
74+
75+
synchronized void release() {
76+
notifyAll();
77+
}
78+
}
79+
}

0 commit comments

Comments
 (0)