From 416eb872eab90161ee92706d8e7d18243f5fd885 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Mon, 8 Jan 2024 22:35:40 +0100 Subject: [PATCH 01/34] bug: intermediate commit to allow work on another bug --- .../com/microsoft/gctoolkit/GCToolKit.java | 74 ++++++++++++------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java b/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java index ef78b82a..7a84dc17 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java +++ b/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java @@ -106,13 +106,19 @@ public GCToolKit() { * */ public void loadAggregationsFromServiceLoader() { - ServiceLoader.load(Aggregation.class) - .stream() - .map(ServiceLoader.Provider::get) - .forEach(aggregation -> { - registeredAggregations.add(aggregation); - LOG_DEBUG_MESSAGE(() -> "ServiceLoader provided: " + aggregation.getClass().getName()); - }); + try { + ServiceLoader.load(Aggregation.class) + .stream() + .map(ServiceLoader.Provider::get) + .forEach(aggregation -> { + registeredAggregations.add(aggregation); + LOG_DEBUG_MESSAGE(() -> "ServiceLoader provided: " + aggregation.getClass().getName()); + }); + } catch (Throwable e) { + System.out.println(e.getMessage()); + e.printStackTrace(); + throw e; + } } /** @@ -315,30 +321,42 @@ public JavaVirtualMachine analyze(DataSource dataSource) throws IOException private List> filterAggregations(Set events) { List> aggregators = new ArrayList<>(); - for (Aggregation aggregation : registeredAggregations) { - LOG_DEBUG_MESSAGE(() -> "Evaluating: " + aggregation.getClass().getName()); - Constructor> constructor = constructor(aggregation); - if (constructor == null) { - LOGGER.log(Level.WARNING, "Cannot find one of: default constructor or @Collates annotation for " + aggregation.getClass().getName()); - continue; - } + //try { + for (Aggregation aggregation : registeredAggregations) { + System.out.println("Evaluating: " + aggregation.getClass().getName()); + LOG_DEBUG_MESSAGE(() -> "Evaluating: " + aggregation.getClass().getName()); + Constructor> constructor = constructor(aggregation); + if (constructor == null) { + LOGGER.log(Level.WARNING, "Cannot find one of: default constructor or @Collates annotation for " + aggregation.getClass().getName()); + continue; + } - Aggregator aggregator = null; - try { - aggregator = constructor.newInstance(aggregation); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - continue; - } - if (events.stream().anyMatch(aggregator::aggregates)) { - LOG_DEBUG_MESSAGE(() -> "Including : " + aggregation.getClass().getName()); - aggregators.add(aggregator); - } else { - LOG_DEBUG_MESSAGE(() -> "Excluding : " + aggregation.getClass().getName()); + Aggregator aggregator = null; + try { + aggregator = constructor.newInstance(aggregation); + } catch (IllegalArgumentException iae) { + // what is the argument, what is expected + System.out.println("found " + aggregation.getClass().getName() + ", expected " + constructor.getParameterTypes()[0].getName()); + continue; + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + continue; + } + if (events.stream().anyMatch(aggregator::aggregates)) { + LOG_DEBUG_MESSAGE(() -> "Including : " + aggregation.getClass().getName()); + System.out.println("Including : " + aggregation.getClass().getName()); + aggregators.add(aggregator); + } else { + LOG_DEBUG_MESSAGE(() -> "Excluding : " + aggregation.getClass().getName()); + System.out.println("Excluding : " + aggregation.getClass().getName()); + } } - } - return aggregators; +// } catch (Throwable t) { +// t.printStackTrace(); +// System.out.println(t.getMessage()); +// } + return aggregators; } @SuppressWarnings("unchecked") From d7ab6fb0175b4ff54c807ad37636eee7180b70ac Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Wed, 24 Jan 2024 22:23:26 +0100 Subject: [PATCH 02/34] bug: add more robust handling of IllegalArgumentException in GCToolKit::analyize --- .../com/microsoft/gctoolkit/GCToolKit.java | 61 +++++++++---------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java b/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java index 7a84dc17..e13acef0 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java +++ b/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java @@ -321,40 +321,37 @@ public JavaVirtualMachine analyze(DataSource dataSource) throws IOException private List> filterAggregations(Set events) { List> aggregators = new ArrayList<>(); - //try { - for (Aggregation aggregation : registeredAggregations) { - System.out.println("Evaluating: " + aggregation.getClass().getName()); - LOG_DEBUG_MESSAGE(() -> "Evaluating: " + aggregation.getClass().getName()); - Constructor> constructor = constructor(aggregation); - if (constructor == null) { - LOGGER.log(Level.WARNING, "Cannot find one of: default constructor or @Collates annotation for " + aggregation.getClass().getName()); - continue; - } + for (Aggregation aggregation : registeredAggregations) { + LOG_DEBUG_MESSAGE(() -> "Evaluating: " + aggregation.getClass().getName()); + Constructor> constructor = constructor(aggregation); + if (constructor == null) { + LOGGER.log(Level.WARNING, "Cannot find one of: default constructor or @Collates annotation for " + aggregation.getClass().getName()); + continue; + } - Aggregator aggregator = null; - try { - aggregator = constructor.newInstance(aggregation); - } catch (IllegalArgumentException iae) { - // what is the argument, what is expected - System.out.println("found " + aggregation.getClass().getName() + ", expected " + constructor.getParameterTypes()[0].getName()); - continue; - } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - continue; - } - if (events.stream().anyMatch(aggregator::aggregates)) { - LOG_DEBUG_MESSAGE(() -> "Including : " + aggregation.getClass().getName()); - System.out.println("Including : " + aggregation.getClass().getName()); - aggregators.add(aggregator); - } else { - LOG_DEBUG_MESSAGE(() -> "Excluding : " + aggregation.getClass().getName()); - System.out.println("Excluding : " + aggregation.getClass().getName()); + Aggregator aggregator = null; + try { + aggregator = constructor.newInstance(aggregation); + } catch (IllegalArgumentException iae) { + Class argumentType = aggregation.getClass(); + LOGGER.log(Level.SEVERE, "Creating a " + constructor.getName() + " requires a " + constructor.getParameterTypes()[0].getName() + " but was supplied with a " + argumentType, iae); + LOGGER.log(Level.SEVERE, "Expanding " + argumentType); + while (argumentType != Aggregation.class) { + argumentType = argumentType.getSuperclass(); + LOGGER.log(Level.SEVERE, " extends " + argumentType.getName()); } + continue; + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + continue; } -// } catch (Throwable t) { -// t.printStackTrace(); -// System.out.println(t.getMessage()); -// } + if (events.stream().anyMatch(aggregator::aggregates)) { + LOG_DEBUG_MESSAGE(() -> "Including : " + aggregation.getClass().getName()); + aggregators.add(aggregator); + } else { + LOG_DEBUG_MESSAGE(() -> "Excluding : " + aggregation.getClass().getName()); + } + } return aggregators; } @@ -362,7 +359,7 @@ private List> filterAggregations(Set> constructor(Aggregation aggregation) { Class> targetClazz = aggregation.collates(); - if ( targetClazz != null) { + if (targetClazz != null) { Constructor[] constructors = targetClazz.getConstructors(); for (Constructor constructor : constructors) { Parameter[] parameters = constructor.getParameters(); From 37250952588eb893523611237c4b5a6b174f7ed7 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 25 Jan 2024 16:32:43 +0100 Subject: [PATCH 03/34] tidy: remove debug to stdout to logging framework --- api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java b/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java index e13acef0..6c258dcb 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java +++ b/api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java @@ -115,8 +115,7 @@ public void loadAggregationsFromServiceLoader() { LOG_DEBUG_MESSAGE(() -> "ServiceLoader provided: " + aggregation.getClass().getName()); }); } catch (Throwable e) { - System.out.println(e.getMessage()); - e.printStackTrace(); + LOGGER.log(Level.SEVERE, e.getMessage(), e); throw e; } } From f7efb6e7549552261b2846fb5e21f8cfeef8786b Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 22 Feb 2024 11:22:09 -0800 Subject: [PATCH 04/34] refactor: Ensure DateTimeStamp cannot be malformed and it behaves if the timestamp is missing --- .../query/MissingTimeStampTest.java | 74 +++++++++++++++++++ .../gctoolkit/aggregator/Aggregation.java | 2 +- .../gctoolkit/io/GCLogFileZipSegment.java | 3 +- .../gctoolkit/io/RotatingLogFileMetadata.java | 7 +- .../com/microsoft/gctoolkit/jvm/Diary.java | 9 ++- .../gctoolkit/time/DateTimeStamp.java | 23 ++++-- .../gctoolkit/time/DateTimeStampTest.java | 7 ++ .../gctoolkit/parser/GCLogParser.java | 8 +- .../gctoolkit/parser/jvm/Decorators.java | 22 +++++- .../gctoolkit/parser/jvm/UnifiedDiarizer.java | 3 +- .../parser/unified/UnifiedLoggingTokens.java | 2 + .../gctoolkit/parser/patterns/ParserTest.java | 2 +- .../parser/unittests/ParserTest.java | 11 ++- 13 files changed, 148 insertions(+), 25 deletions(-) create mode 100644 IT/src/test/java/com/microsoft/gctoolkit/integration/query/MissingTimeStampTest.java diff --git a/IT/src/test/java/com/microsoft/gctoolkit/integration/query/MissingTimeStampTest.java b/IT/src/test/java/com/microsoft/gctoolkit/integration/query/MissingTimeStampTest.java new file mode 100644 index 00000000..34f1cc24 --- /dev/null +++ b/IT/src/test/java/com/microsoft/gctoolkit/integration/query/MissingTimeStampTest.java @@ -0,0 +1,74 @@ +package com.microsoft.gctoolkit.integration.query; + +import com.microsoft.gctoolkit.GCToolKit; +import com.microsoft.gctoolkit.integration.aggregation.PauseTimeSummary; +import com.microsoft.gctoolkit.integration.io.TestLogFile; +import com.microsoft.gctoolkit.io.GCLogFile; +import com.microsoft.gctoolkit.io.SingleGCLogFile; +import com.microsoft.gctoolkit.jvm.JavaVirtualMachine; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.fail; + +@Tag("modulePath") +public class MissingTimeStampTest { + @Test + public void testMain() { + Path path = new TestLogFile("unified/g1gc/gc-no-age-timestamp.log").getFile().toPath(); + analyze(path.toString()); + } + + public void analyze(String gcLogFile) { + /** + * GC log files can come in one of two types: single or series of rolling logs. + * In this sample, we load a single log file. + * The log files can be either in text, zip, or gzip format. + */ + GCLogFile logFile = new SingleGCLogFile(Path.of(gcLogFile)); + GCToolKit gcToolKit = new GCToolKit(); + + /** + * This call will load all implementations of Aggregator that have been declared in module-info.java. + * This mechanism makes use of Module SPI. + */ + gcToolKit.loadAggregationsFromServiceLoader(); + + /** + * The JavaVirtualMachine contains the aggregations as filled out by the Aggregators. + * It also contains configuration information about how the JVM was configured for the runtime. + */ + JavaVirtualMachine machine = null; + try { + machine = gcToolKit.analyze(logFile); + } catch (IOException e) { + fail(e.getMessage()); + } + + // Look at getPercentSpentInGc() + double pausePercent; + try { + pausePercent = machine.getAggregation(PauseTimeSummary.class).get().getPercentPaused(); + } catch(Throwable t) { + fail("getPercentPaused failed", t); + } + + double pauseTimePercentage = 0.0d; + + try { + pauseTimePercentage = machine.getAggregation(PauseTimeSummary.class).get().getPercentPaused(); + } catch (Throwable t) { + fail("pauseTimePercentage failed", t); + } + + System.out.println("-----------------------------------------------"); + System.out.println("Pause time percentage = " + pauseTimePercentage); + System.out.println("-----------------------------------------------"); + Assertions.assertTrue(pauseTimePercentage > 0.0d); + + } +} diff --git a/api/src/main/java/com/microsoft/gctoolkit/aggregator/Aggregation.java b/api/src/main/java/com/microsoft/gctoolkit/aggregator/Aggregation.java index a979dad2..f8fbb40e 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/aggregator/Aggregation.java +++ b/api/src/main/java/com/microsoft/gctoolkit/aggregator/Aggregation.java @@ -89,7 +89,7 @@ public abstract class Aggregation { private DateTimeStamp timeOfFirstEvent = null; - private DateTimeStamp timeOfTermination = new DateTimeStamp(0.0d); + private DateTimeStamp timeOfTermination = DateTimeStamp.baseDate(); /** * Constructor for the module SPI diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java index e592fefe..e2bb6c8e 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java @@ -57,6 +57,7 @@ public String getSegmentName() { private DateTimeStamp ageOfJVMAtLogStart() { if (startTime == null) { startTime = stream() + .filter(s -> ! s.contains(" file created ")) .map(DateTimeStamp::fromGCLogLine) .filter(dateTimeStamp -> dateTimeStamp.hasTimeStamp() || dateTimeStamp.hasDateStamp()) .findFirst() @@ -74,7 +75,7 @@ private DateTimeStamp ageOfJVMAtLogEnd() { .map(DateTimeStamp::fromGCLogLine) .filter(dateTimeStamp -> dateTimeStamp.hasTimeStamp() || dateTimeStamp.hasDateStamp()) .max(Comparator.comparing(dateTimeStamp -> dateTimeStamp != null ? dateTimeStamp.getTimeStamp() : 0)) - .orElse(new DateTimeStamp(-1.0d)); + .orElse(DateTimeStamp.EMPTY_DATE); } return endTime; } diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java index 4f82b338..d5215160 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/RotatingLogFileMetadata.java @@ -137,9 +137,7 @@ private void findSegments() { private void orderSegments() { - if (segments.size() < 2) { - return; - } + if (segments.size() < 2) return; LinkedList orderedList = new LinkedList<>(); List workingList = new ArrayList<>(); @@ -151,7 +149,7 @@ private void orderSegments() { .filter( segment -> segment.getSegmentName().endsWith(basePattern) || segment.getSegmentName().endsWith(".current")) .findFirst().get(); - orderedList.addFirst(current); + orderedList.addLast(current); workingList = removeIneligibleSegments (workingList, current); while ( ! workingList.isEmpty()) { current = workingList.stream() @@ -160,7 +158,6 @@ private void orderSegments() { orderedList.addFirst(current); workingList = removeIneligibleSegments (workingList, current); } - segments = orderedList; } diff --git a/api/src/main/java/com/microsoft/gctoolkit/jvm/Diary.java b/api/src/main/java/com/microsoft/gctoolkit/jvm/Diary.java index 94bc392e..d3f1fc20 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/jvm/Diary.java +++ b/api/src/main/java/com/microsoft/gctoolkit/jvm/Diary.java @@ -386,7 +386,13 @@ public void setTimeOfFirstEvent(DateTimeStamp startTime) { public DateTimeStamp getTimeOfFirstEvent() { return this.timeOfFirstEvent; } -/* GENERATIONAL, + + public boolean hasTimeOfFirstEvent() { + return this.timeOfFirstEvent != null; + } + +/* + GENERATIONAL, CMS, G1GC, SHENANDOAH, @@ -394,7 +400,6 @@ public DateTimeStamp getTimeOfFirstEvent() { SAFEPOINT, SURVIVOR, TENURED; - */ private void evaluate(Set events, SupportedFlags flag, EventSource eventSource) { if ( isStateKnown(flag) & isTrue(flag)) diff --git a/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java b/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java index 9de76776..ba06899b 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java +++ b/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java @@ -2,6 +2,8 @@ // Licensed under the MIT License. package com.microsoft.gctoolkit.time; +import java.time.Instant; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.chrono.ChronoZonedDateTime; import java.time.format.DateTimeFormatter; @@ -36,8 +38,9 @@ public class DateTimeStamp implements Comparable { // Timestamp can never be less than 0 // - use NaN to say it's not set public final static double TIMESTAMP_NOT_SET = Double.NaN; + public final static ZonedDateTime EPOC = ZonedDateTime.ofInstant(Instant.EPOCH, ZoneId.of("GMT")); private final ZonedDateTime dateTime; - private double timeStamp; + private final double timeStamp; public static final Comparator comparator = getComparator(); // For some reason, ISO_DATE_TIME doesn't like that time-zone is -0100. It wants -01:00. @@ -62,8 +65,8 @@ private static double ageFromString(String doubleFormat) { private static final String TIME = INTEGER + DECIMAL_POINT + "\\d{3}"; // Unified Tokens - private static final String DATE_TAG = "\\[" + DATE + "\\]"; - private static final String UPTIME_TAG = "\\[(" + TIME + ")s\\]"; + private static final String DATE_TAG = "\\[" + DATE + "]"; + private static final String UPTIME_TAG = "\\[(" + TIME + ")s]"; // Pre-unified tokens private static final String TIMESTAMP = "(" + TIME + "): "; @@ -74,7 +77,7 @@ private static double ageFromString(String doubleFormat) { private static final Pattern PREUNIFIED_DATE_TIMESTAMP = Pattern.compile(DATE_TIMESTAMP); // JEP 158 has ISO-8601 time and uptime in seconds and milliseconds as the first two decorators. private static final Pattern UNIFIED_DATE_TIMESTAMP = Pattern.compile("^(" + DATE_TAG + ")?(" + UPTIME_TAG + ")?"); - private static final DateTimeStamp EMPTY_DATE = new DateTimeStamp(TIMESTAMP_NOT_SET); + public static final DateTimeStamp EMPTY_DATE = new DateTimeStamp(EPOC,TIMESTAMP_NOT_SET); public static DateTimeStamp fromGCLogLine(String line) { Matcher matcher; @@ -91,6 +94,14 @@ public static DateTimeStamp fromGCLogLine(String line) { return EMPTY_DATE; } + /** + * Provides a minimal date. + * @return + */ + public static DateTimeStamp baseDate() { + return new DateTimeStamp(EPOC, 0.0d); + } + /** * Create a DateTimeStamp by parsing an ISO 8601 date/time string. * @param iso8601DateTime A String in ISO 8601 format. @@ -167,7 +178,7 @@ public ZonedDateTime getDateTime() { * @return {@code true} if the the date stamp is not {@code null}. */ public boolean hasDateStamp() { - return getDateTime() != null; + return ! (getDateTime() == null || EPOC.equals(getDateTime())); } public boolean hasTimeStamp() { @@ -328,7 +339,7 @@ private static Comparator compareDateTimeStamp(DateTimeStamp o1, else if (o1.hasDateStamp() && o2.hasDateStamp()) return comparing(DateTimeStamp::getDateTime, ChronoZonedDateTime::compareTo); else - throw new IllegalStateException("DateTimeStamp parameters cannot be compared as either timestamp or datestamp must be set."); + throw new IllegalStateException("DateTimeStamp parameters cannot be compared as either timestamp or datestamp must be set in both instances."); } public double toEpochInMillis() { diff --git a/api/src/test/java/com/microsoft/gctoolkit/time/DateTimeStampTest.java b/api/src/test/java/com/microsoft/gctoolkit/time/DateTimeStampTest.java index 3b6d6a88..f93d3a55 100644 --- a/api/src/test/java/com/microsoft/gctoolkit/time/DateTimeStampTest.java +++ b/api/src/test/java/com/microsoft/gctoolkit/time/DateTimeStampTest.java @@ -476,4 +476,11 @@ void compareToTransitivityWithEqualTimeStamps() { assertEquals(comp1To2, comp1To3, "compareTo() is not transitive"); } + @Test + void workWithMalformedDateTimeStamps() { + final DateTimeStamp onlyDate = new DateTimeStamp("2021-09-01T11:12:13.111-0100"); + final DateTimeStamp onlyTime = new DateTimeStamp(1.0D); + assertThrows(IllegalStateException.class, () -> onlyDate.after(onlyTime)); + } + } \ No newline at end of file diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java index b3e95066..cf051a58 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java @@ -26,15 +26,21 @@ public abstract class GCLogParser implements DataSourceParser, SharedPatterns { public static final GCParseRule GCID_COUNTER = new GCParseRule("GCID_COUNTER", " GC\\((\\d+)\\) "); private JVMEventChannel consumer; protected Diary diary; - private DateTimeStamp clock = new DateTimeStamp(0.0d); + private DateTimeStamp clock = new DateTimeStamp(DateTimeStamp.EPOC,0.0d); private double lastDuration = 0.0d; public GCLogParser() {} + /** + * Setting the diary will set the current clock time to the time of the first event + * in the GC log. This is a better choice than EPOC though EPOC is better than null. + * @param diary summary the GC log. + */ @Override public void diary(Diary diary) { this.diary = diary; + this.clock = diary.getTimeOfFirstEvent(); } public DateTimeStamp getClock() { diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/Decorators.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/Decorators.java index 36fb0053..955db7d3 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/Decorators.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/Decorators.java @@ -31,13 +31,27 @@ public class Decorators { * level -- The level associated with the log message * tags -- The tag-set associated with the log message * - * This implementation takes advantage of the property that the ordering of tags is stable. - * The default decorators (as of the time of authoring JDK 9-17) include [uptime][level][tags]. + * This implementation takes advantage of the property that the ordering of tags is stable. For example, + * -Xlog:*::pid,time produces the identical ordering as -Xlog:*::time,pid. Moreover, the list of decorators + * is dedupped implying that the second decorator in -Xlog:*::time,time is ignored. + * + * The default decorators (as of the time of authoring JDK 9-21) include [uptime][level][tags]. * For example, [1.361s][info][gc,heap]. This reads uptime of 1.361 seconds. The tag for the log record * is gc.heap at the info level. * - * At the moment, GCToolkit does not differentiate between timenanos and uptimenanos as there is no formatting hint - * to allow for a differentiation. Less importantly, there is currently no way to differentiate between pid and tid. + * At issue is that uptime and timemillis are, on the surface, indistinguishable. The same is true with + * timenanos and uptimenanos as well as with pid and tid. The following logic can be used to help differentiate + * indistinguishable decorators. + * + * 1) If both decorators are present then order can be used to differentiate + * 2) If ms time value - 20 years > 0, then timemillis can be assumed. Otherwise uptime is assumed. The reasoning + * is, unified logging didn't exist that long ago. The value of 20 has been arbitrarily chosen. + * 3) There is no reliable way to differentiate the nanosecond timings if only 1 is present. However this may not + * matter as GCToolKit would only use these values to create a baseline measure to align date/time with uptime. + * This technique is designed to work-around the cases where logs do not contain the uptime decorator. + * 4) There is no known way to reliably differentiate between PID and TID. At this time, GCToolKit ignores these + * decorators. + * * Todo: GCToolkit captures time in the DateTimeStamp class. That class will capture uptime or time or both. If both * are missing, GCToolkit JVMEvents will have no sense of time. It is possible that the other timing fields could fill * in cases where both the time and uptime decorators were missing. diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/UnifiedDiarizer.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/UnifiedDiarizer.java index c64c5cad..1f0c4639 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/UnifiedDiarizer.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/UnifiedDiarizer.java @@ -148,7 +148,8 @@ else if (decorators.tagsContain("gc") && line.contains("Trigger")) } private void timeOfFirstEvent(Decorators decorator) { - diary.setTimeOfFirstEvent(decorator.getDateTimeStamp()); + if ( ! diary.hasTimeOfFirstEvent()) + diary.setTimeOfFirstEvent(decorator.getDateTimeStamp()); } private void extractTagsAndLevels(Decorators decorators) { diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedLoggingTokens.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedLoggingTokens.java index 93614ab9..59b008fd 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedLoggingTokens.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedLoggingTokens.java @@ -22,6 +22,8 @@ public interface UnifiedLoggingTokens extends GenericTokens { level -- The level associated with the log message tags -- The tag-set associated with the log message [2018-04-04T09:10:00.586-0100][0.018s][1522825800586ms][18ms][10026341461044ns][17738937ns][1375][7427][info][gc] Using G1 + + Note: Decorators are outputed in the order of the list above. */ String DATE_STAMP = "\\[" + DATE + "\\]"; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/ParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/ParserTest.java index ddcc6705..425ff178 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/ParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/ParserTest.java @@ -29,7 +29,6 @@ public void setUp() { channel = new ParserTestSupportChannel(); diarizer = diarizer(); parser = parser(); - parser.diary(diarizer.getDiary()); parser.publishTo(channel); } @@ -53,6 +52,7 @@ public void setUp() { */ protected List feedParser(String[] lines) { Arrays.stream(lines).forEach(diarizer::diarize); + parser.diary(diarizer.getDiary()); Arrays.stream(lines).map(String::trim).forEach(parser::receive); return channel.events(); } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java index aaea5b3d..d12b18f5 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java @@ -114,6 +114,10 @@ private Diarizer getJVMConfiguration(GCLogFile gcLogFile) { } return null; } + + TestResults executeParsing(GCLogFile logfile) { + return null; + } TestResults testGenerationalRotatingLogFile(Path path) throws IOException { GCLogFile logfile = loadLogFile(path, true); @@ -139,11 +143,11 @@ TestResults testGenerationalSingleLogFile(Path path) throws IOException { TestResults testUnifiedG1GCSingleFile(Path path) throws IOException { SingleGCLogFile logfile = new SingleGCLogFile(path); - UnifiedDiarizer unifiedJVMConfiguration = new UnifiedDiarizer(); + Diarizer jvmConfiguration = getJVMConfiguration(logfile); UnifiedG1GCParser parser = new UnifiedG1GCParser(); - TestResults testResults = new TestResults(); + TestResults testResults = new TestResults(); // dick sites google/dec light weight monitoring system ---> graphical idiams for looking for locks and lock contention parser.publishTo(testResults); - parser.diary(unifiedJVMConfiguration.getDiary()); + parser.diary(jvmConfiguration.getDiary()); logfile.stream().map(String::trim).forEach(parser::receive); return testResults; } @@ -151,6 +155,7 @@ TestResults testUnifiedG1GCSingleFile(Path path) throws IOException { TestResults testRegionalRotatingLogFile(Path path) throws IOException { GCLogFile logfile = loadLogFile(path, true); Diarizer jvmConfiguration = getJVMConfiguration(logfile); + logfile.stream().map(String::trim).forEach(jvmConfiguration::diarize); PreUnifiedG1GCParser parser = new PreUnifiedG1GCParser(); TestResults testResults = new TestResults(); parser.publishTo(testResults); From 9fa0e34d33b38a8714f5ba86cffec6d27749c470 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 22 Feb 2024 13:54:15 -0800 Subject: [PATCH 05/34] refactor: update pom for new GC log test data bundle --- gclogs/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gclogs/pom.xml b/gclogs/pom.xml index e3fe0ffa..095d300f 100644 --- a/gclogs/pom.xml +++ b/gclogs/pom.xml @@ -17,7 +17,7 @@ pom - 1.0.7 + 1.0.8 From 1e9f8971b003d858c283b0df2f64fbb87febf740 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 22 Feb 2024 17:22:39 -0800 Subject: [PATCH 06/34] tidy: address comments from PR --- .../integration/query/MissingTimeStampTest.java | 10 +++------- .../com/microsoft/gctoolkit/time/DateTimeStamp.java | 2 +- .../microsoft/gctoolkit/time/DateTimeStampTest.java | 2 +- .../gctoolkit/parser/unittests/ParserTest.java | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/IT/src/test/java/com/microsoft/gctoolkit/integration/query/MissingTimeStampTest.java b/IT/src/test/java/com/microsoft/gctoolkit/integration/query/MissingTimeStampTest.java index 34f1cc24..2161b691 100644 --- a/IT/src/test/java/com/microsoft/gctoolkit/integration/query/MissingTimeStampTest.java +++ b/IT/src/test/java/com/microsoft/gctoolkit/integration/query/MissingTimeStampTest.java @@ -24,7 +24,7 @@ public void testMain() { } public void analyze(String gcLogFile) { - /** + /* * GC log files can come in one of two types: single or series of rolling logs. * In this sample, we load a single log file. * The log files can be either in text, zip, or gzip format. @@ -32,13 +32,13 @@ public void analyze(String gcLogFile) { GCLogFile logFile = new SingleGCLogFile(Path.of(gcLogFile)); GCToolKit gcToolKit = new GCToolKit(); - /** + /* * This call will load all implementations of Aggregator that have been declared in module-info.java. * This mechanism makes use of Module SPI. */ gcToolKit.loadAggregationsFromServiceLoader(); - /** + /* * The JavaVirtualMachine contains the aggregations as filled out by the Aggregators. * It also contains configuration information about how the JVM was configured for the runtime. */ @@ -64,10 +64,6 @@ public void analyze(String gcLogFile) { } catch (Throwable t) { fail("pauseTimePercentage failed", t); } - - System.out.println("-----------------------------------------------"); - System.out.println("Pause time percentage = " + pauseTimePercentage); - System.out.println("-----------------------------------------------"); Assertions.assertTrue(pauseTimePercentage > 0.0d); } diff --git a/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java b/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java index ba06899b..1ff53ac5 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java +++ b/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java @@ -77,7 +77,7 @@ private static double ageFromString(String doubleFormat) { private static final Pattern PREUNIFIED_DATE_TIMESTAMP = Pattern.compile(DATE_TIMESTAMP); // JEP 158 has ISO-8601 time and uptime in seconds and milliseconds as the first two decorators. private static final Pattern UNIFIED_DATE_TIMESTAMP = Pattern.compile("^(" + DATE_TAG + ")?(" + UPTIME_TAG + ")?"); - public static final DateTimeStamp EMPTY_DATE = new DateTimeStamp(EPOC,TIMESTAMP_NOT_SET); + public static final DateTimeStamp EMPTY_DATE = new DateTimeStamp(EPOC, TIMESTAMP_NOT_SET); public static DateTimeStamp fromGCLogLine(String line) { Matcher matcher; diff --git a/api/src/test/java/com/microsoft/gctoolkit/time/DateTimeStampTest.java b/api/src/test/java/com/microsoft/gctoolkit/time/DateTimeStampTest.java index f93d3a55..7aedde5e 100644 --- a/api/src/test/java/com/microsoft/gctoolkit/time/DateTimeStampTest.java +++ b/api/src/test/java/com/microsoft/gctoolkit/time/DateTimeStampTest.java @@ -477,7 +477,7 @@ void compareToTransitivityWithEqualTimeStamps() { } @Test - void workWithMalformedDateTimeStamps() { + void malformedDateTimeStampThrowsException() { final DateTimeStamp onlyDate = new DateTimeStamp("2021-09-01T11:12:13.111-0100"); final DateTimeStamp onlyTime = new DateTimeStamp(1.0D); assertThrows(IllegalStateException.class, () -> onlyDate.after(onlyTime)); diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java index d12b18f5..84ad7878 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java @@ -145,7 +145,7 @@ TestResults testUnifiedG1GCSingleFile(Path path) throws IOException { SingleGCLogFile logfile = new SingleGCLogFile(path); Diarizer jvmConfiguration = getJVMConfiguration(logfile); UnifiedG1GCParser parser = new UnifiedG1GCParser(); - TestResults testResults = new TestResults(); // dick sites google/dec light weight monitoring system ---> graphical idiams for looking for locks and lock contention + TestResults testResults = new TestResults(); parser.publishTo(testResults); parser.diary(jvmConfiguration.getDiary()); logfile.stream().map(String::trim).forEach(parser::receive); From 59dc331bd1fc3b84f9b0b9941bf99c07dd502864 Mon Sep 17 00:00:00 2001 From: Martijn Verburg Date: Fri, 23 Feb 2024 14:30:10 +1300 Subject: [PATCH 07/34] Update parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java --- .../main/java/com/microsoft/gctoolkit/parser/GCLogParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java index cf051a58..f6223a84 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java @@ -26,7 +26,7 @@ public abstract class GCLogParser implements DataSourceParser, SharedPatterns { public static final GCParseRule GCID_COUNTER = new GCParseRule("GCID_COUNTER", " GC\\((\\d+)\\) "); private JVMEventChannel consumer; protected Diary diary; - private DateTimeStamp clock = new DateTimeStamp(DateTimeStamp.EPOC,0.0d); + private DateTimeStamp clock = new DateTimeStamp(DateTimeStamp.EPOC, 0.0d); private double lastDuration = 0.0d; From bed6ae64ba6464b5b821e80deb4037dbd5dd8f5e Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Mon, 26 Feb 2024 19:12:54 -0800 Subject: [PATCH 08/34] feature: first push to combine CMS parser with GenerationalHeapParser --- .../aggregation/CMSCycleAggregation.java | 38 +++++ .../aggregation/CMSCycleAggregator.java | 39 +++++ IT/src/main/java/module-info.java | 3 +- .../gctoolkit/integration/CMSEventsTest.java | 59 +++++++ .../parser/CMSTenuredPoolParser.java | 1 - .../parser/GenerationalHeapParser.java | 158 ++++++++++++++---- 6 files changed, 267 insertions(+), 31 deletions(-) create mode 100644 IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregation.java create mode 100644 IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java create mode 100644 IT/src/test/java/com/microsoft/gctoolkit/integration/CMSEventsTest.java diff --git a/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregation.java b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregation.java new file mode 100644 index 00000000..ba9b9991 --- /dev/null +++ b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregation.java @@ -0,0 +1,38 @@ +package com.microsoft.gctoolkit.integration.aggregation; + +import com.microsoft.gctoolkit.aggregator.Aggregation; +import com.microsoft.gctoolkit.aggregator.Collates; + +@Collates(CMSCycleAggregator.class) +public class CMSCycleAggregation extends Aggregation { + + private int initialMark = 0; + private int remark = 0; + private int concurrentEvent = 0; + + public void initialMark() { + initialMark++; + } + + public void remark() { + remark++; + } + + public void concurrentEvent() { + concurrentEvent++; + } + + public int getInitialMark() { return initialMark; } + public int getRemark() { return remark; } + public int getConcurrentEvent() { return concurrentEvent; } + + @Override + public boolean hasWarning() { + return false; + } + + @Override + public boolean isEmpty() { + return false; + } +} diff --git a/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java new file mode 100644 index 00000000..da46308e --- /dev/null +++ b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java @@ -0,0 +1,39 @@ +package com.microsoft.gctoolkit.integration.aggregation; + +import com.microsoft.gctoolkit.aggregator.Aggregates; +import com.microsoft.gctoolkit.aggregator.Aggregator; +import com.microsoft.gctoolkit.aggregator.EventSource; +import com.microsoft.gctoolkit.event.generational.CMSConcurrentEvent; +import com.microsoft.gctoolkit.event.generational.CMSRemark; +import com.microsoft.gctoolkit.event.generational.InitialMark; + +@Aggregates({EventSource.GENERATIONAL,EventSource.CMS_PREUNIFIED}) +public class CMSCycleAggregator extends Aggregator { + + private InitialMark lastInitialMark = null; + private CMSRemark lastRemark = null; + public CMSCycleAggregator(CMSCycleAggregation results) { + super(results); + System.out.println("CMSCycleAggregator"); + register(InitialMark.class, this::count); + register(CMSRemark.class, this::count); + register(CMSConcurrentEvent.class, this::count); + } + + public void count(InitialMark event) { + if ( event.equals(lastInitialMark)) return; + lastInitialMark = event; + aggregation().initialMark(); + } + + public void count(CMSRemark event) { + System.out.println(event.toString()); + if ( event.equals(lastRemark)) return; + lastRemark = event; + aggregation().remark(); + } + + public void count(CMSConcurrentEvent event) { + aggregation().concurrentEvent(); + } +} diff --git a/IT/src/main/java/module-info.java b/IT/src/main/java/module-info.java index c0c115e6..20cd6df0 100644 --- a/IT/src/main/java/module-info.java +++ b/IT/src/main/java/module-info.java @@ -17,5 +17,6 @@ com.microsoft.gctoolkit.integration.aggregation.PauseTimeSummary, com.microsoft.gctoolkit.integration.aggregation.CollectionCycleCountsSummary, com.microsoft.gctoolkit.integration.shared.OneRuntimeReport, - com.microsoft.gctoolkit.integration.shared.TwoRuntimeReport; + com.microsoft.gctoolkit.integration.shared.TwoRuntimeReport, + com.microsoft.gctoolkit.integration.aggregation.CMSCycleAggregation; } \ No newline at end of file diff --git a/IT/src/test/java/com/microsoft/gctoolkit/integration/CMSEventsTest.java b/IT/src/test/java/com/microsoft/gctoolkit/integration/CMSEventsTest.java new file mode 100644 index 00000000..6a8f6227 --- /dev/null +++ b/IT/src/test/java/com/microsoft/gctoolkit/integration/CMSEventsTest.java @@ -0,0 +1,59 @@ +package com.microsoft.gctoolkit.integration; + +import com.microsoft.gctoolkit.GCToolKit; +import com.microsoft.gctoolkit.integration.aggregation.CMSCycleAggregation; +import com.microsoft.gctoolkit.integration.io.TestLogFile; +import com.microsoft.gctoolkit.io.GCLogFile; +import com.microsoft.gctoolkit.io.SingleGCLogFile; +import com.microsoft.gctoolkit.jvm.JavaVirtualMachine; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.fail; + +@Tag("modulePath") +public class CMSEventsTest { + @Test + public void testMain() { + Path path = new TestLogFile("preunified/cms/parnew/details/scavangeBeforeRemarkWithReference.log").getFile().toPath(); + analyze(path.toString()); + } + + public void analyze(String gcLogFile) { + /** + * GC log files can come in one of two types: single or series of rolling logs. + * In this sample, we load a single log file. + * The log files can be either in text, zip, or gzip format. + */ + GCLogFile logFile = new SingleGCLogFile(Path.of(gcLogFile)); + GCToolKit gcToolKit = new GCToolKit(); + + /** + * This call will load all implementations of Aggregator that have been declared in module-info.java. + * This mechanism makes use of Module SPI. + */ + gcToolKit.loadAggregationsFromServiceLoader(); + + /** + * The JavaVirtualMachine contains the aggregations as filled out by the Aggregators. + * It also contains configuration information about how the JVM was configured for the runtime. + */ + JavaVirtualMachine machine = null; + try { + machine = gcToolKit.analyze(logFile); + } catch (IOException e) { + fail(e.getMessage()); + } + + machine.getAggregation(CMSCycleAggregation.class).ifPresent(cmsCycleCounts -> { + Assertions.assertEquals( 1, cmsCycleCounts.getInitialMark(), "Initial Mark events count"); + //Assertions.assertEquals( 1, cmsCycleCounts.getRemark(), "Remark events count"); + //Assertions.assertEquals( 4, cmsCycleCounts.getConcurrentEvent(), "concurrent phase events count"); + }); + + } +} diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java index f2be1bcb..241d8660 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java @@ -64,7 +64,6 @@ else if ((trace = SPLIT_REMARK.parse(line)) != null) else if ((trace = EndOfFile.parse(line)) != null) { super.publish(ChannelName.CMS_TENURED_POOL_PARSER_OUTBOX, new JVMTermination(getClock(), diary.getTimeOfFirstEvent())); } - } /** diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java index b47ab579..d9570b15 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java @@ -9,12 +9,19 @@ import com.microsoft.gctoolkit.event.GarbageCollectionTypes; import com.microsoft.gctoolkit.event.MemoryPoolSummary; import com.microsoft.gctoolkit.event.ReferenceGCSummary; +import com.microsoft.gctoolkit.event.generational.AbortablePreClean; import com.microsoft.gctoolkit.event.generational.BinaryTreeDictionary; +import com.microsoft.gctoolkit.event.generational.CMSConcurrentEvent; import com.microsoft.gctoolkit.event.generational.CMSRemark; +import com.microsoft.gctoolkit.event.generational.ConcurrentMark; import com.microsoft.gctoolkit.event.generational.ConcurrentModeFailure; import com.microsoft.gctoolkit.event.generational.ConcurrentModeInterrupted; +import com.microsoft.gctoolkit.event.generational.ConcurrentPreClean; +import com.microsoft.gctoolkit.event.generational.ConcurrentReset; +import com.microsoft.gctoolkit.event.generational.ConcurrentSweep; import com.microsoft.gctoolkit.event.generational.DefNew; import com.microsoft.gctoolkit.event.generational.FullGC; +import com.microsoft.gctoolkit.event.generational.GenerationalGCPauseEvent; import com.microsoft.gctoolkit.event.generational.InitialMark; import com.microsoft.gctoolkit.event.generational.PSFullGC; import com.microsoft.gctoolkit.event.generational.PSYoungGen; @@ -80,6 +87,7 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim //Expect Remark private boolean expectRemark = false; + private boolean inConcurrentPhase = false; /* Rules not used.... GCParseRule FULL_PARNEW_CMF_META @@ -147,7 +155,6 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim parseRules.put(CORRUPTED_PARNEW_BODY, this::corruptedParNewBody); parseRules.put(CONCURRENT_PHASE_START, this::concurrentPhaseStart); parseRules.put(CONCURRENT_PHASE_END, this::concurrentPhaseEnd); - parseRules.put(ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE, this::abortPrecleanDueToTimeClause); parseRules.put(INITIAL_MARK, this::initialMark); parseRules.put(SCAVENGE_BEFORE_REMARK, this::scavengeBeforeRemark); parseRules.put(SCAVENGE_BEFORE_REMARK_TENURING, this::scavengeBeforeRemarkTenuring); @@ -163,10 +170,8 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim parseRules.put(PARALLEL_RESCAN_WEAK_CLASS_SCRUB, this::remarkAt13); //, 0.1127040 secs]220.624: [weak refs processing, 0.1513820 secs] [1 CMS-remark: 10541305K(16777216K)] 10742883K(18664704K), 0.7371020 secs] //todo: this was capturing records that is shouldn't have so the rule was modified.. now does it work??? Needs through testing now that order of evaluation will change - parseRules.put(REMARK_DETAILS, this::remarkAt1); parseRules.put(SERIAL_REMARK_SCAN_BREAKDOWNS, this::remarkAt15); parseRules.put(REMARK_DETAILS, this::remarkAt1); - parseRules.put(SERIAL_REMARK_SCAN_BREAKDOWNS, this::remarkAt15); parseRules.put(REMARK_REFERENCE_PROCESSING, this::recordRemarkWithReferenceProcessing); parseRules.put(TENURING_DETAILS, this::tenuringDetails); parseRules.put(RESCAN_WEAK_CLASS_SYMBOL_STRING, this::remarkAt11); @@ -177,12 +182,12 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim parseRules.put(PARNEW_DETAILS_PROMOTION_FAILED_WITH_CMS_PHASE, this::parNewDetailsPromotionFailedWithConcurrentMarkSweepPhase); parseRules.put(PARNEW_DETAILS_WITH_CONCURRENT_MODE_FAILURE, this::parNewDetailsWithConcurrentModeFailure); parseRules.put(CONCURRENT_MODE_FAILURE_REFERENCE, this::concurrentModeFailureReference); - parseRules.put(iCMS_PARNEW_DEFNEW_TENURING_DETAILS, this::iCMSParNewDefNewTenuringDetails); parseRules.put(iCMS_CONCURRENT_MODE_FAILURE, this::iCMSConcurrentModeFailure); parseRules.put(iCMS_CONCURRENT_MODE_FAILURE_META, this::iCMSConcurrentModeFailure); parseRules.put(iCMS_CMF_DUIRNG_PARNEW_DEFNEW_DETAILS, this::iCMSConcurrentModeFailureDuringParNewDefNewDetails); parseRules.put(FULL_GC_INTERRUPTS_CONCURRENT_PHASE, this::fullGCInterruptsConcurrentPhase); parseRules.put(FULL_PARNEW_START, this::fullParNewStart); + parseRules.put(CMS_FULL_80, this::psFullReferenceJDK8); parseRules.put(FULL_GC_REFERENCE_CMF, this::fullGCReferenceConcurrentModeFailure); parseRules.put(iCMS_PARNEW, this::iCMSParNew); @@ -267,6 +272,35 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim parseRules.put(CMF_LARGE_BLOCK, this::concurrentModeFailureSplitByLargeBlock); parseRules.put(WEAK_PROCESSING, this::noop); + //this rule must be evaluated before CONCURRENT_PHASE_END_BLOCK + parseRules.put(ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE, this::abortPrecleanDueToTime); + parseRules.put(CONCURRENT_PHASE_START_BLOCK, this::startOfConcurrentPhase); + parseRules.put(CONCURRENT_PHASE_END_BLOCK, this::endOfConcurrentPhase); + parseRules.put(PRECLEAN_REFERENCE, this::endConcurrentPrecleanWithReferenceProcessing); + + /* + //this rule must be evaluated before CONCURRENT_PHASE_END_BLOCK + if ((trace = ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE.parse(line)) != null) +// abortPrecleanDueToTime(trace); + else if ((trace = CONCURRENT_PHASE_START_BLOCK.parse(line)) != null) + startOfConcurrentPhase(trace); + else if ((trace = CONCURRENT_PHASE_END_BLOCK.parse(line)) != null) + endOfConcurrentPhase(trace); + else if ((trace = PRECLEAN_REFERENCE.parse(line)) != null) + endConcurrentPrecleanWithReferenceProcessing(trace); + else if ((trace = INITIAL_MARK.parse(line)) != null) + initialMark(trace); + else if ((trace = REMARK_CLAUSE.parse(line)) != null) + remark(trace, line); + else if ((trace = REMARK_REFERENCE_PROCESSING.parse(line)) != null) + remarkWithReferenceProcessing(trace, line); + else if ((trace = SPLIT_REMARK.parse(line)) != null) + startOfPhase = getClock(); + else if ((trace = EndOfFile.parse(line)) != null) { + super.publish(ChannelName.CMS_TENURED_POOL_PARSER_OUTBOX, new JVMTermination(getClock(), diary.getTimeOfFirstEvent())); + } + */ + parseRules.put(new GCParseRule("END_OF_DATA_SENTINEL", END_OF_DATA_SENTINEL), this::endOfFile); } @@ -338,7 +372,7 @@ private boolean ignoreFrequentButUnwantedEntries(String line) { if (line.startsWith("eden space")) return true; if (line.startsWith("from space")) return true; if (line.startsWith("to space")) return true; - if (line.contains("[0xffff") && line.endsWith("000)")) ; + if (line.contains("[0xffff") && line.endsWith("000)")) return true; if (line.startsWith("Finished ")) return true; @@ -353,7 +387,7 @@ private boolean ignoreFrequentButUnwantedEntries(String line) { } public void endOfFile(GCLogTrace trace, String line) { - publish(new JVMTermination(getClock(),diary.getTimeOfFirstEvent())); + publish(new JVMTermination(getClock(),diary.getTimeOfFirstEvent()), true); } public void defNew(GCLogTrace trace, String line) { @@ -558,7 +592,7 @@ public void parNewToConcurrentModeFailure(GCLogTrace trace, String line) { publish(collection); } - //Very rare occurence where ParNew suffers from a promotion failure during the reset. This is, and isn't a CMF but will be treated as one. + //Very rare occurrence where ParNew suffers from a promotion failure during the reset. This is, and isn't a CMF but will be treated as one. public void parNewToPsudoConcurrentModeFailure(GCLogTrace trace, String line) { ConcurrentModeFailure collection = new ConcurrentModeFailure(scavengeTimeStamp, GCCause.PROMOTION_FAILED, trace.getDoubleGroup(trace.groupCount())); MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1); @@ -784,9 +818,6 @@ public void concurrentPhaseEnd(GCLogTrace trace, String line) { //not interesting to this parser } - public void abortPrecleanDueToTimeClause(GCLogTrace trace, String line) { - } - //12.986: [GC[1 CMS-initial-mark: 33532K(62656K)] 49652K(81280K), 0.0014191 secs] [Times: user=0.00 sys=0.00, real=0.00 secs] //todo: Initial-mark rule maybe too greedy??? public void initialMark(GCLogTrace trace, String line) { @@ -2018,33 +2049,102 @@ private void log(String line) { } + private DateTimeStamp startOfConcurrentPhase; + + //this rule must be evaluated before CONCURRENT_PHASE_END_BLOCK + private void abortPrecleanDueToTime(GCLogTrace trace, String line) { + try { + double cpuTime = trace.getDoubleGroup(4); + double wallClock = trace.getDoubleGroup(5); + publish(new AbortablePreClean(startOfConcurrentPhase, trace.getDateTimeStamp().getTimeStamp() - startOfConcurrentPhase.getTimeStamp(), cpuTime, wallClock, true)); + } catch (Exception e) { + LOGGER.warning("concurrent phase end choked on " + trace); + } + } + + private void startOfConcurrentPhase(GCLogTrace trace, String line) { + startOfConcurrentPhase = trace.getDateTimeStamp(); + } + private void endOfConcurrentPhase(GCLogTrace trace, String line) { + DateTimeStamp endOfPhase = trace.getDateTimeStamp(); + endOfConcurrentPhase(trace, endOfPhase); + } + + private void endConcurrentPrecleanWithReferenceProcessing(GCLogTrace trace, String line) { + try { + publish(new ConcurrentPreClean(startOfConcurrentPhase, trace.getDoubleGroup(14) - startOfConcurrentPhase.getTimeStamp(), trace.getDoubleGroup(16), trace.getDoubleGroup(17))); + } catch (Throwable t) { + LOGGER.warning("concurrent phase choked on " + trace.toString()); + } + } + + private void endOfConcurrentPhase(GCLogTrace trace, DateTimeStamp timeStamp) { + String phase = trace.getGroup(3); + double cpuTime = trace.getDoubleGroup(4); + double wallTime = trace.getDoubleGroup(5); + double duration = timeStamp.getTimeStamp() - startOfConcurrentPhase.getTimeStamp(); + if ("mark".equals(phase)) + publish(new ConcurrentMark(startOfConcurrentPhase, duration, cpuTime, wallTime)); + else if ("preclean".equals(phase)) + publish(new ConcurrentPreClean(startOfConcurrentPhase, duration, cpuTime, wallTime)); + else if ("abortable-preclean".equals(phase)) + publish( new AbortablePreClean(startOfConcurrentPhase, duration, cpuTime, wallTime, false)); + else if ("sweep".equals(phase)) + publish( new ConcurrentSweep(startOfConcurrentPhase, duration, cpuTime, wallTime)); + else if ("reset".equals(phase)) + publish(new ConcurrentReset(startOfConcurrentPhase, duration, cpuTime, wallTime)); + else + LOGGER.warning("concurrent phase choked on " + trace); + } + public void logMissedFirstRecordForEvent(String line) { LOGGER.log(Level.WARNING, "Missing initial record for: {0}", line); } - public void publish(JVMEvent event, boolean clear) { - if (clear) { - garbageCollectionTypeForwardReference = null; - gcCauseForwardReference = GCCause.UNKNOWN_GCCAUSE; - fullGCTimeStamp = null; - scavengeTimeStamp = null; - youngMemoryPoolSummaryForwardReference = null; - tenuredForwardReference = null; - heapForwardReference = null; - scavengeDurationForwardReference = 0.0; - scavengeCPUSummaryForwardReference = null; - referenceGCForwardReference = null; - totalFreeSpaceForwardReference = 0; - maxChunkSizeForwardReference = 0; - numberOfBlocksForwardReference = 0; - averageBlockSizeForwardReference = 0; - treeHeightForwardReference = 0; + private void clear() { + garbageCollectionTypeForwardReference = null; + gcCauseForwardReference = GCCause.UNKNOWN_GCCAUSE; + fullGCTimeStamp = null; + scavengeTimeStamp = null; + youngMemoryPoolSummaryForwardReference = null; + tenuredForwardReference = null; + heapForwardReference = null; + scavengeDurationForwardReference = 0.0; + scavengeCPUSummaryForwardReference = null; + referenceGCForwardReference = null; + totalFreeSpaceForwardReference = 0; + maxChunkSizeForwardReference = 0; + numberOfBlocksForwardReference = 0; + averageBlockSizeForwardReference = 0; + treeHeightForwardReference = 0; + } + + public void publish( JVMEvent event, boolean drain, boolean clear) { + if (drain) { + for( GenerationalGCPauseEvent pauseEvent : queue) + publish(pauseEvent, false); } + publish(event,clear); + } + + public void publish(JVMEvent event, boolean clear) { + if (clear) + clear(); super.publish(ChannelName.GENERATIONAL_HEAP_PARSER_OUTBOX, event); } - public void publish(JVMEvent event) { - this.publish(event, true); + final private ArrayList queue = new ArrayList<>(); + public void publish(GenerationalGCPauseEvent event) { + if ( inConcurrentPhase) { + queue.add(event); + clear(); + } else + this.publish(event, true); + } + + public void publish(CMSConcurrentEvent event) { + inConcurrentPhase = false; + publish( event, true, false); } @Override From 278b7ac4e1c7352baa331cad4271020b97c1539b Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Tue, 27 Feb 2024 12:58:54 -0800 Subject: [PATCH 09/34] depreciate: depreciated the CMS preunified event source --- .../com/microsoft/gctoolkit/aggregator/EventSource.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/src/main/java/com/microsoft/gctoolkit/aggregator/EventSource.java b/api/src/main/java/com/microsoft/gctoolkit/aggregator/EventSource.java index 8360e861..2a0cb55c 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/aggregator/EventSource.java +++ b/api/src/main/java/com/microsoft/gctoolkit/aggregator/EventSource.java @@ -17,6 +17,12 @@ public enum EventSource { */ GENERATIONAL(GENERATIONAL_HEAP_PARSER_OUTBOX), CMS_UNIFIED(GENERATIONAL_HEAP_PARSER_OUTBOX), + + /** + * Events that come from the CMS collection cycles + * @deprecated use the GENERATIONAL event source instead. + */ + @Deprecated(since="3.0.8", forRemoval=true) CMS_PREUNIFIED(CMS_TENURED_POOL_PARSER_OUTBOX), /** * Events come from the G1 collector. From 8012bd0ca3cee8785aab2f32b1168b33ef2d174c Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 29 Feb 2024 17:38:56 -0800 Subject: [PATCH 10/34] refactor: get all tests to pass --- .../aggregation/CMSCycleAggregator.java | 2 +- .../gctoolkit/io/GCLogFileZipSegment.java | 3 +- .../gctoolkit/time/DateTimeStamp.java | 19 ++- .../parser/GenerationalHeapParser.java | 107 ++++++++++----- .../PreUnifiedGenerationalParserTest.java | 31 +++-- .../parser/unittests/CMSParNewParserTest.java | 126 +++++++++--------- .../unittests/ICMSParNewParserTest.java | 48 +++---- .../parser/unittests/ParserTest.java | 6 + .../UnifiedGenerationalParserTest.java | 4 +- 9 files changed, 206 insertions(+), 140 deletions(-) diff --git a/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java index da46308e..a2c12837 100644 --- a/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java +++ b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java @@ -7,7 +7,7 @@ import com.microsoft.gctoolkit.event.generational.CMSRemark; import com.microsoft.gctoolkit.event.generational.InitialMark; -@Aggregates({EventSource.GENERATIONAL,EventSource.CMS_PREUNIFIED}) +@Aggregates({EventSource.GENERATIONAL}) public class CMSCycleAggregator extends Aggregator { private InitialMark lastInitialMark = null; diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java index e2bb6c8e..8ffd0c09 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFileZipSegment.java @@ -54,7 +54,7 @@ public String getSegmentName() { return this.segmentName; } - private DateTimeStamp ageOfJVMAtLogStart() { + private void ageOfJVMAtLogStart() { if (startTime == null) { startTime = stream() .filter(s -> ! s.contains(" file created ")) @@ -63,7 +63,6 @@ private DateTimeStamp ageOfJVMAtLogStart() { .findFirst() .orElse(new DateTimeStamp(-1.0d)); } - return startTime; } private DateTimeStamp ageOfJVMAtLogEnd() { diff --git a/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java b/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java index 1ff53ac5..210a0770 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java +++ b/api/src/main/java/com/microsoft/gctoolkit/time/DateTimeStamp.java @@ -30,7 +30,7 @@ public class DateTimeStamp implements Comparable { // In the case where we have timestamps, the epoch is start of JVM // In the case where we only have date stamps, the epoch is 1970:01:01:00:00:00.000::UTC+0 // All calculations in GCToolKit make use of the double, timeStamp. - // Calculations are based on an startup Epoch of 0.000 seconds. This isn't always the case and + // Calculations are based on the startup Epoch of 0.000 seconds. This isn't always the case and // certainly isn't the case when only date stamp is present. In these cases, start time is estimated. // This is surprisingly difficult to do thus use of timestamp is highly recommended. @@ -61,7 +61,7 @@ private static double ageFromString(String doubleFormat) { private static final String DECIMAL_POINT = "(?:\\.|,)"; private static final String INTEGER = "\\d+"; - private static final String DATE = "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[\\+|\\-]\\d{4}"; + private static final String DATE = "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[+|-]\\d{4}"; private static final String TIME = INTEGER + DECIMAL_POINT + "\\d{3}"; // Unified Tokens @@ -96,7 +96,7 @@ public static DateTimeStamp fromGCLogLine(String line) { /** * Provides a minimal date. - * @return + * @return a minimal date */ public static DateTimeStamp baseDate() { return new DateTimeStamp(EPOC, 0.0d); @@ -158,12 +158,23 @@ public DateTimeStamp(ZonedDateTime dateTime, double timeStamp) { * Return the time stamp value. Allows a consistent time stamp be available to all calculations. * @return The time stamp value, in decimal seconds. */ + @Deprecated public double getTimeStamp() { if (!hasTimeStamp()) return toEpochInMillis(); return timeStamp; } + public double toMilliseconds() { + if (!hasTimeStamp()) + return toEpochInMillis(); + return timeStamp * 1000.0d; + } + + public double toSeconds() { + return toMilliseconds() / 1000.0d; + } + /** * Return the date stamp. * @return The date stamp, which may be {@code null} @@ -175,7 +186,7 @@ public ZonedDateTime getDateTime() { /** * Return {@code true} if the date stamp is not {@code null}. * It is possible to have two DateTimeStamps from the same GC log, one with a DateStamp and one without. - * @return {@code true} if the the date stamp is not {@code null}. + * @return {@code true} if the date stamp is not {@code null}. */ public boolean hasDateStamp() { return ! (getDateTime() == null || EPOC.equals(getDateTime())); diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java index d9570b15..a44ba43c 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java @@ -569,6 +569,8 @@ private void noop(GCLogTrace trace, String line) { //public static final ParseRule PARNEW_CONCURRENT_MODE_END = new ParseRule (GC_PREFIX + PARNEW_BLOCK + TIMESTAMP + "\\[CMS" + CMS_PHASE_END + "(?: " + CPU_BREAKDOWN + ")?$"); //public static final String CMS_PHASE_END = DATE_TIMESTAMP + "\\[CMS-concurrent-(.+): " + CPU_WALLCLOCK + "\\]"; public void parNewConcurrentModeEnd(GCLogTrace trace, String line) { + trace.notYetImplemented(); + concurrentPhaseEnd(trace,line,14); garbageCollectionTypeForwardReference = GarbageCollectionTypes.ParNew; scavengeTimeStamp = getClock(); gcCauseForwardReference = trace.gcCause(); @@ -698,6 +700,8 @@ public void parNewPromotionFailedTenuring(GCLogTrace trace, String line) { * @param line The GC log line being parsed */ public void parNewPromotionFailedInConcurrentMarkSweepPhase(GCLogTrace trace, String line) { + trace.notYetImplemented(); + concurrentPhaseEnd(trace,line,12); int offset = (trace.groupCount() == 16) ? 2 : 0; scavengeTimeStamp = getClock(); garbageCollectionTypeForwardReference = GarbageCollectionTypes.ParNewPromotionFailed; @@ -811,11 +815,43 @@ public void corruptedParNewBody(GCLogTrace trace, String line) { } public void concurrentPhaseStart(GCLogTrace trace, String line) { - //not interesting to this parser + trace.notYetImplemented(); + startOfConcurrentPhase = trace.getDateTimeStamp(); + inConcurrentPhase = true; } public void concurrentPhaseEnd(GCLogTrace trace, String line) { - //not interesting to this parser + concurrentPhaseEnd(trace, line, 0); + } + + public void concurrentPhaseEnd(GCLogTrace trace, String line, int offset) { + trace.notYetImplemented(); + try { + double cpuTime = trace.getDoubleGroup(4 + offset); + double wallClock = trace.getDoubleGroup(5 + offset); + switch (trace.getGroup(3 + offset)) { + case "mark": + publish(new ConcurrentMark(startOfConcurrentPhase, wallClock, cpuTime, wallClock)); + break; + case "preclean" : + publish(new ConcurrentPreClean(startOfConcurrentPhase, wallClock, cpuTime, wallClock)); + break; + case "abortable-preclean" : + publish(new AbortablePreClean(startOfConcurrentPhase, wallClock, cpuTime, wallClock, cpuTime > 0.0d)); + break; + case "sweep" : + publish(new ConcurrentSweep(startOfConcurrentPhase, wallClock, cpuTime, wallClock)); + break; + case "reset" : + publish(new ConcurrentReset(startOfConcurrentPhase, wallClock, cpuTime, wallClock)); + break; + default: + LOGGER.warning("concurrent phase not recognized end statement -> " + trace); + } + + } catch (Exception e) { + LOGGER.warning("concurrent phase end throws " + e.getMessage() + " for " + trace); + } } //12.986: [GC[1 CMS-initial-mark: 33532K(62656K)] 49652K(81280K), 0.0014191 secs] [Times: user=0.00 sys=0.00, real=0.00 secs] @@ -2012,39 +2048,39 @@ private MemoryPoolSummary extractPermspaceRecord(GCLogTrace trace) { */ private void log(String line) { - if (CONCURRENT_PHASE_START.parse(line) != null) return; - if (CONCURRENT_PHASE_END.parse(line) != null) return; - if (ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE.parse(line) != null) return; - if (PRECLEAN_REFERENCE.parse(line) != null) return; - if (line.startsWith("Missed: GC locker: Trying a full collection because scavenge failed")) return; - - if (line.startsWith("PSYoungGen")) return; - if (line.startsWith("eden space")) return; - if (line.startsWith("to")) return; - if (line.startsWith("from")) return; - if (line.startsWith("ParOldGen")) return; - if (line.startsWith("PSOldGen")) return; - if (line.startsWith("space")) return; - if (line.startsWith("object space")) return; - if (line.startsWith("PSPermGen")) return; - if (line.startsWith("{Heap")) return; - if (line.startsWith("}")) return; - if (line.startsWith("Heap")) return; - if (line.startsWith("[Times: user")) return; - if (line.startsWith("par new generation total")) return; - if (line.startsWith("concurrent mark-sweep generation total")) return; - if (line.startsWith("concurrent-mark-sweep perm gen total")) return; +// if (CONCURRENT_PHASE_START.parse(line) != null) return; +// if (CONCURRENT_PHASE_END.parse(line) != null) return; +// if (ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE.parse(line) != null) return; +// if (PRECLEAN_REFERENCE.parse(line) != null) return; +// if (line.startsWith("Missed: GC locker: Trying a full collection because scavenge failed")) return; +// +// if (line.startsWith("PSYoungGen")) return; +// if (line.startsWith("eden space")) return; +// if (line.startsWith("to")) return; +// if (line.startsWith("from")) return; +// if (line.startsWith("ParOldGen")) return; +// if (line.startsWith("PSOldGen")) return; +// if (line.startsWith("space")) return; +// if (line.startsWith("object space")) return; +// if (line.startsWith("PSPermGen")) return; +// if (line.startsWith("{Heap")) return; +// if (line.startsWith("}")) return; +// if (line.startsWith("Heap")) return; +// if (line.startsWith("[Times: user")) return; +// if (line.startsWith("par new generation total")) return; +// if (line.startsWith("concurrent mark-sweep generation total")) return; +// if (line.startsWith("concurrent-mark-sweep perm gen total")) return; if (line.startsWith("(cardTable: ")) return; - if (line.contains("CMS-concurrent-abortable-preclean")) return; - if (line.contains("committed")) return; - if (line.startsWith("def new generation total")) return; - if (line.startsWith("Before GC:")) return; - if (line.startsWith("After GC:")) return; - if (line.contains("GC log file created")) return; - if (line.contains("GC log file has reached the maximum size")) return; - if (line.contains("Large block")) return; - - GCToolKit.LOG_DEBUG_MESSAGE(() -> "GenerationalHeapParser missed: " + line); +// if (line.contains("CMS-concurrent-abortable-preclean")) return; +// if (line.contains("committed")) return; +// if (line.startsWith("def new generation total")) return; +// if (line.startsWith("Before GC:")) return; +// if (line.startsWith("After GC:")) return; +// if (line.contains("GC log file created")) return; +// if (line.contains("GC log file has reached the maximum size")) return; +// if (line.contains("Large block")) return; +// +// GCToolKit.LOG_DEBUG_MESSAGE(() -> "GenerationalHeapParser missed: " + line); LOGGER.log(Level.WARNING, "Missed: {0}", line); } @@ -2056,7 +2092,7 @@ private void abortPrecleanDueToTime(GCLogTrace trace, String line) { try { double cpuTime = trace.getDoubleGroup(4); double wallClock = trace.getDoubleGroup(5); - publish(new AbortablePreClean(startOfConcurrentPhase, trace.getDateTimeStamp().getTimeStamp() - startOfConcurrentPhase.getTimeStamp(), cpuTime, wallClock, true)); + publish(new AbortablePreClean(startOfConcurrentPhase, wallClock, cpuTime, wallClock, true)); } catch (Exception e) { LOGGER.warning("concurrent phase end choked on " + trace); } @@ -2123,6 +2159,7 @@ public void publish( JVMEvent event, boolean drain, boolean clear) { if (drain) { for( GenerationalGCPauseEvent pauseEvent : queue) publish(pauseEvent, false); + queue.clear(); } publish(event,clear); } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java index 1f8b65b5..9de0aab8 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java @@ -3,8 +3,15 @@ package com.microsoft.gctoolkit.parser.patterns; import com.microsoft.gctoolkit.event.GCCause; +import com.microsoft.gctoolkit.event.generational.AbortablePreClean; +import com.microsoft.gctoolkit.event.generational.CMSConcurrentEvent; +import com.microsoft.gctoolkit.event.generational.CMSPhase; import com.microsoft.gctoolkit.event.generational.CMSRemark; +import com.microsoft.gctoolkit.event.generational.ConcurrentMark; import com.microsoft.gctoolkit.event.generational.ConcurrentModeFailure; +import com.microsoft.gctoolkit.event.generational.ConcurrentPreClean; +import com.microsoft.gctoolkit.event.generational.ConcurrentReset; +import com.microsoft.gctoolkit.event.generational.ConcurrentSweep; import com.microsoft.gctoolkit.event.generational.DefNew; import com.microsoft.gctoolkit.event.generational.FullGC; import com.microsoft.gctoolkit.event.generational.InitialMark; @@ -121,8 +128,8 @@ public void testConcurrentModeFailure() { List jvmEvents = feedParser(lines); InitialMark initialMark = (InitialMark) jvmEvents.get(0); - ParNewPromotionFailed parNewPromotionFailed = (ParNewPromotionFailed) jvmEvents.get(1); - ConcurrentModeFailure concurrentModeFailure = (ConcurrentModeFailure) jvmEvents.get(2); + ParNewPromotionFailed parNewPromotionFailed = (ParNewPromotionFailed) jvmEvents.get(3); + //ConcurrentModeFailure concurrentModeFailure = (ConcurrentModeFailure) jvmEvents.get(4); } @Test @@ -183,11 +190,19 @@ public void test70_40CMSDetailsCause() { assertEquals(0.0082790, parNew.getDuration()); InitialMark initialMark = (InitialMark) jvmEvents.get(1); - assertSame(initialMark.getGCCause(), GCCause.CMS_INITIAL_MARK); - - CMSRemark cmsRemark = (CMSRemark) jvmEvents.get(2); - assertSame(cmsRemark.getGCCause(), GCCause.CMS_FINAL_REMARK); - assertEquals(0.0699640, cmsRemark.getDuration()); + assertEquals(0.014794d, initialMark.getDuration()); + ConcurrentMark concurrentPhase = (ConcurrentMark) jvmEvents.get(2); + assertEquals(0.005d, concurrentPhase.getDuration()); + ConcurrentPreClean preClean = (ConcurrentPreClean) jvmEvents.get(3); + assertEquals(0.0d, preClean.getDuration()); + AbortablePreClean abortablePreClean = (AbortablePreClean) jvmEvents.get(4); + assertEquals(0.349d, abortablePreClean.getDuration()); + CMSRemark cmsRemark = (CMSRemark) jvmEvents.get(5); + assertEquals(0.069964d, cmsRemark.getDuration()); + ConcurrentSweep concurrentSweep = (ConcurrentSweep) jvmEvents.get(6); + assertEquals(0.001d, concurrentSweep.getDuration()); + ConcurrentReset reset = (ConcurrentReset) jvmEvents.get(7); + assertEquals(0.005d, reset.getDuration()); } @Test @@ -220,7 +235,7 @@ public void test70CMSDetailsNoCauseDateStamps() { InitialMark initialMark = (InitialMark) jvmEvents.get(1); assertEquals(0.1976100, initialMark.getDuration()); - CMSRemark cmsRemark = (CMSRemark) jvmEvents.get(2); + CMSRemark cmsRemark = (CMSRemark) jvmEvents.get(5); assertEquals(0.6306470, cmsRemark.getDuration()); } } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java index 06433a10..bc727931 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java @@ -62,7 +62,7 @@ public void testForDetails() { "gc_www1_2013_10_19.log", "gc.2011-09-19.141115.log", "GCLog_CMSCollection.log", - "lead_up_to_problem_gc_www1_2013_10_19.log", + "lead_up_to_problem_gc_www1_2013_10_19.log", // todo: audit "LS_gc.log.20121116_193624", "sample.leak.workaround.log", "TC_oy579c2n5.gc.out", @@ -70,30 +70,29 @@ public void testForDetails() { }; private static final int[] detailsCountsNumberOfDifferentCollectors = { - 6, - 5, + 10, + 9, 4, + 9, + 8, 4, - 3, - 4, - 3, - 5, - 3, + 7, + 10, + 8, 4 - }; private static final int[][] detailsCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {0, 0, 3668, 12, 0, 96, 0, 0, 22, 0, 0, 156, 68}, //todo: these numbers don't seem to add up - {0, 0, 2021, 1, 0, 8, 0, 0, 0, 0, 0, 71, 70}, + {0, 0, 3668, 12, 0, 200, 0, 0, 22, 0, 0, 156, 68}, //todo: these numbers don't seem to add up + {0, 0, 2021, 1, 0, 79, 0, 0, 0, 0, 0, 71, 70}, {0, 0, 24981, 0, 0, 0, 0, 0, 0, 0, 2, 334, 333}, - {0, 0, 2762, 0, 0, 0, 0, 0, 3, 0, 0, 50, 49}, - {0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1}, + {0, 0, 2762, 0, 0, 50, 0, 0, 3, 0, 0, 50, 49}, + {0, 0, 713, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1}, {0, 0, 2160, 0, 0, 0, 0, 0, 0, 0, 1, 228, 228}, - {0, 0, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1}, - {0, 0, 57762, 0, 0, 0, 0, 0, 1, 0, 3, 371, 371}, - {0, 0, 20958, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4}, + {0, 0, 1969, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1}, + {0, 0, 57762, 0, 0, 371, 0, 0, 1, 0, 3, 371, 371}, + {0, 0, 20958, 0, 0, 4, 0, 0, 0, 0, 0, 4, 4}, {0, 0, 22821, 0, 0, 0, 0, 0, 0, 0, 1, 106, 105} }; @@ -119,15 +118,15 @@ public void testForDetailsGCCause170() { }; private static final int[] detailsGCCause170CountsNumberOfDifferentCollectors = { - 3, - 3, + 8, + 8, }; private static final int[][] detailsGCCause170Counts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 - {0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 13, 13}, - {0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 13, 13}, + {0, 0, 603, 0, 0, 13, 0, 0, 0, 0, 0, 13, 13}, + {0, 0, 603, 0, 0, 13, 0, 0, 0, 0, 0, 13, 13}, }; @Test @@ -151,10 +150,10 @@ public void testForDetailsTenuring() { "705_pa1.log", "706_pa2.gc", "707_pa3.gc", - "860_gc.log", + "860_gc.log", //todo: perform an audit of results for this and the next 3 logs. "863_gc.log", "929_def.cms.wd.wt.log", - "BF653AA1-F897-4279-9570-2E4837D311E8-1405428823225.log.gz", + "BF653AA1-F897-4279-9570-2E4837D311E8-1405428823225.log.gz", //todo: missing many statements that should be filtered out "cms-gc-06102014.log.zip", "gc.log", "node1.gc", @@ -168,48 +167,47 @@ public void testForDetailsTenuring() { private static final int[] detailsTenuringCountsNumberOfDifferentCollectors = { 1, - 3, - 6, - 4, - 4, - 4, - 5, 8, - 6, - 3, - 4, - 3, - 4, - 4, - 4, - 6, - 3, - 5, - 3 - + 10, + 7, + 7, + 7, + 9, + 12, + 10, + 8, + 8, + 8, + 9, + 9, + 9, + 10, + 7, + 10, + 8 }; private static final int[][] detailsTenuringCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 {0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {0, 0, 14816, 0, 0, 0, 0, 0, 0, 0, 0, 30, 30}, - {0, 0, 3370, 4, 0, 5, 0, 0, 0, 0, 3, 16, 12}, - {0, 0, 35995, 0, 0, 15, 0, 0, 0, 0, 0, 13676, 13673}, //????? - {0, 0, 30334, 0, 0, 12, 0, 0, 0, 0, 0, 12339, 12336}, //????? - {0, 0, 34285, 0, 0, 14, 0, 0, 0, 0, 0, 13354, 13350}, - {0, 0, 4592, 0, 0, 5, 0, 0, 0, 0, 3, 45, 43}, - {0, 0, 47, 1, 0, 22, 17, 0, 19, 0, 2, 210, 172}, - {0, 0, 3369, 6, 0, 6, 0, 0, 0, 0, 3, 17, 11}, - {0, 0, 5307, 0, 0, 0, 0, 0, 0, 0, 0, 918, 917}, //zip - {0, 0, 1468, 0, 0, 1, 0, 0, 0, 0, 0, 1402, 1402}, //zip not done - {0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 15, 15}, - {0, 0, 46228, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1}, - {0, 0, 47000, 0, 0, 0, 0, 0, 0, 0, 15, 2, 2}, - {0, 0, 48222, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1}, - {0, 0, 3370, 4, 0, 5, 0, 0, 0, 0, 3, 16, 12}, - {0, 0, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1}, - {0, 0, 508, 0, 0, 0, 1, 0, 0, 0, 1395, 5, 4}, - {0, 0, 23829, 0, 0, 0, 0, 0, 0, 0, 0, 978, 977} + {0, 0, 14816, 0, 0, 30, 0, 0, 0, 0, 0, 30, 30}, + {0, 0, 3370, 4, 0, 21, 0, 0, 0, 0, 3, 16, 12}, + {0, 0, 35995, 0, 0, 13688, 0, 0, 0, 0, 0, 13676, 13673}, //????? + {0, 0, 30334, 0, 0, 12348, 0, 0, 0, 0, 0, 12339, 12336}, //????? + {0, 0, 34285, 0, 0, 13364, 0, 0, 0, 0, 0, 13354, 13350}, + {0, 0, 4592, 0, 0, 50, 0, 0, 0, 0, 3, 45, 43}, + {0, 0, 47, 1, 0, 224, 14, 0, 18, 0, 2, 208, 172}, + {0, 0, 3369, 6, 0, 23, 0, 0, 0, 0, 3, 17, 11}, + {0, 0, 5305, 0, 0, 917, 0, 0, 0, 0, 0, 918, 917}, //zip todo: audit + {0, 0, 146, 0, 0, 1402, 0, 0, 0, 0, 0, 1402, 1402}, //zip not done + {0, 0, 4046, 0, 0, 15, 0, 0, 0, 0, 0, 15, 15}, + {0, 0, 46228, 0, 0, 1, 0, 0, 0, 0, 17, 1, 1}, + {0, 0, 47000, 0, 0, 2, 0, 0, 0, 0, 15, 2, 2}, + {0, 0, 48222, 0, 0, 1, 0, 0, 0, 0, 17, 1, 1}, + {0, 0, 3370, 4, 0, 21, 0, 0, 0, 0, 3, 16, 12}, + {0, 0, 1333, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1}, + {0, 0, 508, 0, 0, 5, 1, 0, 0, 0, 1395, 5, 4}, + {0, 0, 23829, 0, 0, 977, 0, 0, 0, 0, 0, 978, 977} }; @Test @@ -232,13 +230,13 @@ public void testForDefNewDetails() { }; private static final int[] defNewDetailsNumberOfDifferentCollectors = { - 3, + 8, }; private static final int[][] defnewDetailsCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 - {0, 19114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26} + {0, 19114, 0, 0, 0, 26, 0, 0, 0, 0, 0, 26, 26} }; @Test @@ -260,13 +258,13 @@ public void testForParNewDetailsTenuringReference80() { }; private static final int[] parNewDetailsTenuringReference180NumberOfDifferentCollectors = { - 6 + 10 }; private static final int[][] parNewDetailsTenuringReference180Counts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {0, 0, 15338, 0, 0, 3, 0, 0, 3, 0, 7, 7, 1} + {0, 0, 15338, 0, 0, 7, 0, 0, 3, 0, 7, 7, 1} }; @@ -290,12 +288,12 @@ public void testForBrokenGCLockerWithTLAB() { }; private static final int[] parNewDetailsTenuringTLABNumberOfDifferentCollectors = { - 3, + 8, }; private static final int[][] parNewDetailsTenuringTLABCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1} + {0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1} }; } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java index bf968fcd..68efc375 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java @@ -45,22 +45,22 @@ public void testForSimpleLogs() { }; private static final int[] detailsCountsNumberOfDifferentCollectors = { - 5, - 3, - 3, - 4, - 7, - 7 + 9, + 8, + 8, + 9, + 11, + 11 }; private static final int[][] detailsCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 12 - {0, 0, 12359, 41, 0, 41, 0, 0, 0, 0, 0, 179, 166}, - {0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 13, 13}, - {0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 11, 11}, - {0, 0, 1374, 0, 0, 0, 1, 0, 0, 0, 0, 3, 2}, - {0, 0, 143550, 75, 0, 306, 0, 0, 7, 0, 1, 10272, 10015}, - {0, 0, 72397, 43, 0, 173, 0, 0, 4, 0, 1, 5137, 4985} + {0, 0, 12359, 41, 0, 219, 0, 0, 0, 0, 0, 179, 166}, + {0, 0, 2564, 0, 0, 13, 0, 0, 0, 0, 0, 13, 13}, + {0, 0, 2564, 0, 0, 11, 0, 0, 0, 0, 0, 11, 11}, + {0, 0, 1374, 0, 0, 2, 1, 0, 0, 0, 0, 3, 2}, + {0, 0, 143543, 75, 0, 10434, 0, 0, 7, 0, 1, 10272, 10015}, + {0, 0, 72395, 43, 0, 5254, 0, 0, 4, 0, 1, 5137, 4985} }; @@ -87,21 +87,21 @@ public void testForDetailsTenuring() { }; private static final int[] detailsTenuringCountsNumberOfDifferentCollectors = { - 5, - 4, - 6, - 3, - 4 + 9, + 9, + 10, + 8, + 9 }; private static final int[][] detailsTenuringCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {0, 0, 67, 0, 0, 1, 0, 0, 1, 0, 0, 25, 24}, - {0, 0, 6244, 0, 0, 0, 0, 0, 0, 0, 1, 2087, 2087}, - {0, 0, 417815, 83, 0, 84, 0, 0, 76, 0, 0, 7840, 7809}, - {0, 0, 32558, 0, 0, 0, 0, 0, 0, 0, 0, 103, 103}, - {0, 0, 18507, 0, 0, 0, 1, 0, 0, 0, 0, 6, 5}, + {0, 0, 67, 0, 0, 26, 0, 0, 1, 0, 0, 25, 24}, + {0, 0, 6240, 0, 0, 2087, 0, 0, 0, 0, 1, 2087, 2087}, + {0, 0, 417811, 83, 0, 7908, 0, 0, 76, 0, 0, 7840, 7809}, + {0, 0, 32439, 0, 0, 103, 0, 0, 0, 0, 0, 103, 103}, + {0, 0, 18507, 0, 0, 5, 1, 0, 0, 0, 0, 6, 5}, }; @@ -124,12 +124,12 @@ public void testForDetailsDebug() { }; private static final int[] detailsDebugCountsNumberOfDifferentCollectors = { - 3 + 8 }; private static final int[][] detailsDebugCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {0, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 533, 532} + {0, 0, 3089, 0, 0, 532, 0, 0, 0, 0, 0, 533, 532} }; } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java index 84ad7878..0ac6f8d5 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java @@ -56,6 +56,12 @@ public abstract class ParserTest { Map.entry(GarbageCollectionTypes.G1GCMixedInitialMark, 3), Map.entry(GarbageCollectionTypes.Full, 4), Map.entry(GarbageCollectionTypes.ConcurrentMark, 5), + + Map.entry(GarbageCollectionTypes.Concurrent_Preclean, 13), + Map.entry(GarbageCollectionTypes.Abortable_Preclean,14), + Map.entry(GarbageCollectionTypes.Concurrent_Sweep, 15), + Map.entry(GarbageCollectionTypes.Concurrent_Reset, 16), + Map.entry(GarbageCollectionTypes.G1GCConcurrentMark, 5), Map.entry(GarbageCollectionTypes.G1GCRemark, 7), diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java index b1594396..f2e6c32c 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java @@ -58,11 +58,11 @@ public void testCMSLogsMissingTimeStamps() { }; private static final int[] cmsNumberOfDifferentCollectors = { - 4, + 8, }; private static final int[][] cmsCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - { 0, 0, 4461, 0, 0, 2, 0, 0, 0, 0, 0, 2, 2}, + { 0, 0, 6167, 0, 0, 2, 0, 0, 0, 0, 0, 2, 2}, }; } From 86e2578822abb2cab5953e2bab294b931c188bd1 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Fri, 1 Mar 2024 11:03:06 -0800 Subject: [PATCH 11/34] refactor: continue to get tests to pass --- .../microsoft/gctoolkit/parser/GenerationalHeapParser.java | 3 +++ .../parser/patterns/PreUnifiedGenerationalParserTest.java | 7 +++++-- .../gctoolkit/parser/unittests/CMSParNewParserTest.java | 6 +++--- .../gctoolkit/parser/unittests/ICMSParNewParserTest.java | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java index a44ba43c..e36f74e6 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java @@ -1043,6 +1043,9 @@ public void parNewDetailsConcurrentModeFailure(GCLogTrace trace, String line) { public void parNewDetailsPromotionFailedWithConcurrentMarkSweepPhase(GCLogTrace trace, String line) { youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1); scavengeDurationForwardReference = trace.getDoubleGroup(7); + GCLogTrace concurrentPhase = CONCURRENT_PHASE_END_BLOCK.parse(line); + if (concurrentPhase != null) + endOfConcurrentPhase(concurrentPhase,line); } //: 1069879K->1069879K(1090560K), 0.3135220 secs]2014-09-19T06:07:23.135+0200: 73512.294: [CMS: 1613084K->823344K(2423488K), 3.5186340 secs] 2639961K->823344K(3514048K), [CMS Perm : 205976K->205949K(343356K)], 3.8323790 secs] [Times: user=4.44 sys=0.00, real=3.83 secs] diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java index 9de0aab8..58aca6cc 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java @@ -128,8 +128,11 @@ public void testConcurrentModeFailure() { List jvmEvents = feedParser(lines); InitialMark initialMark = (InitialMark) jvmEvents.get(0); - ParNewPromotionFailed parNewPromotionFailed = (ParNewPromotionFailed) jvmEvents.get(3); - //ConcurrentModeFailure concurrentModeFailure = (ConcurrentModeFailure) jvmEvents.get(4); + assertEquals(0.0008727d, initialMark.getDuration()); + ParNewPromotionFailed parNewPromotionFailed = (ParNewPromotionFailed) jvmEvents.get(4); + assertEquals(0.0023005d, parNewPromotionFailed.getDuration()); + ConcurrentModeFailure concurrentModeFailure = (ConcurrentModeFailure) jvmEvents.get(5); + assertEquals(0.095695d, concurrentModeFailure.getDuration()); } @Test diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java index bc727931..b8f15153 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java @@ -192,9 +192,9 @@ public void testForDetailsTenuring() { {0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, {0, 0, 14816, 0, 0, 30, 0, 0, 0, 0, 0, 30, 30}, {0, 0, 3370, 4, 0, 21, 0, 0, 0, 0, 3, 16, 12}, - {0, 0, 35995, 0, 0, 13688, 0, 0, 0, 0, 0, 13676, 13673}, //????? - {0, 0, 30334, 0, 0, 12348, 0, 0, 0, 0, 0, 12339, 12336}, //????? - {0, 0, 34285, 0, 0, 13364, 0, 0, 0, 0, 0, 13354, 13350}, + {0, 0, 35995, 0, 0, 13691, 0, 0, 0, 0, 0, 13676, 13673}, //????? + {0, 0, 30334, 0, 0, 12350, 0, 0, 0, 0, 0, 12339, 12336}, //????? + {0, 0, 34285, 0, 0, 13368, 0, 0, 0, 0, 0, 13354, 13350}, {0, 0, 4592, 0, 0, 50, 0, 0, 0, 0, 3, 45, 43}, {0, 0, 47, 1, 0, 224, 14, 0, 18, 0, 2, 208, 172}, {0, 0, 3369, 6, 0, 23, 0, 0, 0, 0, 3, 17, 11}, diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java index 68efc375..e6cd446b 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java @@ -99,7 +99,7 @@ public void testForDetailsTenuring() { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 {0, 0, 67, 0, 0, 26, 0, 0, 1, 0, 0, 25, 24}, {0, 0, 6240, 0, 0, 2087, 0, 0, 0, 0, 1, 2087, 2087}, - {0, 0, 417811, 83, 0, 7908, 0, 0, 76, 0, 0, 7840, 7809}, + {0, 0, 417811, 83, 0, 7924, 0, 0, 76, 0, 0, 7840, 7809}, {0, 0, 32439, 0, 0, 103, 0, 0, 0, 0, 0, 103, 103}, {0, 0, 18507, 0, 0, 5, 1, 0, 0, 0, 0, 6, 5}, }; From 82e8f4f704c8e8129aebbe293c3fb4c7f6de866d Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Fri, 1 Mar 2024 11:13:27 -0800 Subject: [PATCH 12/34] refactor: continue to get tests to pass --- .../parser/GenerationalHeapParser.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java index e36f74e6..3a45c2e5 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java @@ -524,7 +524,7 @@ private void parNewFLSTime(GCLogTrace trace, String line) { fullGc.add(heap); publish(fullGc); } else - trace.notYetImplemented(); + LOGGER.warning("Unable to parse -> " + trace); } private void parNewFLSHeader(GCLogTrace trace, String line) { @@ -569,7 +569,6 @@ private void noop(GCLogTrace trace, String line) { //public static final ParseRule PARNEW_CONCURRENT_MODE_END = new ParseRule (GC_PREFIX + PARNEW_BLOCK + TIMESTAMP + "\\[CMS" + CMS_PHASE_END + "(?: " + CPU_BREAKDOWN + ")?$"); //public static final String CMS_PHASE_END = DATE_TIMESTAMP + "\\[CMS-concurrent-(.+): " + CPU_WALLCLOCK + "\\]"; public void parNewConcurrentModeEnd(GCLogTrace trace, String line) { - trace.notYetImplemented(); concurrentPhaseEnd(trace,line,14); garbageCollectionTypeForwardReference = GarbageCollectionTypes.ParNew; scavengeTimeStamp = getClock(); @@ -580,7 +579,7 @@ public void parNewConcurrentModeEnd(GCLogTrace trace, String line) { } public void parNewCardTable(GCLogTrace trace, String line) { - trace.notYetImplemented(); + LOGGER.warning("Not Yet Implemented -> " + trace); } @@ -700,7 +699,6 @@ public void parNewPromotionFailedTenuring(GCLogTrace trace, String line) { * @param line The GC log line being parsed */ public void parNewPromotionFailedInConcurrentMarkSweepPhase(GCLogTrace trace, String line) { - trace.notYetImplemented(); concurrentPhaseEnd(trace,line,12); int offset = (trace.groupCount() == 16) ? 2 : 0; scavengeTimeStamp = getClock(); @@ -815,7 +813,6 @@ public void corruptedParNewBody(GCLogTrace trace, String line) { } public void concurrentPhaseStart(GCLogTrace trace, String line) { - trace.notYetImplemented(); startOfConcurrentPhase = trace.getDateTimeStamp(); inConcurrentPhase = true; } @@ -825,7 +822,6 @@ public void concurrentPhaseEnd(GCLogTrace trace, String line) { } public void concurrentPhaseEnd(GCLogTrace trace, String line, int offset) { - trace.notYetImplemented(); try { double cpuTime = trace.getDoubleGroup(4 + offset); double wallClock = trace.getDoubleGroup(5 + offset); @@ -1004,7 +1000,7 @@ public void concurrentModeFailureDetails(GCLogTrace trace, String line) { if (youngMemoryPoolSummaryForwardReference == null) youngMemoryPoolSummaryForwardReference = heapSummary.minus(tenuredPoolSummary); } else { - trace.notYetImplemented(); + LOGGER.warning("Unable to parse -> " + trace); return; } @@ -1161,7 +1157,7 @@ public void iCMSConcurrentModeFailure(GCLogTrace trace, String line) { failure.add(extractCPUSummary(line)); publish(failure); } else { - trace.notYetImplemented(); + LOGGER.warning("Unable to parse -> " + trace); } } @@ -1337,7 +1333,7 @@ private void iCMSFullGC(GCLogTrace trace, String line) { else if ((cause == GCCause.UNKNOWN_GCCAUSE) || (cause == GCCause.GCCAUSE_NOT_SET)) { collection = new FullGC(getClock(), cause, trace.getDoubleGroup(trace.groupCount())); } else { - trace.notYetImplemented(); + LOGGER.warning("Unable to parse -> " + trace); return; } @@ -1869,40 +1865,46 @@ public void psPromotionFailed(GCLogTrace trace, String line) { fullGC.add(cpuSummary); publish(fullGC); } else { - trace.notYetImplemented(); + LOGGER.warning("Unable to parse -> " + trace); } scavengeTimeStamp = getClock(); garbageCollectionTypeForwardReference = GarbageCollectionTypes.DefNew; } public void rescanSplitUnloadingString(GCLogTrace trace, String line) { - trace.notYetImplemented(); + LOGGER.warning("Not Yet Implemented -> " + trace); } public void parNewConcurrentPhaseCards(GCLogTrace trace, String line) { - trace.notYetImplemented(); + LOGGER.warning("Not Yet Implemented -> " + trace); } public void concurrentPhaseYields(GCLogTrace trace, String line) { + LOGGER.warning("Not Yet Implemented -> " + trace); } private void precleanTimedoutWithCards(GCLogTrace trace, String line) { + LOGGER.warning("Not Yet Implemented -> " + trace); } private void shouldCollectConcurrent(GCLogTrace trace, String line) { + LOGGER.warning("Not Yet Implemented -> " + trace); } public void parNewShouldConcurrentCollect(GCLogTrace trace, String line) { - trace.notYetImplemented(); + LOGGER.warning("Not yet Implemented -> " + trace); } private void psYoungAdaptivePolicySizeStart(GCLogTrace trace, String line) { + LOGGER.warning("Not yet Implemented -> " + trace); } private void psYoungAdaptivePolicySizeBody(GCLogTrace trace, String line) { + LOGGER.warning("Not yet Implemented -> " + trace); } private void adaptivePolicySizeBody(GCLogTrace trace, String line) { + LOGGER.warning("Not yet Implemented -> " + trace); } public void remarkSplitByDebug(GCLogTrace trace, String line) { From 115996b5c492d839f7fa08bab6e0b342dea28ad1 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Mon, 11 Mar 2024 08:27:35 -0700 Subject: [PATCH 13/34] refactor: collapse CMS phases parser into generational parser --- .../aggregation/CMSCycleAggregator.java | 1 - .../integration/EndToEndIntegrationTest.java | 1 - .../event/g1gc/G1ConcurrentMark.java | 2 +- .../gctoolkit/parser/CMSPatterns.java | 8 +- .../parser/GenerationalHeapParser.java | 107 ++-- .../gctoolkit/parser/ParallelPatterns.java | 3 + .../parser/jvm/PreUnifiedDiarizer.java | 11 +- .../ConcurrentMarkSweepParserRulesTest.java | 24 +- .../GenerationalHeapParserRulesTest.java | 596 ++++++++++++++++++ .../parser/GenerationalHeapParserTest.java | 95 ++- .../parser/{patterns => }/ParserTest.java | 2 +- .../parser/ShenandoahParserTest.java | 1 - .../gctoolkit/parser/ZGCParserTest.java | 1 - .../PreUnifiedCMSTenuredParserTest.java | 1 + .../PreUnifiedGenerationalParserTest.java | 3 +- .../UnifiedGenerationalEventsTest.java | 1 + .../parser/unittests/CMSParNewParserTest.java | 134 ++-- .../unittests/ICMSParNewParserTest.java | 50 +- .../parser/unittests/ParserTest.java | 15 +- .../UnifiedGenerationalParserTest.java | 4 +- 20 files changed, 882 insertions(+), 178 deletions(-) create mode 100644 parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserRulesTest.java rename parser/src/test/java/com/microsoft/gctoolkit/parser/{patterns => }/ParserTest.java (98%) diff --git a/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java index a2c12837..b37455ef 100644 --- a/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java +++ b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java @@ -14,7 +14,6 @@ public class CMSCycleAggregator extends Aggregator { private CMSRemark lastRemark = null; public CMSCycleAggregator(CMSCycleAggregation results) { super(results); - System.out.println("CMSCycleAggregator"); register(InitialMark.class, this::count); register(CMSRemark.class, this::count); register(CMSConcurrentEvent.class, this::count); diff --git a/IT/src/test/java/com/microsoft/gctoolkit/integration/EndToEndIntegrationTest.java b/IT/src/test/java/com/microsoft/gctoolkit/integration/EndToEndIntegrationTest.java index c94b2904..6737c9d9 100644 --- a/IT/src/test/java/com/microsoft/gctoolkit/integration/EndToEndIntegrationTest.java +++ b/IT/src/test/java/com/microsoft/gctoolkit/integration/EndToEndIntegrationTest.java @@ -62,7 +62,6 @@ public void analyze(String gcLogFile) { .map(HeapOccupancyAfterCollectionSummary::get) .ifPresent(summary -> { summary.forEach((gcType, dataSet) -> { - System.out.printf(message, gcType, dataSet.size()); switch (gcType) { case DefNew: defNewCount = dataSet.size(); diff --git a/api/src/main/java/com/microsoft/gctoolkit/event/g1gc/G1ConcurrentMark.java b/api/src/main/java/com/microsoft/gctoolkit/event/g1gc/G1ConcurrentMark.java index 7ac3c021..9f100804 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/event/g1gc/G1ConcurrentMark.java +++ b/api/src/main/java/com/microsoft/gctoolkit/event/g1gc/G1ConcurrentMark.java @@ -24,7 +24,7 @@ public class G1ConcurrentMark extends G1GCConcurrentEvent { * @param duration duration of the event */ public G1ConcurrentMark(DateTimeStamp timeStamp, double duration) { - super(timeStamp, GarbageCollectionTypes.ConcurrentMark, GCCause.GCCAUSE_NOT_SET, duration); + super(timeStamp, GarbageCollectionTypes.G1GCConcurrentMark, GCCause.GCCAUSE_NOT_SET, duration); } /** diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java index eadef70d..e3ad7b0d 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java @@ -43,10 +43,12 @@ public interface CMSPatterns extends SharedPatterns { //12.986: [GC[1 CMS-initial-mark: 33532K(62656K)] 49652K(81280K), 0.0014191 secs] [Times: user=0.00 sys=0.00, real=0.00 secs] GCParseRule INITIAL_MARK = new GCParseRule("INITIAL_MARK", GC_PREFIX + "\\[1 CMS-initial-mark: " + OCCUPANCY_CONFIGURED + "\\] " + OCCUPANCY_CONFIGURED_PAUSE + "\\]"); GCParseRule CONCURRENT_PHASE_START = new GCParseRule("CONCURRENT_PHASE_START", "^" + CMS_PHASE_START); - GCParseRule CONCURRENT_PHASE_END = new GCParseRule("CONCURRENT_PHASE_END", "^" + CMS_PHASE_END); + + GCParseRule CONCURRENT_PHASE_END = new GCParseRule("CONCURRENT_PHASE_END", "^" + CMS_PHASE_END + "$"); + GCParseRule CONCURRENT_PHASE_END_WITH_CPU_SUMMARY = new GCParseRule("CONCURRENT_PHASE_END", "^" + CMS_PHASE_END + " " + CPU_SUMMARY); GCParseRule CONCURRENT_PHASE_START_BLOCK = new GCParseRule("CONCURRENT_PHASE_START_BLOCK", CMS_PHASE_START); - GCParseRule CONCURRENT_PHASE_END_BLOCK = new GCParseRule("CONCURRENT_PHASE_END_BLOCK", CMS_PHASE_END); + GCParseRule CONCURRENT_PHASE_END_BLOCK = new GCParseRule("CONCURRENT_PHASE_END_BLOCK", CMS_PHASE_END + " " + CPU_SUMMARY); GCParseRule ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE = new GCParseRule("ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE", "^" + ABORT_PRECLEAN_DUE_TO_TIME_BLOCK); /********** Remark statements **********/ @@ -172,7 +174,7 @@ public interface CMSPatterns extends SharedPatterns { GCParseRule SERIAL_FULL = new GCParseRule("SERIAL_FULL", FULL_GC_PREFIX + SERIAL_TENURED_BLOCK + " " + BEFORE_AFTER_CONFIGURED + ", " + PERM_RECORD + ", " + PAUSE_TIME); //1.244: [Full GC (Metadata GC Threshold) 1.244: [Tenured: 11155K->92012K(21888K), 0.0399330 secs] 18738K->09212K(31680K), [Metaspace: 20778K->20778K(1067008K)], 0.0405118 secs] [Times: user=0.04 sys=0.00, real=0.04 secs] - GCParseRule SERIAL_FULL80 = new GCParseRule("SERIAL_FULL80", FULL_GC_PREFIX + SERIAL_TENURED_BLOCK + " " + BEFORE_AFTER_CONFIGURED + ", " + META_RECORD + ", " + PAUSE_TIME); + //GCParseRule SERIAL_FULL80 = new GCParseRule("SERIAL_FULL80", FULL_GC_PREFIX + SERIAL_TENURED_BLOCK + " " + BEFORE_AFTER_CONFIGURED + ", " + META_RECORD + ", " + PAUSE_TIME); //11.675: [GC 11.675: [ParNew: 3782K->402K(3904K), 0.0012156 secs]11.676: [Tenured: 8673K->6751K(8840K), 0.1268332 secs] 12373K->6751K(12744K), [Perm : 10729K->10675K(21248K)], 0.1281985 secs] //89.260: [GC 89.260: [ParNew: 19135K->19135K(19136K), 0.0000156 secs]89.260: [CMS: 105875K->107775K(107776K), 0.5703972 secs] 125011K->116886K(126912K), [CMS Perm : 15589K->15584K(28412K)], 0.5705219 secs] diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java index 3a45c2e5..9caa4daf 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java @@ -88,6 +88,7 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim //Expect Remark private boolean expectRemark = false; private boolean inConcurrentPhase = false; + private boolean abortPrecleanDueToTime = false; /* Rules not used.... GCParseRule FULL_PARNEW_CMF_META @@ -109,7 +110,6 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim parseRules.put(DEFNEW, this::defNew); parseRules.put(DEFNEW_TENURING, this::defNewWithTenuring); parseRules.put(SERIAL_FULL, this::serialFull); - parseRules.put(SERIAL_FULL80, this::serialFull80); parseRules.put(PARNEW, this::parNew); parseRules.put(PARNEW_TENURING, this::parNewWithTenuring); parseRules.put(PARNEW_CONCURRENT_MODE_END, this::parNewConcurrentModeEnd); @@ -155,6 +155,7 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim parseRules.put(CORRUPTED_PARNEW_BODY, this::corruptedParNewBody); parseRules.put(CONCURRENT_PHASE_START, this::concurrentPhaseStart); parseRules.put(CONCURRENT_PHASE_END, this::concurrentPhaseEnd); + parseRules.put(CONCURRENT_PHASE_END_WITH_CPU_SUMMARY, this::concurrentPhaseEnd); parseRules.put(INITIAL_MARK, this::initialMark); parseRules.put(SCAVENGE_BEFORE_REMARK, this::scavengeBeforeRemark); parseRules.put(SCAVENGE_BEFORE_REMARK_TENURING, this::scavengeBeforeRemarkTenuring); @@ -274,34 +275,9 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim //this rule must be evaluated before CONCURRENT_PHASE_END_BLOCK parseRules.put(ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE, this::abortPrecleanDueToTime); - parseRules.put(CONCURRENT_PHASE_START_BLOCK, this::startOfConcurrentPhase); - parseRules.put(CONCURRENT_PHASE_END_BLOCK, this::endOfConcurrentPhase); + parseRules.put(CONCURRENT_PHASE_CARDS, this::abortPreclean); parseRules.put(PRECLEAN_REFERENCE, this::endConcurrentPrecleanWithReferenceProcessing); - - /* - //this rule must be evaluated before CONCURRENT_PHASE_END_BLOCK - if ((trace = ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE.parse(line)) != null) -// abortPrecleanDueToTime(trace); - else if ((trace = CONCURRENT_PHASE_START_BLOCK.parse(line)) != null) - startOfConcurrentPhase(trace); - else if ((trace = CONCURRENT_PHASE_END_BLOCK.parse(line)) != null) - endOfConcurrentPhase(trace); - else if ((trace = PRECLEAN_REFERENCE.parse(line)) != null) - endConcurrentPrecleanWithReferenceProcessing(trace); - else if ((trace = INITIAL_MARK.parse(line)) != null) - initialMark(trace); - else if ((trace = REMARK_CLAUSE.parse(line)) != null) - remark(trace, line); - else if ((trace = REMARK_REFERENCE_PROCESSING.parse(line)) != null) - remarkWithReferenceProcessing(trace, line); - else if ((trace = SPLIT_REMARK.parse(line)) != null) - startOfPhase = getClock(); - else if ((trace = EndOfFile.parse(line)) != null) { - super.publish(ChannelName.CMS_TENURED_POOL_PARSER_OUTBOX, new JVMTermination(getClock(), diary.getTimeOfFirstEvent())); - } - */ - - + parseRules.put( new GCParseRule("FLOATING_CPU_BREAKOUT", "^" + CPU_SUMMARY), this::noop); parseRules.put(new GCParseRule("END_OF_DATA_SENTINEL", END_OF_DATA_SENTINEL), this::endOfFile); } @@ -363,11 +339,13 @@ private boolean ignoreFrequentButUnwantedEntries(String line) { return true; } + if (line.startsWith("Heap")) return true; if (line.startsWith("Heap after")) return true; if (line.startsWith("PSYoungGen")) return true; if (line.startsWith("ParOldGen")) return true; if (line.startsWith("PSOldGen")) return true; if (line.startsWith("PSPermGen")) return true; + if (line.startsWith("concurrent mark-sweep")) return true; if (line.startsWith("object space")) return true; if (line.startsWith("eden space")) return true; if (line.startsWith("from space")) return true; @@ -424,18 +402,6 @@ public void serialFull(GCLogTrace trace, String line) { publish(collection); } - - //3.299: [Full GC (Metadata GC Threshold) 3.299: [Tenured: 21006K->20933K(21888K), 0.0230475 secs] 29003K->20933K(31680K), [Metapace: 12111K->12111K(12672K)], 0.0231557 secs] - public void serialFull80(GCLogTrace trace, String line) { - FullGC collection = new FullGC(getClock(), trace.gcCause(), trace.getDoubleGroup(trace.groupCount())); - MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(5); - MemoryPoolSummary heap = this.getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 12); - collection.add(heap.minus(tenured), tenured, heap); - collection.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); - collection.add(extractCPUSummary(line)); - publish(collection); - } - //62.616: [GC 62.616: [ParNew: 5033216K->129451K(5662336K), 0.2536590 secs] 5097075K->193310K(24536704K), 0.2538510 secs] //48.021: [GC48.021: [ParNew: 306686K->34046K(306688K), 0.3196120 secs] 1341473K->1125818K(8669952K), 0.3197540 secs] public void parNew(GCLogTrace trace, String line) { @@ -812,6 +778,16 @@ public void corruptedParNewBody(GCLogTrace trace, String line) { publish(concurrentModeFailure); } + /* + private void startOfConcurrentPhase(GCLogTrace trace, String line) { + startOfConcurrentPhase = trace.getDateTimeStamp(); + } + private void endOfConcurrentPhase(GCLogTrace trace, String line) { + DateTimeStamp endOfPhase = trace.getDateTimeStamp(); + endOfConcurrentPhase(trace, endOfPhase); + } + */ + public void concurrentPhaseStart(GCLogTrace trace, String line) { startOfConcurrentPhase = trace.getDateTimeStamp(); inConcurrentPhase = true; @@ -1020,7 +996,11 @@ public void parNewDetailsConcurrentModeFailure(GCLogTrace trace, String line) { parNew.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1), summary); parNew.add(extractCPUSummary(line)); publish(parNew, false); - ConcurrentModeFailure collection = new ConcurrentModeFailure(fullGCTimeStamp, gcCauseForwardReference, trace.getDoubleGroup(trace.groupCount())); + if ( inConcurrentPhase) { + // todo: publish closing out concurrent phase + LOGGER.warning("concurrent phase not closed"); + } + ConcurrentModeFailure collection = new ConcurrentModeFailure(fullGCTimeStamp, gcCauseForwardReference, trace.getDuration()); collection.add( new MemoryPoolSummary(trace.toKBytes(3), trace.toKBytes(5), 0L, trace.toKBytes(28) - trace.toKBytes(21)), trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(17), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 24)); @@ -1880,14 +1860,16 @@ public void parNewConcurrentPhaseCards(GCLogTrace trace, String line) { } public void concurrentPhaseYields(GCLogTrace trace, String line) { - LOGGER.warning("Not Yet Implemented -> " + trace); + endOfConcurrentPhase( trace, line); } private void precleanTimedoutWithCards(GCLogTrace trace, String line) { - LOGGER.warning("Not Yet Implemented -> " + trace); + abortPrecleanDueToTime = true; + endOfConcurrentPhase(trace, trace.getDateTimeStamp(), 3); } private void shouldCollectConcurrent(GCLogTrace trace, String line) { + trace.notYetImplemented(); LOGGER.warning("Not Yet Implemented -> " + trace); } @@ -2072,13 +2054,13 @@ private void log(String line) { // if (line.startsWith("}")) return; // if (line.startsWith("Heap")) return; // if (line.startsWith("[Times: user")) return; -// if (line.startsWith("par new generation total")) return; + if (line.startsWith("par new generation total")) return; // if (line.startsWith("concurrent mark-sweep generation total")) return; -// if (line.startsWith("concurrent-mark-sweep perm gen total")) return; + if (line.startsWith("concurrent-mark-sweep perm gen total")) return; if (line.startsWith("(cardTable: ")) return; // if (line.contains("CMS-concurrent-abortable-preclean")) return; // if (line.contains("committed")) return; -// if (line.startsWith("def new generation total")) return; + if (line.startsWith("def new generation total")) return; // if (line.startsWith("Before GC:")) return; // if (line.startsWith("After GC:")) return; // if (line.contains("GC log file created")) return; @@ -2098,6 +2080,18 @@ private void abortPrecleanDueToTime(GCLogTrace trace, String line) { double cpuTime = trace.getDoubleGroup(4); double wallClock = trace.getDoubleGroup(5); publish(new AbortablePreClean(startOfConcurrentPhase, wallClock, cpuTime, wallClock, true)); + abortPrecleanDueToTime = false; + } catch (Exception e) { + LOGGER.warning("concurrent phase end choked on " + trace); + } + } + + private void abortPreclean(GCLogTrace trace, String line) { + try { + double cpuTime = trace.getDoubleGroup(7); + double wallClock = trace.getDoubleGroup(8); + publish(new AbortablePreClean(startOfConcurrentPhase, wallClock, cpuTime, wallClock, false)); + abortPrecleanDueToTime = false; } catch (Exception e) { LOGGER.warning("concurrent phase end choked on " + trace); } @@ -2108,28 +2102,31 @@ private void startOfConcurrentPhase(GCLogTrace trace, String line) { } private void endOfConcurrentPhase(GCLogTrace trace, String line) { DateTimeStamp endOfPhase = trace.getDateTimeStamp(); - endOfConcurrentPhase(trace, endOfPhase); + endOfConcurrentPhase(trace, endOfPhase, 0); } private void endConcurrentPrecleanWithReferenceProcessing(GCLogTrace trace, String line) { + GCLogTrace concurrentBlock = CONCURRENT_PHASE_END_BLOCK.parse(line); try { - publish(new ConcurrentPreClean(startOfConcurrentPhase, trace.getDoubleGroup(14) - startOfConcurrentPhase.getTimeStamp(), trace.getDoubleGroup(16), trace.getDoubleGroup(17))); + publish(new ConcurrentPreClean(startOfConcurrentPhase, concurrentBlock.getDoubleGroup(8), concurrentBlock.getDoubleGroup(4), concurrentBlock.getDoubleGroup(5))); } catch (Throwable t) { LOGGER.warning("concurrent phase choked on " + trace.toString()); } } - private void endOfConcurrentPhase(GCLogTrace trace, DateTimeStamp timeStamp) { - String phase = trace.getGroup(3); - double cpuTime = trace.getDoubleGroup(4); - double wallTime = trace.getDoubleGroup(5); - double duration = timeStamp.getTimeStamp() - startOfConcurrentPhase.getTimeStamp(); + private void endOfConcurrentPhase(GCLogTrace trace, DateTimeStamp timeStamp, int offset) { + String phase = trace.getGroup(3 + offset); + double cpuTime = trace.getDoubleGroup(4 + offset); + double wallTime = trace.getDoubleGroup(5 + offset); + double duration = timeStamp.toSeconds() - startOfConcurrentPhase.toSeconds(); if ("mark".equals(phase)) publish(new ConcurrentMark(startOfConcurrentPhase, duration, cpuTime, wallTime)); else if ("preclean".equals(phase)) publish(new ConcurrentPreClean(startOfConcurrentPhase, duration, cpuTime, wallTime)); - else if ("abortable-preclean".equals(phase)) - publish( new AbortablePreClean(startOfConcurrentPhase, duration, cpuTime, wallTime, false)); + else if ("abortable-preclean".equals(phase)) { + publish(new AbortablePreClean(startOfConcurrentPhase, duration, cpuTime, wallTime, abortPrecleanDueToTime)); + abortPrecleanDueToTime = false; + } else if ("sweep".equals(phase)) publish( new ConcurrentSweep(startOfConcurrentPhase, duration, cpuTime, wallTime)); else if ("reset".equals(phase)) diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/ParallelPatterns.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/ParallelPatterns.java index d088ea24..3014cd6f 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/ParallelPatterns.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/ParallelPatterns.java @@ -25,6 +25,9 @@ public interface ParallelPatterns extends SharedPatterns { //43446.876: [Full GC (System.gc()) 43446.876: [Tenured43447.145: [SoftReference, 347 refs, 0.0000739 secs]43447.146: [WeakReference, 637 refs, 0.0000575 secs]43447.146: [FinalReference, 460 refs, 0.0004814 secs]43447.146: [PhantomReference, 0 refs, 4 refs, 0.0000047 secs]43447.146: [JNI Weak Reference, 0.0000713 secs]: 290960K->292298K(699072K), 0.6541540 secs] 460438K->292298K(1013824K), [Metaspace: 99359K->99359K(1140736K)], 0.6543427 secs] [Times: user=0.64 sys=0.00, real=0.65 secs] //47047.534: [Full GC (System.gc()) 47047.534: [Tenured47047.808: [SoftReference, 258 refs, 0.0000630 secs]47047.808: [WeakReference, 628 refs, 0.0000561 secs]47047.808: [FinalReference, 170 refs, 0.0001564 secs]47047.808: [PhantomReference, 0 refs, 3 refs, 0.0000039 secs]47047.808: [JNI Weak Reference, 0.0000800 secs]: 292298K->278286K(699072K), 0.7450752 secs] 350258K->278286K(1013824K), [Metaspace: 99239K->99239K(1140736K)], 0.7451777 secs] [Times: user=0.74 sys=0.00, real=0.75 secs] GCParseRule PS_FULL_REFERENCE_8 = new GCParseRule("PS_FULL_REFERENCE_8", FULL_GC_PREFIX + " " + DATE_TIMESTAMP + "\\[Tenured" + REFERENCE_RECORDS + ": " + BEFORE_AFTER_CONFIGURED_PAUSE + "\\] " + BEFORE_AFTER_CONFIGURED + ", " + PERM_RECORD + ", " + PAUSE_TIME); + /* + 34.518: [Full GC (System.gc()) 34.518: [Tenured34.525: [SoftReference, 0 refs, 0.0000520 secs]34.525: [WeakReference, 479 refs, 0.0000505 secs]34.525: [FinalReference, 1051 refs, 0.0000654 secs]34.525: [PhantomReference, 0 refs, 1 refs, 0.0000067 secs]34.525: [JNI Weak Reference, 0.0000216 secs]: 3614K->5193K(174784K), 0.0235099 secs] 14881K->5193K(253440K), [Metaspace: 16161K->16161K(1064960K)], 0.0236206 secs] [Times: user=0.02 sys=0.00, real=0.02 secs] + */ GCParseRule SERIAL_FULL_REFERENCE = new GCParseRule("SERIAL_FULL_REFERENCE", FULL_GC_PREFIX + DATE_TIMESTAMP + "\\[Tenured" + REFERENCE_RECORDS + ": " + BEFORE_AFTER_CONFIGURED_PAUSE + "\\] " + BEFORE_AFTER_CONFIGURED + "\\, " + PERM_RECORD + ", " + PAUSE_TIME); GCParseRule PS_FULL_REFERENCE = new GCParseRule("PS_FULL_REFERENCE", FULL_GC_PREFIX + REFERENCE_RECORDS + "\\s*" + PS_BLOCK + " \\[(?:PSFull|PSOldGen|ParOldGen): " + BEFORE_AFTER_CONFIGURED + "\\] " + BEFORE_AFTER_CONFIGURED + "(?:,)? " + PERM_RECORD + ", " + PAUSE_TIME); diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java index 65204cbb..b9398aee 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java @@ -459,14 +459,15 @@ else if (ParallelPatterns.PS_FULL_GC_META.parse(line) != null) { //todo: this rule is in the wrong place else if (CMSPatterns.SERIAL_FULL.parse(line) != null) { collectionCount++; - getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.G1GC); - getDiary().setTrue(SupportedFlags.SERIAL); - - getDiary().setTrue(SupportedFlags.GC_DETAILS); + getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1); + getDiary().setTrue(SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS); //todo: private final int SupportedFlags.GC_CAUSE = 12; getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); if (line.contains("Metaspace")) { getDiary().setFalse(SupportedFlags.JDK70); + getDiary().setTrue(SupportedFlags.JDK80, SupportedFlags.GC_CAUSE); + } else if ( line.contains("Perm")) { // todo: maybe look for GC_CAUSE in JDK 7??? + getDiary().setFalse(SupportedFlags.JDK80); } if (line.contains("Tenured")) { getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS); @@ -475,7 +476,7 @@ else if (CMSPatterns.SERIAL_FULL.parse(line) != null) { getDiary().setFalse(SupportedFlags.SERIAL); getDiary().setTrue(SupportedFlags.CMS); } - } else if (CMSPatterns.SERIAL_FULL80.parse(line) != null) { + } else if (CMSPatterns.SERIAL_FULL.parse(line) != null) { collectionCount++; getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.JDK70); getDiary().setTrue(SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS, SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepParserRulesTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepParserRulesTest.java index 1b336101..363be7c7 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepParserRulesTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepParserRulesTest.java @@ -29,17 +29,16 @@ public void testCMSParseRules() { } } } - assertTrue(true); } /* Code that is useful when testing individual records */ - // @Test + //@Test //@Ignore("Not a real test, only for debugging") public void testDebugCMSParseRule() { int index = rules.length-1; // awesome fix from David.. thanks :-) - //index = 36; + index = 43; GCParseRule rule = rules[index]; evaluate(rule, lines[index][0]); } @@ -99,7 +98,11 @@ private void evaluate(GCParseRule rule, String string) { SPLIT_PARNEW_PROMOTION_FAILED_IN_CMS_PHASE, FULL_SPLIT_BY_CONCURRENT_PHASE, CMF_LARGE_BLOCK, - PRECLEAN_REFERENCE_PAR_NEW_REFERENCE + PRECLEAN_REFERENCE_PAR_NEW_REFERENCE, // 40 + CONC_PHASE_YIELDS, + CONCURRENT_PHASE_START, + CONCURRENT_PHASE_END, + CONCURRENT_PHASE_END_WITH_CPU_SUMMARY }; private String[][] lines = { @@ -233,6 +236,19 @@ private void evaluate(GCParseRule rule, String string) { }, { // 40 "2020-03-27T17:20:09.966+0000: 61372.980: [Preclean SoftReferences, 0.0000064 secs]2020-03-27T17:20:09.966+0000: 61372.980: [Preclean WeakReferences, 0.0010360 secs]2020-03-27T17:20:09.967+0000: 61372.981: [Preclean FinalReferences, 0.0017374 secs]2020-03-27T17:20:09.969+0000: 61372.983: [Preclean PhantomReferences, 0.0000900 secs]2020-03-27T17:20:16.627+0000: 61379.640: [GC (Allocation Failure) 2020-03-27T17:20:16.627+0000: 61379.640: [ParNew2020-03-27T17:20:16.828+0000: 61379.842: [SoftReference, 0 refs, 0.0006072 secs]2020-03-27T17:20:16.829+0000: 61379.842: [WeakReference, 155571 refs, 0.0025794 secs]2020-03-27T17:20:16.831+0000: 61379.845: [FinalReference, 52557 refs, 0.0025628 secs]2020-03-27T17:20:16.834+0000: 61379.847: [PhantomReference, 0 refs, 4 refs, 0.0007787 secs]2020-03-27T17:20:16.835+0000: 61379.848: [JNI Weak Reference, 0.0731165 secs]" + }, + { // 41 + "2012-11-20T00:51:32.585+0100: 7.274: [CMS-concurrent-mark: 0.015/0.015 secs] (CMS-concurrent-mark yielded 0 times)" + }, + { // 42 + "8.325: [CMS-concurrent-sweep-start]" + }, + { // 43 + "8.327: [CMS-concurrent-sweep: 0.002/0.002 secs]" + }, + { // 44 + "8.327: [CMS-concurrent-sweep: 0.002/0.002 secs] [Times: user=0.01 sys=0.00, real=0.00 secs]", + "27.626: [CMS-concurrent-mark: 0.070/0.089 secs] [Times: user=0.14 sys=0.00, real=0.09 secs]" } }; } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserRulesTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserRulesTest.java new file mode 100644 index 00000000..54edf0eb --- /dev/null +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserRulesTest.java @@ -0,0 +1,596 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.parser; + +import org.junit.jupiter.api.Test; + +import java.util.logging.Logger; + +import static com.microsoft.gctoolkit.parser.CommonTestHelper.captureTest; +import static org.junit.jupiter.api.Assertions.*; + +public class GenerationalHeapParserRulesTest implements SimplePatterns, SerialPatterns, ParallelPatterns, CMSPatterns, ICMSPatterns { + + private static final Logger LOGGER = Logger.getLogger(GenerationalHeapParserRulesTest.class.getName()); + + @Test + public void testGenerationalRules() { + assertEquals(rules.length,lines.length); + for (int i = 0; i < rules.length; i++) + for (int j = 0; j < lines.length; j++) { + assertTrue(lines[j].length > 0, "No lines to test for index " + j); + int captured = captureTest(rules[i], lines[j]); + if (i == j) { + assertEquals(captured, lines[j].length, i + " failed to captured it's lines"); + } else { + assertEquals(0, captured, i + " captured " + j); + } + } + } + + /* Code that is useful when testing individual records */ + + // @Test + public void testDebugGenerationalRules() { + int index = 4; + GCParseRule rule = rules[index]; + //evaluate( rule, lines[index][0], debugging); + evaluate(rule, lines[index][1], true); + } + + + private void evaluate(GCParseRule rule, String string, boolean dump) { + + GCLogTrace trace = rule.parse(string); + assertNotNull(trace); + if (dump) { + trace.notYetImplemented(); +// LOGGER.fine("matches groups " + trace.groupCount()); +// for (int i = 0; i <= trace.groupCount(); i++) { +// LOGGER.fine(i + ": " + trace.getGroup(i)); +// } + } + } + + private GCParseRule[] specialCaseRules = { + CONCURRENT_PHASE_START_BLOCK, + CONCURRENT_PHASE_END_BLOCK + }; + + + private GCParseRule[] rules = { + PSYOUNGGEN_NO_DETAILS, + PSFULL, + PS_FULL_BODY_FLOATING, + PSFULL_ADAPTIVE_SIZE, + PS_FULL_REFERENCE, + + SERIAL_FULL_REFERENCE, + PSYOUNGGEN_REFERENCE, + PSYOUNGGEN_PROMOTION_FAILED, + PSYOUNG_ADAPTIVE_SIZE_POLICY, + PSFULL_ERGONOMICS_PHASES, + + PSFULL_REFERENCE_PHASE, + DEFNEW, + DEFNEW_TENURING, + SERIAL_FULL, +// SERIAL_FULL80, + +// PARNEW, +// PARNEW_TENURING, +// PARNEW_CONCURRENT_MODE_END, +// PARNEW_CARDTABLE, +// PARNEW_TO_CMF_PERM, +// +// PARNEW_TO_CMF_META, +// PARNEW_REFERENCE, +// PARNEW_REFERENCE_SPLIT, +// JVMPatterns.TLAB_START, +// PARNEW_REFERENCE_SPLIT_BY_TLAB, +// +// DEFNEW_REFERENCE, +// PARNEW_PROMOTION_FAILED, +// PARNEW_PROMOTION_FAILED_DETAILS, +// PARNEW_PROMOTION_FAILED_REFERENCE, +// FLOATING_REFERENCE, +// +// PARNEW_PROMOTION_FAILED_TENURING, +// PARNEW_PROMOTION_FAILED_IN_CMS_PHASE, +// CMS_BAILING_TO_FOREGROUND, +// PROMOTION_FAILED_TO_FULL, +// PARNEW_PLAB, +// +// PLAB_ENTRY, +// PLAB_SUMMARY, +// FULLGC_FLS_BEFORE, +// PARNEW_FLS_BEFORE, +// PARNEW_FLS_AFTER, +// +// PARNEW_FLS_BODY, +// PARNEW_PROMOTION_FAILED_DETAILS_AFTER, +// PARNEW_FLS_TIME, +// FLS_HEADER, +// FLS_SEPARATOR, +// +// FLS_TOTAL_FREE_SPACE, +// FLS_MAX_CHUNK_SIZE, +// FLS_NUMBER_OF_BLOCKS, +// FLS_AVERAGE_BLOCK_SIZE, +// FLS_TREE_HEIGHT, +// +// FLS_LARGE_BLOCK_PROXIMITY, +// FLS_LARGE_BLOCK, +// PARNEW_PROMOTION_FAILED_TIME_ABORT_PRECLEAN, +// PARNEW_PROMOTION_FAILED_CONCURRENT_PHASE, +// CORRUPTED_PARNEW_CONCURRENT_PHASE, +// +// CORRUPTED_PARNEW_BODY, +// CONCURRENT_PHASE_START, +// CONCURRENT_PHASE_END, +// CONCURRENT_PHASE_END_WITH_CPU_SUMMARY, +// INITIAL_MARK, +// +// SCAVENGE_BEFORE_REMARK, +// SCAVENGE_BEFORE_REMARK_TENURING, +// PARALLEL_REMARK_WEAK_REF, +// PARALLEL_REMARK_CLASS_UNLOADING, +// REMARK_PARNEW_PROMOTION_FAILED, +// +// PARALLEL_REMARK_STRING_SYMBOL, +// PARALLEL_REMARK_WEAK_CLASS_SYMBOL_STRING, +// PARALLEL_REMARK_WEAK_STRING, +// PARALLEL_RESCAN, +// REMARK, +// +// PARALLEL_RESCAN_V2, +// PARALLEL_RESCAN_WEAK_CLASS_SCRUB, +// //, 0.1127040 secs]220.624: [weak refs processing, 0.1513820 secs] [1 CMS-remark: 10541305K(16777216K)] 10742883K(18664704K), 0.7371020 secs] +// //todo: this was capturing records that is shouldn't have so the rule was modified.. now does it work??? Needs through testing now that order of evaluation will change +// SERIAL_REMARK_SCAN_BREAKDOWNS, +// REMARK_DETAILS, +// REMARK_REFERENCE_PROCESSING, +// +// TENURING_DETAILS, +// RESCAN_WEAK_CLASS_SYMBOL_STRING, +// CONCURRENT_MODE_FAILURE_DETAILS, +// CONCURRENT_MODE_FAILURE_DETAILS_META, +// PARNEW_DETAILS_CONCURRENT_MODE_FAILURE_PERM, +// +// PARNEW_DETAILS_CONCURRENT_MODE_FAILURE_META, +// PARNEW_DETAILS_PROMOTION_FAILED_WITH_CMS_PHASE, +// PARNEW_DETAILS_WITH_CONCURRENT_MODE_FAILURE, +// CONCURRENT_MODE_FAILURE_REFERENCE, +// iCMS_CONCURRENT_MODE_FAILURE, +// +// iCMS_CONCURRENT_MODE_FAILURE_META, +// iCMS_CMF_DUIRNG_PARNEW_DEFNEW_DETAILS, +// FULL_GC_INTERRUPTS_CONCURRENT_PHASE, +// FULL_PARNEW_START, +// CMS_FULL_80, +// +// FULL_GC_REFERENCE_CMF, +// iCMS_PARNEW, +// iCMS_PARNEW_PROMOTION_FAILURE_RECORD, +// iCMS_PARNEW_PROMOTION_FAILURE, +// FULL_GC_ICMS, +// +// iCMS_PARNEW_DEFNEW_TENURING_DETAILS, +// iCMS_FULL, +// iCMS_PROMOTION_FAILED, +// iCMS_PROMOTION_FAILED_PERM, +// iCMS_PROMOTION_FAILED_META, +// +// iCMS_MISLABELED_FULL, +// iCMS_FULL_AFTER_CONCURRENT_MODE_FAILURE, +// iCMS_FULL_AFTER_CONCURRENT_MODE_FAILURE_META, +// iCMS_CONCURRENT_MODE_INTERRUPTED, +// PS_FULL_GC_META, +// +// PS_FULL_GC_V2_META, +// CMS_FULL_META, +// FULL_PARNEW_CMF_META, +// FULL_PARNEW_CMF_PERM, +// PARNEW_CONCURRENT_MODE_FAILURE_PERM, +// +// PARNEW_CONCURRENT_MODE_FAILURE_META, +// PS_FULL_GC_V2_PERM, +// PS_FULL_GC_PERM, +// CMS_FULL_PERM, +// CMS_FULL_PERM_META_REFERENCE, +// +// PARNEW_NO_DETAILS, +// YOUNG_NO_DETAILS, +// CMS_NO_DETAILS, +// FULL_NO_GC_DETAILS, +// PARNEW_START, +// +// GC_START, +// YOUNG_SPLIT_NO_DETAILS, +// CMF_SIMPLE, +// DEFNEW_DETAILS, +// PRECLEAN_REFERENCE_PAR_NEW_REFERENCE, +// +// PSYOUNGGEN, +// PSYOUNGGEN_REFERENCE_SPLIT, +// PS_TENURING_START, +// PSFULL_SPLIT, +// PS_FULL_REFERENCE_SPLIT, +// +// PS_DETAILS_WITH_TENURING, +// PS_FAILURE, +// PSOLD_ADAPTIVE_SIZE_POLICY, +// PSYOUNG_DETAILS_FLOATING, +// FULL_REFERENCE_ADAPTIVE_SIZE, +// +// PS_PROMOTION_FAILED, +// RESCAN_SPLIT_UNLOADING_STRING, +// PARNEW_CONCURRENT_PHASE_CARDS, +// CONC_PHASE_YIELDS, +// PRECLEAN_TIMED_OUT_WITH_CARDS, +// +// PARNEW_SHOULD_CONCURRENT_COLLECT, +// SHOULD_CONCURRENT_COLLECT, +// REMARK_SPLIT_BY_DEBUG, +// SCAVENGE_BEFORE_REMARK_PRINT_HEAP_AT_GC, +// SPLIT_REMARK_REFERENCE_BUG, +// +// SPLIT_REMARK_REFERENCE, +// PSYOUNG_ADAPTIVE_SIZE_POLICY_START, +// PS_ADAPTIVE_SIZE_POLICY_BODY, +// ADAPTIVE_SIZE_POLICY_BODY, +// ADAPTIVE_SIZE_POLICY_STOP, +// +// SCAVENGE_BEFORE_REMARK_REFERENCE, +// SCAVENGE_BEFORE_REMARK_REFERENCE_SPLIT, +// PARNEW_TO_CONCURRENT_MODE_FAILURE, +// SPLIT_PARNEW_PROMOTION_FAILED_IN_CMS_PHASE, +// FULL_SPLIT_BY_CONCURRENT_PHASE, +// +// CMF_LARGE_BLOCK, +// //this rule must be evaluated before CONCURRENT_PHASE_END_BLOCK +// ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE, + PRECLEAN_REFERENCE + }; + + private String[][] lines = { + { // 0 + "2017-03-28T12:17:34.744+0200: 1.895: [GC (Allocation Failure) 137969K->21806K(491008K), 0.0082985 secs]", + "2017-03-28T12:17:33.823+0200: 0.974: [GC (System.gc()) 92578K->9947K(491008K), 0.0135426 secs]" // buggy full gc + }, + { // 1 + "2017-03-28T12:17:33.837+0200: 0.988: [Full GC (System.gc()) 9947K->9457K(491008K), 0.0593030 secs]" + }, + { // 2 + "[PSYoungGen: 6149K->0K(305856K)] [PSOldGen: 0K->5876K(699072K)] 6149K->5876K(1004928K) [PSPermGen: 22697K->22697K(262144K)], 0.0654700 secs] [Times: user=0.06 sys=0.00, real=0.06 secs]" + }, + { // 3 + "2015-03-01T11:51:52.629-0500: 138040.983: [Full GC (System)AdaptiveSizeStart: 138072.609 collection: 312" + }, + { // 4 + "12.700: [Full GC (Metadata GC Threshold) 12.720: [SoftReference, 0 refs, 0.0000402 secs]12.720: [WeakReference, 626 refs, 0.0000522 secs]12.720: [FinalReference, 11038 refs, 0.0033807 secs]12.724: [PhantomReference, 0 refs, 0 refs, 0.0000053 secs]12.724: [JNI Weak Reference, 0.0000039 secs][PSYoungGen: 2286K->0K(255488K)] [ParOldGen: 127438K->86892K(175104K)] 129725K->86892K(430592K), [Metaspace: 20902K->20902K(1069056K)], 0.0835405 secs] [Times: user=0.30 sys=0.01, real=0.08 secs]", + "2016-12-12T06:49:17.108-0500: 213439.976: [Full GC (Ergonomics)2016-12-12T06:49:17.247-0500: 213440.115: [SoftReference, 588 refs, 0.0001210 secs]2016-12-12T06:49:17.247-0500: 213440.115: [WeakReference, 2157 refs, 0.0001830 secs]2016-12-12T06:49:17.247-0500: 213440.115: [FinalReference, 11619 refs, 0.0010820 secs]2016-12-12T06:49:17.248-0500: 213440.116: [PhantomReference, 1 refs, 36 refs, 0.0000140 secs]2016-12-12T06:49:17.248-0500: 213440.116: [JNI Weak Reference, 0.0000100 secs] [PSYoungGen: 272896K->272894K(310784K)] [ParOldGen: 699382K->699382K(699392K)] 972278K->972276K(1010176K) [PSPermGen: 125994K->125994K(126464K)], 0.3060010 secs] [Times: user=0.00 sys=0.00, real=0.30 secs]" + }, + { // 5 + "47047.534: [Full GC (System.gc()) 47047.534: [Tenured47047.808: [SoftReference, 258 refs, 0.0000630 secs]47047.808: [WeakReference, 628 refs, 0.0000561 secs]47047.808: [FinalReference, 170 refs, 0.0001564 secs]47047.808: [PhantomReference, 0 refs, 3 refs, 0.0000039 secs]47047.808: [JNI Weak Reference, 0.0000800 secs]: 292298K->278286K(699072K), 0.7450752 secs] 350258K->278286K(1013824K), [Metaspace: 99239K->99239K(1140736K)], 0.7451777 secs] [Times: user=0.74 sys=0.00, real=0.75 secs]", + "127.920: [Full GC127.920: [Tenured127.952: [SoftReference, 0 refs, 0.0000380 secs]127.952: [WeakReference, 444 refs, 0.0000370 secs]127.952: [FinalReference, 2070 refs, 0.0006330 secs]127.953: [PhantomReference, 0 refs, 2 refs, 0.0000040 secs]127.953: [JNI Weak Reference, 0.0000050 secs]: 100366K->91734K(699072K), 0.0936930 secs] 193212K->91734K(1013632K), [Perm : 26687K->26687K(26688K)], 0.0937770 secs] [Times: user=0.10 sys=0.00, real=0.09 secs]", + "756402.356: [Full GC (System.gc()) 756402.356: [Tenured756402.715: [SoftReference, 292 refs, 0.0000726 secs]756402.716: [WeakReference, 808 refs, 0.0000643 secs]756402.716: [FinalReference, 126 refs, 0.0001561 secs]756402.716: [PhantomReference, 0 refs, 2 refs, 0.0000042 secs]756402.716: [JNI Weak Reference, 0.0000796 secs]: 356460K->356882K(699072K), 0.7858180 secs] 391950K->356882K(1013952K), [Metaspace: 111616K->111616K(1153024K)], 0.7859359 secs] [Times: user=0.77 sys=0.00, real=0.79 secs]" + }, + { // 6 + "0.385: [GC (Allocation Failure) 0.388: [SoftReference, 0 refs, 0.0000262 secs]0.388: [WeakReference, 10 refs, 0.0000051 secs]0.388: [FinalReference, 1054 refs, 0.0011154 secs]0.389: [PhantomReference, 0 refs, 0 refs, 0.0000496 secs]0.389: [JNI Weak Reference, 0.0000037 secs][PSYoungGen: 15872K->2532K(18432K)] 15872K->3180K(60928K), 0.0043799 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]" + }, + { // 7 + "2017-08-17T08:14:22.677-0500: 8179.905: [GC (Allocation Failure) --[PSYoungGen: 3990443K->3990443K(4030464K)] 5012990K->5013022K(5079040K), 0.5837002 secs] [Times: user=0.67 sys=0.00, real=0.58 secs] \n" + }, + { // 8 + "2017-08-25T08:49:16.474-0500: 20.296: [GC (Allocation Failure) 2017-08-25T08:49:16.511-0500: 20.333: [SoftReference, 0 refs, 0.0000564 secs]2017-08-25T08:49:16.511-0500: 20.333: [WeakReference, 2902 refs, 0.0002190 secs]2017-08-25T08:49:16.512-0500: 20.334: [FinalReference, 115330 refs, 0.1086704 secs]2017-08-25T08:49:16.620-0500: 20.442: [PhantomReference, 0 refs, 1 refs, 0.0000209 secs]2017-08-25T08:49:16.620-0500: 20.442: [JNI Weak Reference, 0.0000242 secs]AdaptiveSizePolicy::update_averages: survived: 281180224 promoted: 16384 overflow: false", + "2017-08-25T08:49:16.474-0500: 20.296: [GC (Allocation Failure) AdaptiveSizePolicy::update_averages: survived: 281180224 promoted: 16384 overflow: false", + "2015-02-27T21:31:15.258-0500: 3.612: [GCAdaptiveSizePolicy::compute_survivor_space_size_and_thresh: survived: 6297408 promoted: 0 overflow: falseAdaptiveSizeStart: 3.625 collection: 1" + }, + { // 9 + "2017-08-25T10:38:23.920-0500: 6567.742: [Full GC (Ergonomics) 2017-08-25T10:38:23.921-0500: 6567.743: [marking phase2017-08-25T10:38:23.921-0500: 6567.743: [par mark, 0.1255544 secs]" + }, + { // 10 + "2017-08-25T10:38:24.047-0500: 6567.869: [reference processing2017-08-25T10:38:24.047-0500: 6567.869: [SoftReference, 29583 refs, 0.0025044 secs]2017-08-25T10:38:24.049-0500: 6567.871: [WeakReference, 7594 refs, 0.0006457 secs]2017-08-25T10:38:24.050-0500: 6567.872: [FinalReference, 1038 refs, 0.0009855 secs]2017-08-25T10:38:24.051-0500: 6567.873: [PhantomReference, 0 refs, 0 refs, 0.0000083 secs]2017-08-25T10:38:24.051-0500: 6567.873: [JNI Weak Reference, 0.0000212 secs], 0.0042334 secs]" + }, + { // 11 ver 7.0, 8.0 + "24.677: [GC24.677: [DefNew: 73140K->3189K(78656K), 0.0021028 secs] 76755K->6803K(253440K), 0.0022014 secs] [Times: user=0.01 sys=0.00, real=0.00 secs]", + "24.677: [GC (Allocation Failure) 24.677: [DefNew: 73140K->3189K(78656K), 0.0021028 secs] 76755K->6803K(253440K), 0.0022014 secs] [Times: user=0.01 sys=0.00, real=0.00 secs]" + }, + { // 12 ver 7.0, 8.0 + "24.665: [GC24.665: [DefNew", + "24.665: [GC (Allocation Failure) 24.665: [DefNew" + }, + { // 13 ver 7.0 + "15.287: [Full GC15.287: [Tenured: 0K->4602K(174784K), 0.0371090 secs] 25253K->4602K(253440K), [Perm : 13657K->13657K(21248K)], 0.0373080 secs] [Times: user=0.03 sys=0.00, real=0.04 secs]", + "13.906: [Full GC (System.gc()) 13.906: [Tenured: 0K->4267K(174784K), 0.0276425 secs] 29492K->4267K(253440K), [Metaspace: 15675K->15675K(1062912K)], 0.0280831 secs] [Times: user=0.02 sys=0.01, real=0.03 secs]" + }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, +// { +// }, + { + "2016-04-01T15:03:42.171-0700: 11025.637: [Preclean SoftReferences, 0.0000530 secs]2016-04-01T15:03:42.172-0700: 11025.637: [Preclean WeakReferences, 0.0006860 secs]2016-04-01T15:03:42.172-0700: 11025.638: [Preclean FinalReferences, 0.0005450 secs]2016-04-01T15:03:42.173-0700: 11025.639: [Preclean PhantomReferences, 0.0000230 secs]2016-04-01T15:03:42.197-0700: 11025.663: [CMS-concurrent-preclean: 0.025/0.026 secs] [Times: user=0.04 sys=0.01, real=0.03 secs]" + } + }; +} diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java index 399466b2..084c090c 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java @@ -2,12 +2,29 @@ // Licensed under the MIT License. package com.microsoft.gctoolkit.parser; +import com.microsoft.gctoolkit.event.generational.AbortablePreClean; +import com.microsoft.gctoolkit.event.generational.CMSRemark; +import com.microsoft.gctoolkit.event.generational.ConcurrentMark; +import com.microsoft.gctoolkit.event.generational.ConcurrentPreClean; +import com.microsoft.gctoolkit.event.generational.ConcurrentReset; +import com.microsoft.gctoolkit.event.generational.ConcurrentSweep; +import com.microsoft.gctoolkit.event.generational.InitialMark; +import com.microsoft.gctoolkit.event.jvm.JVMEvent; +import com.microsoft.gctoolkit.jvm.Diarizer; import com.microsoft.gctoolkit.jvm.Diary; +import com.microsoft.gctoolkit.parser.GCLogParser; +import com.microsoft.gctoolkit.parser.GenerationalHeapParser; +import com.microsoft.gctoolkit.parser.jvm.PreUnifiedDiarizer; +import com.microsoft.gctoolkit.parser.ParserTest; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -public class GenerationalHeapParserTest { +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +public class GenerationalHeapParserTest extends ParserTest { //todo: reactivate this code. @@ -50,4 +67,80 @@ public void canParseLFSFullGC() { // Assertions.assertTrue(eventCreated.get()); } + + @Test + public void testConcurrentModeFailure() { + String[] lines = new String[]{ + "27.537: [GC [1 CMS-initial-mark: 44399K(64768K)] 60362K(83392K), 0.0008727 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]", + "27.538: [CMS-concurrent-mark-start]", + "27.626: [CMS-concurrent-mark: 0.070/0.089 secs] [Times: user=0.14 sys=0.00, real=0.09 secs]", + "27.627: [CMS-concurrent-preclean-start]", + "27.627: [CMS-concurrent-preclean: 0.000/0.000 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]", + "27.627: [CMS-concurrent-abortable-preclean-start]", + "28.437: [GC 28.437: [ParNew (promotion failed)", + "Desired survivor size 1343488 bytes, new threshold 3 (max 4)", + "- age 1: 722736 bytes, 722736 total", + "- age 2: 84464 bytes, 807200 total", + "- age 3: 616504 bytes, 1423704 total", + "- age 4: 40 bytes, 1423744 total", + ": 17194K->17392K(18624K), 0.0023005 secs]28.440: [CMS28.440: [CMS-concurrent-abortable-preclean: 0.032/0.813 secs] [Times: user=1.30 sys=0.06, real=0.81 secs]", + " (concurrent mode failure): 62354K->8302K(64768K), 0.0931888 secs] 79477K->8302K(83392K), [CMS Perm : 10698K->10698K(21248K)], 0.0956950 secs] [Times: user=0.09 sys=0.00, real=0.09 secs]", + GCLogParser.END_OF_DATA_SENTINEL + }; + + List jvmEvents = feedParser(lines); + + try { + InitialMark initialMark = (InitialMark) jvmEvents.get(0); + ConcurrentMark concurrentMark = (ConcurrentMark) jvmEvents.get(1); + ConcurrentPreClean concurrentPreClean = (ConcurrentPreClean) jvmEvents.get(2); + AbortablePreClean abortablePreClean = (AbortablePreClean) jvmEvents.get(3); + } catch (ClassCastException cce) { + fail(cce.getMessage()); + } + assertTrue(true); + } + + @Test + public void testReferenceProcessingInConcurrentCycle() { + String[] lines = new String[]{ + "2016-04-01T15:03:41.107-0700: 11024.572: [GC (CMS Initial Mark) [1 CMS-initial-mark: 3165109K(4218880K)] 3203364K(5946048K), 0.0583850 secs] [Times: user=0.06 sys=0.00, real=0.06 secs]", + "2016-04-01T15:03:41.167-0700: 11024.632: [CMS-concurrent-mark-start]", + "2016-04-01T15:03:42.171-0700: 11025.636: [CMS-concurrent-mark: 0.991/1.004 secs] [Times: user=3.04 sys=0.13, real=1.00 secs]", + "2016-04-01T15:03:42.171-0700: 11025.637: [CMS-concurrent-preclean-start]", + "2016-04-01T15:03:42.171-0700: 11025.637: [Preclean SoftReferences, 0.0000530 secs]2016-04-01T15:03:42.172-0700: 11025.637: [Preclean WeakReferences, 0.0006860 secs]2016-04-01T15:03:42.172-0700: 11025.638: [Preclean FinalReferences, 0.0005450 secs]2016-04-01T15:03:42.173-0700: 11025.639: [Preclean PhantomReferences, 0.0000230 secs]2016-04-01T15:03:42.197-0700: 11025.663: [CMS-concurrent-preclean: 0.025/0.026 secs] [Times: user=0.04 sys=0.01, real=0.03 secs]", + "2016-04-01T15:03:42.198-0700: 11025.664: [CMS-concurrent-abortable-preclean-start]", + " CMS: abort preclean due to time 2016-04-01T15:03:47.227-0700: 11030.693: [CMS-concurrent-abortable-preclean: 5.002/5.029 secs] [Times: user=7.19 sys=0.28, real=5.03 secs]", + "2016-04-01T15:03:47.240-0700: 11030.706: [GC (CMS Final Remark)[YG occupancy: 1334704 K (1727168 K)]2016-04-01T15:03:47.241-0700: 11030.707: [Rescan (parallel) , 0.2638220 secs]2016-04-01T15:03:47.505-0700: 11030.970: [weak refs processing2016-04-01T15:03:47.505-0700: 11030.970: [SoftReference, 92 refs, 0.0000520 secs]2016-04-01T15:03:47.505-0700: 11030.971: [WeakReference, 1075 refs, 0.0003030 secs]2016-04-01T15:03:47.505-0700: 11030.971: [FinalReference, 5581 refs, 0.0232140 secs]2016-04-01T15:03:47.528-0700: 11030.994: [PhantomReference, 0 refs, 0.0000200 secs]2016-04-01T15:03:47.528-0700: 11030.994: [JNI Weak Reference, 0.0000270 secs], 0.0237120 secs]2016-04-01T15:03:47.528-0700: 11030.994: [scrub string table, 0.0047220 secs] [1 CMS-remark: 3165109K(4218880K)] 4499814K(5946048K), 0.2937310 secs] [Times: user=1.85 sys=0.00, real=0.30 secs]", + "2016-04-01T15:03:47.535-0700: 11031.001: [CMS-concurrent-sweep-start]", + "2016-04-01T15:03:48.111-0700: 11031.577: [CMS-concurrent-sweep: 0.576/0.576 secs] [Times: user=0.67 sys=0.04, real=0.57 secs]", + "2016-04-01T15:03:48.112-0700: 11031.578: [CMS-concurrent-reset-start]", + "2016-04-01T15:03:48.139-0700: 11031.605: [CMS-concurrent-reset: 0.027/0.027 secs] [Times: user=0.02 sys=0.00, real=0.03 secs]" + }; + + List jvmEvents = feedParser(lines); + + try { + InitialMark initialMark = (InitialMark) jvmEvents.get(0); + ConcurrentMark concurrentMark = (ConcurrentMark) jvmEvents.get(1); + ConcurrentPreClean concurrentPreClean = (ConcurrentPreClean) jvmEvents.get(2); + AbortablePreClean abortablePreClean = (AbortablePreClean) jvmEvents.get(3); + CMSRemark remark = (CMSRemark) jvmEvents.get(4); + ConcurrentSweep sweep = (ConcurrentSweep) jvmEvents.get(5); + ConcurrentReset reset = (ConcurrentReset) jvmEvents.get(6); + } catch (ClassCastException cce) { + fail(cce.getMessage()); + } + assertTrue(true); + } + + @Override + protected Diarizer diarizer() { + return new PreUnifiedDiarizer(); + } + + @Override + protected GCLogParser parser() { + return new GenerationalHeapParser(); + } } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/ParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/ParserTest.java similarity index 98% rename from parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/ParserTest.java rename to parser/src/test/java/com/microsoft/gctoolkit/parser/ParserTest.java index 425ff178..a4ae4f89 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/ParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/ParserTest.java @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package com.microsoft.gctoolkit.parser.patterns; +package com.microsoft.gctoolkit.parser; import com.microsoft.gctoolkit.event.MemoryPoolSummary; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/ShenandoahParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/ShenandoahParserTest.java index 98543dde..785c0734 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/ShenandoahParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/ShenandoahParserTest.java @@ -6,7 +6,6 @@ import com.microsoft.gctoolkit.event.shenandoah.ShenandoahCycle; import com.microsoft.gctoolkit.jvm.Diarizer; import com.microsoft.gctoolkit.parser.jvm.UnifiedDiarizer; -import com.microsoft.gctoolkit.parser.patterns.ParserTest; import org.junit.jupiter.api.Assertions; import java.util.List; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/ZGCParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/ZGCParserTest.java index d693c06a..1c61f99c 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/ZGCParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/ZGCParserTest.java @@ -10,7 +10,6 @@ import com.microsoft.gctoolkit.event.zgc.ZGCMetaspaceSummary; import com.microsoft.gctoolkit.jvm.Diarizer; import com.microsoft.gctoolkit.parser.jvm.UnifiedDiarizer; -import com.microsoft.gctoolkit.parser.patterns.ParserTest; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedCMSTenuredParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedCMSTenuredParserTest.java index 29050a54..69dcf3b1 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedCMSTenuredParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedCMSTenuredParserTest.java @@ -14,6 +14,7 @@ import com.microsoft.gctoolkit.jvm.Diarizer; import com.microsoft.gctoolkit.parser.CMSTenuredPoolParser; import com.microsoft.gctoolkit.parser.GCLogParser; +import com.microsoft.gctoolkit.parser.ParserTest; import com.microsoft.gctoolkit.parser.jvm.PreUnifiedDiarizer; import org.junit.jupiter.api.Test; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java index 58aca6cc..02fb973b 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java @@ -4,8 +4,6 @@ import com.microsoft.gctoolkit.event.GCCause; import com.microsoft.gctoolkit.event.generational.AbortablePreClean; -import com.microsoft.gctoolkit.event.generational.CMSConcurrentEvent; -import com.microsoft.gctoolkit.event.generational.CMSPhase; import com.microsoft.gctoolkit.event.generational.CMSRemark; import com.microsoft.gctoolkit.event.generational.ConcurrentMark; import com.microsoft.gctoolkit.event.generational.ConcurrentModeFailure; @@ -24,6 +22,7 @@ import com.microsoft.gctoolkit.jvm.Diarizer; import com.microsoft.gctoolkit.parser.GCLogParser; import com.microsoft.gctoolkit.parser.GenerationalHeapParser; +import com.microsoft.gctoolkit.parser.ParserTest; import com.microsoft.gctoolkit.parser.jvm.PreUnifiedDiarizer; import org.junit.jupiter.api.Test; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/UnifiedGenerationalEventsTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/UnifiedGenerationalEventsTest.java index edec29e9..bb11bf06 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/UnifiedGenerationalEventsTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/UnifiedGenerationalEventsTest.java @@ -19,6 +19,7 @@ import com.microsoft.gctoolkit.event.jvm.JVMEvent; import com.microsoft.gctoolkit.jvm.Diarizer; import com.microsoft.gctoolkit.parser.GCLogParser; +import com.microsoft.gctoolkit.parser.ParserTest; import com.microsoft.gctoolkit.parser.UnifiedGenerationalParser; import com.microsoft.gctoolkit.parser.jvm.UnifiedDiarizer; import org.junit.jupiter.api.Test; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java index b8f15153..46654b82 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java @@ -70,30 +70,30 @@ public void testForDetails() { }; private static final int[] detailsCountsNumberOfDifferentCollectors = { + 11, 10, - 9, - 4, - 9, - 8, - 4, - 7, + 4, + 9, + 8, + 4, + 7, 10, - 8, - 4 + 8, + 4 }; private static final int[][] detailsCounts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {0, 0, 3668, 12, 0, 200, 0, 0, 22, 0, 0, 156, 68}, //todo: these numbers don't seem to add up - {0, 0, 2021, 1, 0, 79, 0, 0, 0, 0, 0, 71, 70}, - {0, 0, 24981, 0, 0, 0, 0, 0, 0, 0, 2, 334, 333}, - {0, 0, 2762, 0, 0, 50, 0, 0, 3, 0, 0, 50, 49}, - {0, 0, 713, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1}, - {0, 0, 2160, 0, 0, 0, 0, 0, 0, 0, 1, 228, 228}, - {0, 0, 1969, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1}, - {0, 0, 57762, 0, 0, 371, 0, 0, 1, 0, 3, 371, 371}, - {0, 0, 20958, 0, 0, 4, 0, 0, 0, 0, 0, 4, 4}, - {0, 0, 22821, 0, 0, 0, 0, 0, 0, 0, 1, 106, 105} + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + {0, 0, 3668, 12, 0, 95, 0, 0, 22, 0, 0, 156, 68, 105, 73, 71, 66, 66}, //todo: these numbers don't seem to add up + {0, 0, 2021, 1, 0, 8, 0, 0, 0, 0, 0, 71, 70, 71, 71, 70, 66, 63}, + {0, 0, 24981, 0, 0, 0, 0, 0, 0, 0, 2, 334, 333, 0, 0, 0, 0, 0}, + {0, 0, 2762, 0, 0, 0, 0, 0, 3, 0, 0, 50, 49, 50, 49, 49, 49, 49}, + {0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1}, + {0, 0, 2160, 0, 0, 0, 0, 0, 0, 0, 1, 228, 228, 0, 0, 0, 0, 0}, + {0, 0, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}, + {0, 0, 57762, 0, 0, 0, 0, 0, 1, 0, 3, 371, 371, 371, 371, 369, 371, 371}, + {0, 0, 20958, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4}, + {0, 0, 22821, 0, 0, 0, 0, 0, 0, 0, 1, 106, 105, 0, 0, 0, 0, 0} }; @@ -124,9 +124,9 @@ public void testForDetailsGCCause170() { }; private static final int[][] detailsGCCause170Counts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 - {0, 0, 603, 0, 0, 13, 0, 0, 0, 0, 0, 13, 13}, - {0, 0, 603, 0, 0, 13, 0, 0, 0, 0, 0, 13, 13}, + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + {0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 13, 13, 13, 13, 13, 13, 13}, + {0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 13, 13, 13, 13, 13, 13, 13}, }; @Test @@ -150,7 +150,7 @@ public void testForDetailsTenuring() { "705_pa1.log", "706_pa2.gc", "707_pa3.gc", - "860_gc.log", //todo: perform an audit of results for this and the next 3 logs. + "860_gc.log", "863_gc.log", "929_def.cms.wd.wt.log", "BF653AA1-F897-4279-9570-2E4837D311E8-1405428823225.log.gz", //todo: missing many statements that should be filtered out @@ -166,48 +166,48 @@ public void testForDetailsTenuring() { }; private static final int[] detailsTenuringCountsNumberOfDifferentCollectors = { - 1, - 8, - 10, - 7, - 7, - 7, - 9, - 12, - 10, - 8, - 8, - 8, - 9, - 9, - 9, + 1, + 8, + 11, + 8, + 8, + 8, 10, - 7, + 13, + 11, + 8, + 8, + 8, + 9, + 9, + 9, + 11, + 7, 10, - 8 + 8 }; private static final int[][] detailsTenuringCounts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 - {0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {0, 0, 14816, 0, 0, 30, 0, 0, 0, 0, 0, 30, 30}, - {0, 0, 3370, 4, 0, 21, 0, 0, 0, 0, 3, 16, 12}, - {0, 0, 35995, 0, 0, 13691, 0, 0, 0, 0, 0, 13676, 13673}, //????? - {0, 0, 30334, 0, 0, 12350, 0, 0, 0, 0, 0, 12339, 12336}, //????? - {0, 0, 34285, 0, 0, 13368, 0, 0, 0, 0, 0, 13354, 13350}, - {0, 0, 4592, 0, 0, 50, 0, 0, 0, 0, 3, 45, 43}, - {0, 0, 47, 1, 0, 224, 14, 0, 18, 0, 2, 208, 172}, - {0, 0, 3369, 6, 0, 23, 0, 0, 0, 0, 3, 17, 11}, - {0, 0, 5305, 0, 0, 917, 0, 0, 0, 0, 0, 918, 917}, //zip todo: audit - {0, 0, 146, 0, 0, 1402, 0, 0, 0, 0, 0, 1402, 1402}, //zip not done - {0, 0, 4046, 0, 0, 15, 0, 0, 0, 0, 0, 15, 15}, - {0, 0, 46228, 0, 0, 1, 0, 0, 0, 0, 17, 1, 1}, - {0, 0, 47000, 0, 0, 2, 0, 0, 0, 0, 15, 2, 2}, - {0, 0, 48222, 0, 0, 1, 0, 0, 0, 0, 17, 1, 1}, - {0, 0, 3370, 4, 0, 21, 0, 0, 0, 0, 3, 16, 12}, - {0, 0, 1333, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1}, - {0, 0, 508, 0, 0, 5, 1, 0, 0, 0, 1395, 5, 4}, - {0, 0, 23829, 0, 0, 977, 0, 0, 0, 0, 0, 978, 977} + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + { 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + { 0, 0, 14816, 0, 0, 0, 0, 0, 0, 0, 0, 30, 30, 30, 30, 30, 30, 30 }, + { 0, 0, 3370, 4, 0, 5, 0, 0, 0, 0, 3, 16, 12, 16, 16, 13, 11, 11}, + { 0, 0, 35995, 0, 0, 15, 0, 0, 0, 0, 0, 13676, 13673, 13673, 13673, 0, 13672, 13672}, + { 0, 0, 30334, 0, 0, 12, 0, 0, 0, 0, 0, 12339, 12336, 12336, 12336, 0, 12336, 12336}, + { 0, 0, 34285, 0, 0, 14, 0, 0, 0, 0, 0, 13354, 13350, 13350, 13350, 0, 13349, 13349}, // todo: audit cmf count feels wrong + { 0, 0, 4592, 0, 0, 5, 0, 0, 0, 0, 3, 45, 43, 45, 44, 16, 40, 40}, // todo: audit abortable pre-clean feels wrong + { 0, 0, 47, 1, 0, 22, 14, 0, 18, 0, 2, 208, 172, 202, 197, 169, 172, 172}, // todo: audit expected 202 - 22 = 186 + { 0, 0, 3369, 6, 0, 6, 0, 0, 0, 0, 3, 17, 11, 17, 17, 15, 11, 11}, + { 0, 0, 5305, 0, 0, 0, 0, 0, 0, 0, 0, 918, 917, 917, 917, 915, 917, 917}, + { 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 1402, 1402, 1402, 1375, 1402, 1402}, + { 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 15, 15, 2, 15, 15}, + { 0, 0, 46228, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1, 1, 1, 1, 1, 1}, + { 0, 0, 47000, 0, 0, 0, 0, 0, 0, 0, 15, 2, 2, 2, 2, 2, 2, 2}, + { 0, 0, 48222, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1, 1, 1, 1, 1, 1}, + { 0, 0, 3370, 4, 0, 5, 0, 0, 0, 0, 3, 16, 12, 16, 16, 13, 11, 11}, + { 0, 0, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}, + { 0, 0, 508, 0, 0, 0, 1, 0, 0, 0, 1395, 5, 4, 5, 5, 4, 4, 4}, // todo: audit, expected 5 at the end + { 0, 0, 23829, 0, 0, 0, 0, 0, 0, 0, 0, 978, 977, 977, 977, 959, 975, 975} // todo: audit, how were 2 concurrent cycles lost }; @Test @@ -235,8 +235,8 @@ public void testForDefNewDetails() { }; private static final int[][] defnewDetailsCounts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 - {0, 19114, 0, 0, 0, 26, 0, 0, 0, 0, 0, 26, 26} + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + {0, 19114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 2, 26, 26} }; @Test @@ -258,13 +258,13 @@ public void testForParNewDetailsTenuringReference80() { }; private static final int[] parNewDetailsTenuringReference180NumberOfDifferentCollectors = { - 10 + 11 }; private static final int[][] parNewDetailsTenuringReference180Counts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {0, 0, 15338, 0, 0, 7, 0, 0, 3, 0, 7, 7, 1} + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + {0, 0, 15338, 0, 0, 3, 0, 0, 3, 0, 7, 7, 1, 4, 1, 1, 1, 1} }; @@ -293,7 +293,7 @@ public void testForBrokenGCLockerWithTLAB() { }; private static final int[][] parNewDetailsTenuringTLABCounts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1} + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + {0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1} }; } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java index e6cd446b..acdc9b89 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java @@ -45,22 +45,22 @@ public void testForSimpleLogs() { }; private static final int[] detailsCountsNumberOfDifferentCollectors = { - 9, - 8, - 8, - 9, - 11, - 11 + 10, + 8, + 8, + 9, + 12, + 12 }; private static final int[][] detailsCounts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 12 - {0, 0, 12359, 41, 0, 219, 0, 0, 0, 0, 0, 179, 166}, - {0, 0, 2564, 0, 0, 13, 0, 0, 0, 0, 0, 13, 13}, - {0, 0, 2564, 0, 0, 11, 0, 0, 0, 0, 0, 11, 11}, - {0, 0, 1374, 0, 0, 2, 1, 0, 0, 0, 0, 3, 2}, - {0, 0, 143543, 75, 0, 10434, 0, 0, 7, 0, 1, 10272, 10015}, - {0, 0, 72395, 43, 0, 5254, 0, 0, 4, 0, 1, 5137, 4985} + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + { 0, 0, 12359, 41, 0, 41, 0, 0, 0, 0, 0, 179, 166, 178, 169, 166, 149, 138}, + { 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 13, 13, 13, 13, 13, 13, 13}, + { 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 11, 11, 11, 11, 11}, + { 0, 0, 1374, 0, 0, 0, 1, 0, 0, 0, 0, 3, 2, 2, 2, 2, 2, 2}, + { 0, 0, 143543, 75, 0, 306, 0, 0, 7, 0, 1, 10272, 10015, 10128, 10035, 10018, 10014, 9969}, + { 0, 0, 72395, 43, 0, 173, 0, 0, 4, 0, 1, 5137, 4985, 5081, 4994, 4986, 4985, 4966} }; @@ -87,21 +87,21 @@ public void testForDetailsTenuring() { }; private static final int[] detailsTenuringCountsNumberOfDifferentCollectors = { - 9, - 9, 10, - 8, - 9 + 9, + 11, + 8, + 9 }; private static final int[][] detailsTenuringCounts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {0, 0, 67, 0, 0, 26, 0, 0, 1, 0, 0, 25, 24}, - {0, 0, 6240, 0, 0, 2087, 0, 0, 0, 0, 1, 2087, 2087}, - {0, 0, 417811, 83, 0, 7924, 0, 0, 76, 0, 0, 7840, 7809}, - {0, 0, 32439, 0, 0, 103, 0, 0, 0, 0, 0, 103, 103}, - {0, 0, 18507, 0, 0, 5, 1, 0, 0, 0, 0, 6, 5}, + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + { 0, 0, 67, 0, 0, 1, 0, 0, 1, 0, 0, 25, 24, 25, 24, 24, 24, 24}, + { 0, 0, 6240, 0, 0, 0, 0, 0, 0, 0, 1, 2087, 2087, 2087, 2087, 2087, 2086, 2086}, + { 0, 0, 417811, 83, 0, 84, 0, 0, 76, 0, 0, 7840, 7809, 7824, 7809, 7783, 7756, 7756}, + { 0, 0, 32439, 0, 0, 0, 0, 0, 0, 0, 0, 103, 103, 103, 103, 102, 102, 102}, // todo: audit as 103 expected + { 0, 0, 18507, 0, 0, 0, 1, 0, 0, 0, 0, 6, 5, 5, 5, 5, 5, 5}, }; @@ -129,7 +129,7 @@ public void testForDetailsDebug() { }; private static final int[][] detailsDebugCounts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {0, 0, 3089, 0, 0, 532, 0, 0, 0, 0, 0, 533, 532} + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + { 0, 0, 3089, 0, 0, 0, 0, 0, 0, 0, 0, 533, 532, 532, 532, 532, 532, 532} }; } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java index 0ac6f8d5..43cfb18c 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java @@ -51,17 +51,17 @@ public abstract class ParserTest { Map.entry(GarbageCollectionTypes.InitialMark, 11), Map.entry(GarbageCollectionTypes.Remark, 12), Map.entry(GarbageCollectionTypes.PSFull, 8), // bit of a hack to account that the parser is now differentiating between Full and PSFull. (kcp 11/8/15) + Map.entry(GarbageCollectionTypes.ConcurrentMark, 13), + + Map.entry(GarbageCollectionTypes.Concurrent_Preclean, 14), + Map.entry(GarbageCollectionTypes.Abortable_Preclean,15), + Map.entry(GarbageCollectionTypes.Concurrent_Sweep, 16), + Map.entry(GarbageCollectionTypes.Concurrent_Reset, 17), + Map.entry(GarbageCollectionTypes.Mixed, 1), Map.entry(GarbageCollectionTypes.G1GCYoungInitialMark, 2), Map.entry(GarbageCollectionTypes.G1GCMixedInitialMark, 3), Map.entry(GarbageCollectionTypes.Full, 4), - Map.entry(GarbageCollectionTypes.ConcurrentMark, 5), - - Map.entry(GarbageCollectionTypes.Concurrent_Preclean, 13), - Map.entry(GarbageCollectionTypes.Abortable_Preclean,14), - Map.entry(GarbageCollectionTypes.Concurrent_Sweep, 15), - Map.entry(GarbageCollectionTypes.Concurrent_Reset, 16), - Map.entry(GarbageCollectionTypes.G1GCConcurrentMark, 5), Map.entry(GarbageCollectionTypes.G1GCRemark, 7), @@ -92,7 +92,6 @@ public void analyzeResults(String gcLogName, TestResults testResults, int number for (int i = 0; i < invocationCounts.length; i++) { assertEquals(invocationCounts[i], testResults.getCount(i), "Phase Count Differs @ " + i + " for " + findGarbageCollector(i) + " in " + gcLogName); } - } private GCLogFile loadLogFile(Path path, boolean rotating) throws IOException { diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java index f2e6c32c..6eb85d48 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java @@ -62,7 +62,7 @@ public void testCMSLogsMissingTimeStamps() { }; private static final int[][] cmsCounts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - { 0, 0, 6167, 0, 0, 2, 0, 0, 0, 0, 0, 2, 2}, + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + { 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2}, }; } From e21cb1c113e4f74aab204743ba70439d30ce2373 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 14 Mar 2024 10:15:49 -0700 Subject: [PATCH 14/34] refactor: GenerationalHeapParser was duplicating CMF failures under certain conditions --- .../gctoolkit/parser/AbstractLogTrace.java | 19 +++- .../parser/GenerationalHeapParser.java | 8 +- .../parser/GenerationalHeapParserTest.java | 103 ++++++++++++------ .../parser/unittests/CMSParNewParserTest.java | 16 +-- .../unittests/ICMSParNewParserTest.java | 14 +-- 5 files changed, 106 insertions(+), 54 deletions(-) diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java index 89f2115c..dc238e74 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java @@ -76,13 +76,28 @@ public String getDateStamp() { } } + /** + * + * @return the date timestamp paring found at the beginning of the GC log line. + */ public DateTimeStamp getDateTimeStamp() { return getDateTimeStamp(1); } - public DateTimeStamp getDateTimeStamp(int index) { + /** + * If a line contains multiple date timestamp group pairing then return the nth pair. + * The following example contains 3 different date timestamps. index of 1 yields 57724.218 + * whereas index 3 yields 2010-04-21T10:45:33.367+0100@57724.319 + * + * 57724.218: [Full GC 57724.218: [CMS2010-04-21T10:45:33.367+0100: 57724.319: [CMS-concurrent-mark: 2.519/2.587 secs] + * + * one time stamp or a date timestamp pairing + * @param nth is date timestamp field pair to be returned + * @return the nth date timestamp pairing + */ + public DateTimeStamp getDateTimeStamp(int nth) { Matcher matcher = DATE_TIME_STAMP_RULE.matcher(trace.group(0)); - for (int i = 0; i < index - 1; i++) + for (int i = 0; i < nth - 1; i++) if (!matcher.find()) break; if (matcher.find()) { diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java index 9caa4daf..b6ff0f89 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java @@ -488,6 +488,7 @@ private void parNewFLSTime(GCLogTrace trace, String line) { MemoryPoolSummary heap = new MemoryPoolSummary(heapForwardReference.getOccupancyBeforeCollection(), heapForwardReference.getSizeAfterCollection(), heapForwardReference.getOccupancyBeforeCollection(), heapForwardReference.getSizeAfterCollection()); FullGC fullGc = new FullGC(fullGCTimeStamp, gcCauseForwardReference, trace.getDuration()); fullGc.add(heap); + fullGc.add(extractCPUSummary(line)); publish(fullGc); } else LOGGER.warning("Unable to parse -> " + trace); @@ -1173,6 +1174,7 @@ public void fullGCInterruptsConcurrentPhase(GCLogTrace trace, String line) { fullGCTimeStamp = getClock(); garbageCollectionTypeForwardReference = GarbageCollectionTypes.FullGC; gcCauseForwardReference = trace.gcCause(); + endOfConcurrentPhase(trace,trace.getDateTimeStamp(3), 5); } public void fullGCReferenceConcurrentModeFailure(GCLogTrace trace, String line) { @@ -2068,6 +2070,7 @@ private void log(String line) { // if (line.contains("Large block")) return; // // GCToolKit.LOG_DEBUG_MESSAGE(() -> "GenerationalHeapParser missed: " + line); + if (line.contains("CMSCMS: Large block")) return; LOGGER.log(Level.WARNING, "Missed: {0}", line); } @@ -2119,6 +2122,9 @@ private void endOfConcurrentPhase(GCLogTrace trace, DateTimeStamp timeStamp, int double cpuTime = trace.getDoubleGroup(4 + offset); double wallTime = trace.getDoubleGroup(5 + offset); double duration = timeStamp.toSeconds() - startOfConcurrentPhase.toSeconds(); + // The wallTime measure is a very good estimate of duration. + if ( duration == Double.NaN) + duration = wallTime; if ("mark".equals(phase)) publish(new ConcurrentMark(startOfConcurrentPhase, duration, cpuTime, wallTime)); else if ("preclean".equals(phase)) @@ -2158,12 +2164,12 @@ private void clear() { } public void publish( JVMEvent event, boolean drain, boolean clear) { + publish(event,clear); if (drain) { for( GenerationalGCPauseEvent pauseEvent : queue) publish(pauseEvent, false); queue.clear(); } - publish(event,clear); } public void publish(JVMEvent event, boolean clear) { diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java index 084c090c..d70c7c5b 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java @@ -5,10 +5,13 @@ import com.microsoft.gctoolkit.event.generational.AbortablePreClean; import com.microsoft.gctoolkit.event.generational.CMSRemark; import com.microsoft.gctoolkit.event.generational.ConcurrentMark; +import com.microsoft.gctoolkit.event.generational.ConcurrentModeFailure; import com.microsoft.gctoolkit.event.generational.ConcurrentPreClean; import com.microsoft.gctoolkit.event.generational.ConcurrentReset; import com.microsoft.gctoolkit.event.generational.ConcurrentSweep; +import com.microsoft.gctoolkit.event.generational.FullGC; import com.microsoft.gctoolkit.event.generational.InitialMark; +import com.microsoft.gctoolkit.event.generational.ParNew; import com.microsoft.gctoolkit.event.jvm.JVMEvent; import com.microsoft.gctoolkit.jvm.Diarizer; import com.microsoft.gctoolkit.jvm.Diary; @@ -16,13 +19,13 @@ import com.microsoft.gctoolkit.parser.GenerationalHeapParser; import com.microsoft.gctoolkit.parser.jvm.PreUnifiedDiarizer; import com.microsoft.gctoolkit.parser.ParserTest; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.*; public class GenerationalHeapParserTest extends ParserTest { @@ -31,41 +34,35 @@ public class GenerationalHeapParserTest extends ParserTest { @Test public void canParseLFSFullGC() { - String fragment = "2019-11-14T23:50:29.896+0000: 91334.028: [Full GC (Allocation Failure) Before GC:\n" + - "Statistics for BinaryTreeDictionary:\n" + - "------------------------------------\n" + - "Total Free Space: 1261\n" + - "Max Chunk Size: 1261\n" + - "Number of Blocks: 1\n" + - "Av. Block Size: 1261\n" + - "Tree Height: 1\n" + - "2019-11-14T23:50:29.896+0000: 91334.028: [CMSCMS: Large block 0x00000007bfffd898\n" + - ": 2097142K->2097142K(2097152K), 1.9092744 secs] 4063215K->4063215K(4063232K), [Metaspace: 99441K->99441K(1140736K)]After GC:\n" + - "Statistics for BinaryTreeDictionary:\n" + - "------------------------------------\n" + - "Total Free Space: 1261\n" + - "Max Chunk Size: 1261\n" + - "Number of Blocks: 1\n" + - "Av. Block Size: 1261\n" + - "Tree Height: 1\n" + - ", 1.9094806 secs] [Times: user=1.91 sys=0.00, real=1.91 secs]\n" + - "2019-11-14T23:50:31.806+0000: 91335.938: Total time for which application threads were stopped: 4.0762194 seconds, Stopping threads took: 0.0000522 seconds\n"; - - AtomicBoolean eventCreated = new AtomicBoolean(false); - GenerationalHeapParser parser = new GenerationalHeapParser(); //, event -> { - parser.diary(new Diary()); -// parser. -// Assertions.assertTrue(event instanceof FullGC); -// FullGC fgc = (FullGC) event; -// Assertions.assertEquals(1.9094806d, fgc.getDuration()); -// Assertions.assertEquals(4063232, fgc.getHeap().getSizeAfterCollection()); -// eventCreated.set(true); -// }); -// -// Arrays.stream(fragment.split("\n")).forEach(parser::receive); -// -// Assertions.assertTrue(eventCreated.get()); + String[] lines = {"2019-11-14T23:50:29.896+0000: 91334.028: [Full GC (Allocation Failure) Before GC:", + "Statistics for BinaryTreeDictionary:", + "------------------------------------", + "Total Free Space: 1261", + "Max Chunk Size: 1261", + "Number of Blocks: 1", + "Av. Block Size: 1261", + "Tree Height: 1", + "2019-11-14T23:50:29.896+0000: 91334.028: [CMSCMS: Large block 0x00000007bfffd898", + ": 2097142K->2097142K(2097152K), 1.9092744 secs] 4063215K->4063215K(4063232K), [Metaspace: 99441K->99441K(1140736K)]After GC:", + "Statistics for BinaryTreeDictionary:", + "------------------------------------", + "Total Free Space: 1261", + "Max Chunk Size: 1261", + "Number of Blocks: 1", + "Av. Block Size: 1261", + "Tree Height: 1", + ", 1.9094806 secs] [Times: user=1.91 sys=0.00, real=1.91 secs]" + }; + + List jvmEvents = feedParser(lines); + try { + FullGC fgc = (FullGC) jvmEvents.get(0); + Assertions.assertEquals(1.9094806d, fgc.getDuration()); + Assertions.assertEquals(4063232, fgc.getHeap().getSizeAfterCollection()); + } catch(Throwable t) { + fail(t); + } } @Test @@ -134,6 +131,40 @@ public void testReferenceProcessingInConcurrentCycle() { assertTrue(true); } + @Test + public void testThat2CMFDoNotHappen() { + String[][] lines = new String[][]{ + { + "57721.729: [GC [1 CMS-initial-mark: 763361K(786432K)] 767767K(1022400K), 0.0022735 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]", + "57721.732: [CMS-concurrent-mark-start]", + "57722.918: [GC 57722.918: [ParNew: 209792K->26176K(235968K), 0.0618431 secs] 973153K->802453K(1022400K), 0.0620347 secs] [Times: user=0.38 sys=0.00, real=0.06 secs]", + "57724.218: [Full GC 57724.218: [CMS2010-04-21T10:45:33.367+0100: 57724.319: [CMS-concurrent-mark: 2.519/2.587 secs] [Times: user=12.58 sys=0.09, real=2.59 secs]", + "(concurrent mode failure): 776277K->770654K(786432K), 6.0499857 secs] 1012245K->770654K(1022400K), [CMS Perm : 23211K->23211K(38736K)], 6.0501617 secs] [Times: user=6.09 sys=0.00, real=6.05 secs]" + }, + { + "58272.354: [GC [1 CMS-initial-mark: 786431K(786432K)] 794666K(1022400K), 0.0088514 secs] [Times: user=0.01 sys=0.00, real=0.02 secs]", + "58272.363: [CMS-concurrent-mark-start]", + "58273.778: [Full GC 58273.778: [CMS2010-04-21T10:54:43.688+0100: 58274.663: [CMS-concurrent-mark: 2.299/2.300 secs] [Times: user=8.69 sys=0.11, real=2.30 secs]", + "(concurrent mode failure): 786431K->785452K(786432K), 6.5738696 secs] 1022399K->785452K(1022400K), [CMS Perm : 23211K->23211K(38736K)], 6.5740517 secs] [Times: user=7.44 sys=0.00, real=6.56 secs]" + } + }; + + Class[][] eventTypes = { + {InitialMark.class, ConcurrentMark.class, ParNew.class, ConcurrentModeFailure.class}, + {InitialMark.class, ConcurrentMark.class, ConcurrentModeFailure.class} + }; + + for (int i = 0; i < lines.length; i++) { + List jvmEvents = feedParser(lines[i]); + + for (int j = 0; j < eventTypes[i].length; j++) { + assertEquals(jvmEvents.get(j).getClass(), eventTypes[i][j]); + } + jvmEvents.clear(); + } + } + + @Override protected Diarizer diarizer() { return new PreUnifiedDiarizer(); diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java index 46654b82..8dd2f1bb 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java @@ -84,8 +84,8 @@ public void testForDetails() { private static final int[][] detailsCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 - {0, 0, 3668, 12, 0, 95, 0, 0, 22, 0, 0, 156, 68, 105, 73, 71, 66, 66}, //todo: these numbers don't seem to add up - {0, 0, 2021, 1, 0, 8, 0, 0, 0, 0, 0, 71, 70, 71, 71, 70, 66, 63}, + {0, 0, 3668, 12, 0, 96, 0, 0, 22, 0, 0, 156, 68, 156, 75, 71, 68, 66}, //todo: these numbers don't seem to add up + {0, 0, 2021, 1, 0, 8, 0, 0, 0, 0, 0, 71, 70, 71, 71, 70, 70, 63}, {0, 0, 24981, 0, 0, 0, 0, 0, 0, 0, 2, 334, 333, 0, 0, 0, 0, 0}, {0, 0, 2762, 0, 0, 0, 0, 0, 3, 0, 0, 50, 49, 50, 49, 49, 49, 49}, {0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1}, @@ -191,12 +191,12 @@ public void testForDetailsTenuring() { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 { 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, { 0, 0, 14816, 0, 0, 0, 0, 0, 0, 0, 0, 30, 30, 30, 30, 30, 30, 30 }, - { 0, 0, 3370, 4, 0, 5, 0, 0, 0, 0, 3, 16, 12, 16, 16, 13, 11, 11}, + { 0, 0, 3370, 4, 0, 5, 0, 0, 0, 0, 3, 16, 12, 16, 16, 13, 12, 11}, { 0, 0, 35995, 0, 0, 15, 0, 0, 0, 0, 0, 13676, 13673, 13673, 13673, 0, 13672, 13672}, { 0, 0, 30334, 0, 0, 12, 0, 0, 0, 0, 0, 12339, 12336, 12336, 12336, 0, 12336, 12336}, { 0, 0, 34285, 0, 0, 14, 0, 0, 0, 0, 0, 13354, 13350, 13350, 13350, 0, 13349, 13349}, // todo: audit cmf count feels wrong - { 0, 0, 4592, 0, 0, 5, 0, 0, 0, 0, 3, 45, 43, 45, 44, 16, 40, 40}, // todo: audit abortable pre-clean feels wrong - { 0, 0, 47, 1, 0, 22, 14, 0, 18, 0, 2, 208, 172, 202, 197, 169, 172, 172}, // todo: audit expected 202 - 22 = 186 + { 0, 0, 4592, 0, 0, 5, 0, 0, 0, 0, 3, 45, 43, 45, 45, 17, 43, 40}, + { 0, 0, 47, 1, 0, 22, 17, 0, 19, 0, 2, 210, 172, 210, 200, 178, 172, 172}, // todo: audit expected 210 - 22 = 188, not 172 { 0, 0, 3369, 6, 0, 6, 0, 0, 0, 0, 3, 17, 11, 17, 17, 15, 11, 11}, { 0, 0, 5305, 0, 0, 0, 0, 0, 0, 0, 0, 918, 917, 917, 917, 915, 917, 917}, { 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 1402, 1402, 1402, 1375, 1402, 1402}, @@ -204,9 +204,9 @@ public void testForDetailsTenuring() { { 0, 0, 46228, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1, 1, 1, 1, 1, 1}, { 0, 0, 47000, 0, 0, 0, 0, 0, 0, 0, 15, 2, 2, 2, 2, 2, 2, 2}, { 0, 0, 48222, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1, 1, 1, 1, 1, 1}, - { 0, 0, 3370, 4, 0, 5, 0, 0, 0, 0, 3, 16, 12, 16, 16, 13, 11, 11}, + { 0, 0, 3370, 4, 0, 5, 0, 0, 0, 0, 3, 16, 12, 16, 16, 13, 12, 11}, { 0, 0, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}, - { 0, 0, 508, 0, 0, 0, 1, 0, 0, 0, 1395, 5, 4, 5, 5, 4, 4, 4}, // todo: audit, expected 5 at the end + { 0, 0, 508, 0, 0, 0, 1, 0, 0, 0, 1395, 5, 4, 5, 5, 5, 4, 4}, // todo: audit, expected 5 at the end { 0, 0, 23829, 0, 0, 0, 0, 0, 0, 0, 0, 978, 977, 977, 977, 959, 975, 975} // todo: audit, how were 2 concurrent cycles lost }; @@ -264,7 +264,7 @@ public void testForParNewDetailsTenuringReference80() { private static final int[][] parNewDetailsTenuringReference180Counts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 - {0, 0, 15338, 0, 0, 3, 0, 0, 3, 0, 7, 7, 1, 4, 1, 1, 1, 1} + {0, 0, 15338, 0, 0, 3, 0, 0, 3, 0, 7, 7, 1, 7, 1, 1, 1, 1} }; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java index acdc9b89..747adcaa 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java @@ -55,12 +55,12 @@ public void testForSimpleLogs() { private static final int[][] detailsCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 - { 0, 0, 12359, 41, 0, 41, 0, 0, 0, 0, 0, 179, 166, 178, 169, 166, 149, 138}, + { 0, 0, 12359, 41, 0, 41, 0, 0, 0, 0, 0, 179, 166, 179, 177, 166, 166, 138}, { 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 13, 13, 13, 13, 13, 13, 13}, { 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 11, 11, 11, 11, 11}, - { 0, 0, 1374, 0, 0, 0, 1, 0, 0, 0, 0, 3, 2, 2, 2, 2, 2, 2}, - { 0, 0, 143543, 75, 0, 306, 0, 0, 7, 0, 1, 10272, 10015, 10128, 10035, 10018, 10014, 9969}, - { 0, 0, 72395, 43, 0, 173, 0, 0, 4, 0, 1, 5137, 4985, 5081, 4994, 4986, 4985, 4966} + { 0, 0, 1374, 0, 0, 0, 1, 0, 0, 0, 0, 3, 2, 3, 2, 2, 2, 2}, + { 0, 0, 143543, 75, 0, 306, 0, 0, 7, 0, 1, 10272, 10015, 10271, 10039, 10018, 10015, 9969}, + { 0, 0, 72395, 43, 0, 173, 0, 0, 4, 0, 1, 5137, 4985, 5136, 4994, 4986, 4985, 4966} }; @@ -99,9 +99,9 @@ public void testForDetailsTenuring() { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 { 0, 0, 67, 0, 0, 1, 0, 0, 1, 0, 0, 25, 24, 25, 24, 24, 24, 24}, { 0, 0, 6240, 0, 0, 0, 0, 0, 0, 0, 1, 2087, 2087, 2087, 2087, 2087, 2086, 2086}, - { 0, 0, 417811, 83, 0, 84, 0, 0, 76, 0, 0, 7840, 7809, 7824, 7809, 7783, 7756, 7756}, - { 0, 0, 32439, 0, 0, 0, 0, 0, 0, 0, 0, 103, 103, 103, 103, 102, 102, 102}, // todo: audit as 103 expected - { 0, 0, 18507, 0, 0, 0, 1, 0, 0, 0, 0, 6, 5, 5, 5, 5, 5, 5}, + { 0, 0, 417811, 83, 0, 84, 0, 0, 76, 0, 0, 7840, 7809, 7824, 7809, 7783, 7757, 7756}, + { 0, 0, 32439, 0, 0, 0, 0, 0, 0, 0, 0, 103, 103, 103, 103, 102, 102, 102}, + { 0, 0, 18507, 0, 0, 0, 1, 0, 0, 0, 0, 6, 5, 6, 5, 5, 5, 5}, }; From 40595f07b30feaf776a156f2c0c42e9bad7d59f3 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 14 Mar 2024 14:04:33 -0700 Subject: [PATCH 15/34] refactor: ensure GC log is closed when discovering the format --- api/src/main/java/com/microsoft/gctoolkit/io/GCLogFile.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFile.java b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFile.java index bbe7d09a..cf0e7e63 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFile.java +++ b/api/src/main/java/com/microsoft/gctoolkit/io/GCLogFile.java @@ -127,8 +127,8 @@ public final String endOfData() { * @throws IOException Thrown from reading the stream. */ private TripleState discoverFormat() { - try { - boolean isUnified = firstNLines(stream(), SHOULD_HAVE_SEEN_A_UNIFIED_DECORATOR_BY_THIS_LINE_IN_THE_LOG) + try (Stream stream = stream()) { // contribution from MansuyDavid @github + boolean isUnified = firstNLines(stream, SHOULD_HAVE_SEEN_A_UNIFIED_DECORATOR_BY_THIS_LINE_IN_THE_LOG) .map(LINE_STARTS_WITH_DECORATOR::matcher) .anyMatch(Matcher::find); return TripleState.valueOf(isUnified); From cd7f1a2e94d866aeb7efd43cca95d61a35df5ffa Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Fri, 15 Mar 2024 11:22:38 -0700 Subject: [PATCH 16/34] refactor: add test for reported bug --- .../parser/GenerationalHeapParserTest.java | 5 - .../parser/UnifiedG1GCParserFragmentTest.java | 95 +++++++++++++++++++ 2 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java index d70c7c5b..7d22b90c 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java @@ -14,16 +14,11 @@ import com.microsoft.gctoolkit.event.generational.ParNew; import com.microsoft.gctoolkit.event.jvm.JVMEvent; import com.microsoft.gctoolkit.jvm.Diarizer; -import com.microsoft.gctoolkit.jvm.Diary; -import com.microsoft.gctoolkit.parser.GCLogParser; -import com.microsoft.gctoolkit.parser.GenerationalHeapParser; import com.microsoft.gctoolkit.parser.jvm.PreUnifiedDiarizer; -import com.microsoft.gctoolkit.parser.ParserTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.jupiter.api.Assertions.*; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java new file mode 100644 index 00000000..bc83b558 --- /dev/null +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +package com.microsoft.gctoolkit.parser; + +import com.microsoft.gctoolkit.event.g1gc.G1Young; +import com.microsoft.gctoolkit.event.jvm.JVMEvent; +import com.microsoft.gctoolkit.jvm.Diarizer; +import com.microsoft.gctoolkit.parser.diary.TestLogFile; +import com.microsoft.gctoolkit.parser.jvm.UnifiedDiarizer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.fail; + +public class UnifiedG1GCParserFragmentTest extends ParserTest { + + @Override + protected Diarizer diarizer() { + return new UnifiedDiarizer(); + } + + @Override + protected GCLogParser parser() { + return new UnifiedG1GCParser(); + } + + + @Test + public void testNewDecoratorCombination() { + String[] lines = { + "[2023-12-06T07:32:54.117+0000][29ms] Version: 17.0.2+8-86 (release)", + "[2023-12-06T07:32:54.117+0000][25ms] Using G1", + "[2023-12-06T07:32:54.117+0000][29ms] CPUs: 2 total, 2 available", + "[2023-12-06T07:32:54.117+0000][29ms] Memory: 6531M", + "[2023-12-06T07:32:54.117+0000][29ms] Large Page Support: Disabled", + "[2023-12-06T07:32:54.117+0000][29ms] NUMA Support: Disabled", + "[2023-12-06T07:32:54.117+0000][29ms] Compressed Oops: Enabled (32-bit)", + "[2023-12-06T07:32:54.117+0000][29ms] Heap Region Size: 1M", + "[2023-12-06T07:32:54.117+0000][29ms] Heap Min Capacity: 8M", + "[2023-12-06T07:32:54.117+0000][29ms] Heap Initial Capacity: 104M", + "[2023-12-06T07:32:54.117+0000][29ms] Heap Max Capacity: 1634M", + "[2023-12-06T07:32:54.117+0000][29ms] Pre-touch: Disabled", + "[2023-12-06T07:32:54.117+0000][29ms] Parallel Workers: 2", + "[2023-12-06T07:32:54.117+0000][29ms] Concurrent Workers: 1", + "[2023-12-06T07:32:54.117+0000][29ms] Concurrent Refinement Workers: 2", + "[2023-12-06T07:32:54.117+0000][29ms] Periodic GC: Disabled", + "[2023-12-06T07:32:54.117+0000][29ms] CDS archive(s) mapped at: [0x0000000800000000-0x0000000800bdf000-0x0000000800bdf000), size 12447744, SharedBaseAddress: 0x0000000800000000, ArchiveRelocationMode: 0.", + "[2023-12-06T07:32:54.117+0000][29ms] Compressed class space mapped at: 0x0000000800c00000-0x0000000840c00000, reserved size: 1073741824", + "[2023-12-06T07:32:54.117+0000][29ms] Narrow klass base: 0x0000000800000000, Narrow klass shift: 0, Narrow klass range: 0x100000000", + "[2023-12-06T07:32:54.359+0000][270ms] GC(0) Pause Young (Normal) (G1 Evacuation Pause)", + "[2023-12-06T07:32:54.359+0000][270ms] GC(0) Using 2 workers of 2 for evacuation", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Pre Evacuate Collection Set: 0.1ms", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Merge Heap Roots: 0.0ms", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Evacuate Collection Set: 5.2ms", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Post Evacuate Collection Set: 3.9ms", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Other: 0.2ms", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Eden regions: 6->0(8)", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Survivor regions: 0->1(1)", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Old regions: 0->0", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Archive regions: 2->2", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Humongous regions: 3->3", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Metaspace: 858K(1024K)->858K(1024K) NonClass: 780K(832K)->780K(832K) Class: 77K(192K)->77K(192K)", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Pause Young (Normal) (G1 Evacuation Pause) 9M->4M(106M) 9.598ms", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) User=0.00s Sys=0.01s Real=0.01s", + "[2023-12-06T07:32:54.696+0000][608ms] GC(1) Pause Young (Normal) (G1 Evacuation Pause)", + "[2023-12-06T07:32:54.696+0000][608ms] GC(1) Using 2 workers of 2 for evacuation", + "[2023-12-06T07:32:54.701+0000][612ms] GC(1) Pre Evacuate Collection Set: 0.1ms", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Merge Heap Roots: 0.0ms", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Evacuate Collection Set: 4.3ms", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Other: 0.1ms", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Eden regions: 8->0(60)", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Survivor regions: 1->2(2)", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Old regions: 0->0", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Archive regions: 2->2", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Humongous regions: 3->3", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Metaspace: 3551K(3712K)->3551K(3712K) NonClass: 3113K(3200K)->3113K(3200K) Class: 438K(512K)->438K(512K)", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Pause Young (Normal) (G1 Evacuation Pause) 12M->5M(106M) 4.985ms", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) User=0.00s Sys=0.00s Real=0.00s" + }; + + List jvmEvents = feedParser(lines); + + try { + Assertions.assertEquals(2, jvmEvents.size()); + Assertions.assertEquals(G1Young.class, jvmEvents.get(0).getClass()); + Assertions.assertEquals(G1Young.class, jvmEvents.get(1).getClass()); + } catch(Throwable t) { + fail(t); + } + } +} From 7a8e8f7ca80c2b1f21ff15a30afd06cfbd8b96b8 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Mon, 25 Mar 2024 19:00:13 -0700 Subject: [PATCH 17/34] refactor: add ability to parse very simple logs preunified logs --- .../com/microsoft/gctoolkit/jvm/Diarizer.java | 2 - .../parser/jvm/PreUnifiedDiarizer.java | 499 +++++++++--------- .../gctoolkit/parser/jvm/UnifiedDiarizer.java | 91 ++-- .../parser/GenerationalHeapParserTest.java | 34 ++ .../gctoolkit/parser/ParserTest.java | 4 + .../parser/UnifiedG1GCParserFragmentTest.java | 5 +- .../gctoolkit/parser/diary/LogDiaryTest.java | 2 +- .../parser/unittests/ParserTest.java | 2 - 8 files changed, 339 insertions(+), 300 deletions(-) diff --git a/api/src/main/java/com/microsoft/gctoolkit/jvm/Diarizer.java b/api/src/main/java/com/microsoft/gctoolkit/jvm/Diarizer.java index 4ddb8adf..89329a57 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/jvm/Diarizer.java +++ b/api/src/main/java/com/microsoft/gctoolkit/jvm/Diarizer.java @@ -22,8 +22,6 @@ public interface Diarizer { boolean completed(); - void fillInKnowns(); - boolean diarize(String line); } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java index b9398aee..d23815de 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java @@ -109,6 +109,7 @@ public PreUnifiedDiarizer() {} @Override public Diary getDiary() { + fillInKnowns(); return diary; } @@ -145,56 +146,56 @@ public int getMaxTenuringThreshold() { @Override public boolean hasJVMEvents() { - return getDiary().isApplicationStoppedTime() || - getDiary().isApplicationRunningTime() || - getDiary().isApplicationRunningTime() || - getDiary().isTLABData(); + return diary.isApplicationStoppedTime() || + diary.isApplicationRunningTime() || + diary.isApplicationRunningTime() || + diary.isTLABData(); } private boolean versionIsKnown() { - return (getDiary().isStateKnown(SupportedFlags.JDK80) && getDiary().isStateKnown(SupportedFlags.JDK70)) && getDiary().isStateKnown(SupportedFlags.PRE_JDK70_40); + return (diary.isStateKnown(SupportedFlags.JDK80) && diary.isStateKnown(SupportedFlags.JDK70)) && diary.isStateKnown(SupportedFlags.PRE_JDK70_40); } @Override public boolean completed() { - return getDiary().isComplete() || lineCount < 1 || (simpleCMSCycleDetected && (simpleCMSCycleDetected || simpleFullGCDetected)); + return diary.isComplete() || lineCount < 1 || (simpleCMSCycleDetected && (simpleCMSCycleDetected || simpleFullGCDetected)); } // Things that if we've not seen them, we know that they are false. - public void fillInKnowns() { + private void fillInKnowns() { if (simpleCMSCycleDetected) { - getDiary().setTrue(SupportedFlags.CMS); - getDiary().setFalse(SupportedFlags.SERIAL, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.CMS); + diary.setFalse(SupportedFlags.SERIAL, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80); // if we have age table or if no tenuring distribution, assume default ParNew/CMS - if (youngCollectionCount == 0 || ageTableDetected || !getDiary().isTenuringDistribution()) { - getDiary().setTrue(SupportedFlags.PARNEW); - getDiary().setFalse(SupportedFlags.DEFNEW); + if (youngCollectionCount == 0 || ageTableDetected || !diary.isTenuringDistribution()) { + diary.setTrue(SupportedFlags.PARNEW); + diary.setFalse(SupportedFlags.DEFNEW); } else { //CMS with no age table implies DEFNEW/CMS - getDiary().setTrue(SupportedFlags.DEFNEW); - getDiary().setFalse(SupportedFlags.PARNEW); + diary.setTrue(SupportedFlags.DEFNEW); + diary.setFalse(SupportedFlags.PARNEW); } } else if (simpleFullGCDetected) { if (ageTableDetected) { - getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.SERIAL); + diary.setTrue(SupportedFlags.PARNEW, SupportedFlags.SERIAL); } else { //at this point we can't tell if it's serial or parallel so assume defaults - getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); + diary.setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); } - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.SERIAL, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.SERIAL, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80); } //Not much information here, assume defaults else if (simpleParallelOrParNewDetected) { if (ageTableDetected) { - getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS); - getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); + diary.setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS); + diary.setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); } else { - getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); - getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS); + diary.setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); + diary.setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS); } - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.SERIAL, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.SERIAL, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80); } - getDiary().setFalse(SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION, SupportedFlags.PRINT_PROMOTION_FAILURE, SupportedFlags.PRINT_FLS_STATISTICS); + diary.setFalse(SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION, SupportedFlags.PRINT_PROMOTION_FAILURE, SupportedFlags.PRINT_FLS_STATISTICS); } /** @@ -222,7 +223,6 @@ public boolean diarize(String line) { version(line); details(line); } - return this.completed(); } @@ -249,23 +249,23 @@ private void timeOfFirstEvent(String line) { private boolean jvmActivityFlag(String line) { - if (getDiary().isStateKnown(SupportedFlags.APPLICATION_STOPPED_TIME, SupportedFlags.APPLICATION_CONCURRENT_TIME, SupportedFlags.TLAB_DATA)) + if (diary.isStateKnown(SupportedFlags.APPLICATION_STOPPED_TIME, SupportedFlags.APPLICATION_CONCURRENT_TIME, SupportedFlags.TLAB_DATA)) return false; if ((JVMPatterns.APPLICATION_STOP_TIME.parse(line) != null) || (JVMPatterns.SIMPLE_APPLICATION_STOP_TIME.parse(line) != null) || JVMPatterns.APPLICATION_STOP_TIME_WITH_STOPPING_TIME.parse(line) != null) { - getDiary().setTrue(SupportedFlags.APPLICATION_STOPPED_TIME); + diary.setTrue(SupportedFlags.APPLICATION_STOPPED_TIME); return true; } else if ((JVMPatterns.APPLICATION_TIME.parse(line) != null) || (JVMPatterns.SIMPLE_APPLICATION_TIME.parse(line) != null)) { - getDiary().setTrue(SupportedFlags.APPLICATION_CONCURRENT_TIME); + diary.setTrue(SupportedFlags.APPLICATION_CONCURRENT_TIME); return true; } else if (JVMPatterns.TLAB_CONT.parse(line) != null) { - getDiary().setTrue(SupportedFlags.TLAB_DATA); + diary.setTrue(SupportedFlags.TLAB_DATA); return true; } //This will be reported along size a collection so if we don't see them by 3nd collection.... // maybe some rubbish lines between collection and log so wait a bit longer than just the reporting of the first collection if (collectionCount > 1) { - getDiary().setFalse(SupportedFlags.APPLICATION_STOPPED_TIME, SupportedFlags.APPLICATION_CONCURRENT_TIME, SupportedFlags.TLAB_DATA); + diary.setFalse(SupportedFlags.APPLICATION_STOPPED_TIME, SupportedFlags.APPLICATION_CONCURRENT_TIME, SupportedFlags.TLAB_DATA); } return false; @@ -280,70 +280,73 @@ private void collector(String line) { GCLogTrace trace; - if ((!getDiary().isStateKnown(SupportedFlags.PARNEW, SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.G1GC)) || - !getDiary().isStateKnown(SupportedFlags.GC_DETAILS) || collectionCount < 3) { + if ((!diary.isStateKnown(SupportedFlags.PARNEW, SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.G1GC)) || + !diary.isStateKnown(SupportedFlags.GC_DETAILS) || collectionCount < 3) { if (line.contains("[PSYoungGen:")) { - getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION, SupportedFlags.RSET_STATS); + diary.setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION, SupportedFlags.RSET_STATS); collectionCount++; youngCollectionCount++; if ((trace = ParallelPatterns.PSYOUNGGEN.parse(line)) != null) { - getDiary().setTrue(SupportedFlags.GC_DETAILS); - getDiary().setFalse(SupportedFlags.TENURING_DISTRIBUTION); - getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC); + diary.setTrue(SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.TENURING_DISTRIBUTION); + diary.setFalse(SupportedFlags.PRINT_HEAP_AT_GC); setGCCause(trace.getGroup(3)); } else if (ParallelPatterns.PS_DETAILS_WITH_TENURING.parse(line) != null) { - getDiary().setTrue(SupportedFlags.GC_DETAILS); - getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC); + diary.setTrue(SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.PRINT_HEAP_AT_GC); } } else if (line.contains("ParNew")) { - getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); + diary.setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); collectionCount++; youngCollectionCount++; if ((trace = CMSPatterns.PARNEW.parse(line)) != null) { - getDiary().setTrue(SupportedFlags.GC_DETAILS); - getDiary().setFalse(SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); + diary.setTrue(SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); setGCCause(trace.getGroup(4)); } else if ((trace = CMSPatterns.PARNEW_TENURING.parse(line)) != null) { - getDiary().setTrue(SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION); - getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION); + diary.setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); setGCCause(trace.getGroup(3)); } else if ((trace = SimplePatterns.PARNEW_NO_DETAILS.parse(line)) != null) { - getDiary().setFalse(SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); + diary.setFalse(SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); setGCCause(trace.getGroup(3)); } else if ((trace = SimplePatterns.PARNEW_START.parse(line)) != null) { - getDiary().setFalse(SupportedFlags.GC_DETAILS, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC); + diary.setFalse(SupportedFlags.GC_DETAILS, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC); setGCCause(trace.groupCount() > 2 ? trace.getGroup(3) : null); } else if ((trace = CMSPatterns.PARNEW_REFERENCE_SPLIT.parse(line)) != null) { - getDiary().setTrue(SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.PRINT_REFERENCE_GC); + diary.setTrue(SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.PRINT_REFERENCE_GC); setGCCause(trace.getGroup(3)); } } else if ((trace = SerialPatterns.DEFNEW.parse(line)) != null) { collectionCount++; youngCollectionCount++; - getDiary().setTrue(SupportedFlags.DEFNEW, SupportedFlags.GC_DETAILS); - getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.PARNEW, SupportedFlags.G1GC, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.RSET_STATS, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); + diary.setTrue(SupportedFlags.DEFNEW, SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.PARNEW, SupportedFlags.G1GC, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.RSET_STATS, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); setGCCause(trace.getGroup(3)); } else if ((trace = SerialPatterns.DEFNEW_TENURING.parse(line)) != null) { collectionCount++; youngCollectionCount++; - getDiary().setTrue(SupportedFlags.DEFNEW, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION); - getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); + diary.setTrue(SupportedFlags.DEFNEW, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION); + diary.setFalse(SupportedFlags.PARNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); setGCCause(trace.getGroup(3)); } else if (SimplePatterns.PARNEW_NO_DETAILS.parse(line) != null) { collectionCount++; youngCollectionCount++; - getDiary().setTrue(SupportedFlags.PARNEW); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); + diary.setTrue(SupportedFlags.PARNEW); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); } else if (SimplePatterns.YOUNG_NO_DETAILS.parse(line) != null) { collectionCount++; youngCollectionCount++; simpleParallelOrParNewDetected = true; + diary.setFalse(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, + SupportedFlags.GC_CAUSE, SupportedFlags.JDK80, SupportedFlags.PRINT_HEAP_AT_GC, + SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION, SupportedFlags.ADAPTIVE_SIZING); } else if (SimplePatterns.FULL_NO_GC_DETAILS.parse(line) != null) { collectionCount++; simpleFullGCDetected = true; @@ -352,7 +355,7 @@ private void collector(String line) { collectionCount++; simpleParallelOrParNewDetected = true; youngCollectionCount++; - getDiary().setFalse(SupportedFlags.G1GC, SupportedFlags.RSET_STATS); + diary.setFalse(SupportedFlags.G1GC, SupportedFlags.RSET_STATS); setGCCause(trace.getGroup(3)); } else if (SimplePatterns.CMS_NO_DETAILS.parse(line) != null) { // could be parallel or CMS.. look for Full GC but even that may be a trick @@ -363,61 +366,61 @@ private void collector(String line) { } else if ((trace = G1GCPatterns.G1_YOUNG_SPLIT_START.parse(line)) != null) { collectionCount++; youngCollectionCount++; - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); - getDiary().setTrue(SupportedFlags.G1GC); - getDiary().setTrue(SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.G1GC); + diary.setTrue(SupportedFlags.GC_DETAILS); if (trace.gcCause() == GCCause.GCCAUSE_NOT_SET) { - getDiary().setFalse(SupportedFlags.GC_CAUSE); + diary.setFalse(SupportedFlags.GC_CAUSE); } else if (trace.gcCause() == GCCause.METADATA_GENERATION_THRESHOLD) { - getDiary().setTrue(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); - getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); } else if ((trace.gcCause() == GCCause.G1_EVACUATION_PAUSE) || (trace.gcCause(3, 0) == GCCause.G1_HUMONGOUS_ALLOCATION)) { - getDiary().setTrue(SupportedFlags.GC_CAUSE); - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.GC_CAUSE); + diary.setFalse(SupportedFlags.PRE_JDK70_40); } } else if ((trace = G1GCPatterns.G1_DETAILS.parse(line)) != null) { collectionCount++; youngCollectionCount++; - getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); + diary.setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); if (trace.gcCause(3, 0) == GCCause.GCCAUSE_NOT_SET) { - getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); - getDiary().setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70); } else { - getDiary().setFalse(SupportedFlags.GC_CAUSE); - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setFalse(SupportedFlags.GC_CAUSE); + diary.setFalse(SupportedFlags.PRE_JDK70_40); } } else if ((trace = G1GCPatterns.YOUNG.parse(line)) != null) { collectionCount++; youngCollectionCount++; - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); - getDiary().setTrue(SupportedFlags.G1GC); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); + diary.setTrue(SupportedFlags.G1GC); checkForGCCause(trace); //2014-10-21T11:49:08.954-0500: 12053.551: [GC pause (young)12054.116: [SoftReference, 0 refs, 0.0000070 secs]12054.116: [WeakReference, 234 refs, 0.0000640 secs]12054.116: [FinalReference, 3805 refs, 0.0034010 secs]12054.119: [PhantomReference, 9 refs, 0.0000040 secs]12054.119: [JNI Weak Reference, 0.0001960 secs], 0.58191800 secs] } else if ((trace = G1GCPatterns.G1_DETAILS_REFERENCE_GC.parse(line)) != null) { collectionCount++; youngCollectionCount++; - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); - getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.PRINT_REFERENCE_GC); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); + diary.setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.PRINT_REFERENCE_GC); if (trace.getGroup(3) != null) - getDiary().setTrue(SupportedFlags.GC_CAUSE); + diary.setTrue(SupportedFlags.GC_CAUSE); else - getDiary().setFalse(SupportedFlags.GC_CAUSE); + diary.setFalse(SupportedFlags.GC_CAUSE); } else if (G1GCPatterns.G1_INITIAL_MARK.parse(line) != null) { collectionCount++; youngCollectionCount++; - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); - getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.GC_CAUSE, SupportedFlags.TENURING_DISTRIBUTION); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); + diary.setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.GC_CAUSE, SupportedFlags.TENURING_DISTRIBUTION); } } - if (getDiary().isTrue(SupportedFlags.ADAPTIVE_SIZING)) { + if (diary.isTrue(SupportedFlags.ADAPTIVE_SIZING)) { if ((trace = G1GCPatterns.YOUNG_SPLIT_BY_G1ERGONOMICS.parse(line)) != null) { collectionCount++; youngCollectionCount++; - getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); checkForGCCause(trace); } } @@ -425,90 +428,90 @@ private void collector(String line) { //This has been verified true as of 1.7.0_51. Check later versions to make sure it hasn't been back ported if (ParallelPatterns.PS_FULL_GC_PERM.parse(line) != null) { collectionCount++; - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setFalse(SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.JDK80); if (line.contains(" [PSYoungGen: ")) - getDiary().setTrue(SupportedFlags.PARALLELGC); - getDiary().setTrue(SupportedFlags.PARALLELOLDGC, SupportedFlags.GC_DETAILS); - getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC); + diary.setTrue(SupportedFlags.PARALLELGC); + diary.setTrue(SupportedFlags.PARALLELOLDGC, SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC); - getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); - getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC); + diary.setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setFalse(SupportedFlags.PRINT_HEAP_AT_GC); } //This has been verified true as of 1.7.0_51. Check later versions to make sure it hasn't been back ported else if (ParallelPatterns.PS_FULL_GC_META.parse(line) != null) { collectionCount++; - getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); - getDiary().setTrue(SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.JDK80); if (line.contains(" [PSYoungGen: ")) - getDiary().setTrue(SupportedFlags.PARALLELGC); - getDiary().setTrue(SupportedFlags.PARALLELOLDGC, SupportedFlags.GC_DETAILS); - getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.SERIAL, SupportedFlags.ICMS, SupportedFlags.G1GC); + diary.setTrue(SupportedFlags.PARALLELGC); + diary.setTrue(SupportedFlags.PARALLELOLDGC, SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.SERIAL, SupportedFlags.ICMS, SupportedFlags.G1GC); - getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); - getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC); + diary.setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setFalse(SupportedFlags.PRINT_HEAP_AT_GC); } else if (line.contains("CMS-initial-mark")) { collectionCount++; - getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC); - getDiary().setTrue(SupportedFlags.CMS, SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC); + diary.setTrue(SupportedFlags.CMS, SupportedFlags.GC_DETAILS); } //todo: this rule is in the wrong place else if (CMSPatterns.SERIAL_FULL.parse(line) != null) { collectionCount++; - getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1); - getDiary().setTrue(SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS); + diary.setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS); //todo: private final int SupportedFlags.GC_CAUSE = 12; - getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); if (line.contains("Metaspace")) { - getDiary().setFalse(SupportedFlags.JDK70); - getDiary().setTrue(SupportedFlags.JDK80, SupportedFlags.GC_CAUSE); + diary.setFalse(SupportedFlags.JDK70); + diary.setTrue(SupportedFlags.JDK80, SupportedFlags.GC_CAUSE); } else if ( line.contains("Perm")) { // todo: maybe look for GC_CAUSE in JDK 7??? - getDiary().setFalse(SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.JDK80); } if (line.contains("Tenured")) { - getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS); - getDiary().setTrue(SupportedFlags.SERIAL); + diary.setFalse(SupportedFlags.CMS, SupportedFlags.ICMS); + diary.setTrue(SupportedFlags.SERIAL); } else if (line.contains("CMS")) { - getDiary().setFalse(SupportedFlags.SERIAL); - getDiary().setTrue(SupportedFlags.CMS); + diary.setFalse(SupportedFlags.SERIAL); + diary.setTrue(SupportedFlags.CMS); } } else if (CMSPatterns.SERIAL_FULL.parse(line) != null) { collectionCount++; - getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.JDK70); - getDiary().setTrue(SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS, SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.JDK70); + diary.setTrue(SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS, SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); } else if (TENURED_BLOCK.parse(line) != null) { collectionCount++; - getDiary().setFalse(SupportedFlags.CMS); - getDiary().setFalse(SupportedFlags.ICMS); - getDiary().setTrue(SupportedFlags.SERIAL); - getDiary().setFalse(SupportedFlags.G1GC); + diary.setFalse(SupportedFlags.CMS); + diary.setFalse(SupportedFlags.ICMS); + diary.setTrue(SupportedFlags.SERIAL); + diary.setFalse(SupportedFlags.G1GC); } - if (getDiary().isTrue(SupportedFlags.CMS) && !getDiary().isStateKnown(SupportedFlags.ICMS)) { + if (diary.isTrue(SupportedFlags.CMS) && !diary.isStateKnown(SupportedFlags.ICMS)) { if (firstCMSCycle) { if (line.contains("ParNew") || line.contains("DefNew")) youngCountAfterFirstCMSCycle++; if (line.contains("icms_dc")) - getDiary().setTrue(SupportedFlags.ICMS); + diary.setTrue(SupportedFlags.ICMS); else if (youngCountAfterFirstCMSCycle > 1) - getDiary().setFalse(SupportedFlags.ICMS); + diary.setFalse(SupportedFlags.ICMS); } //The first CMS cycle is needed to kick off iCMS if (line.contains("concurrent-reset")) firstCMSCycle = true; - } else if (getDiary().isTrue(SupportedFlags.G1GC)) { + } else if (diary.isTrue(SupportedFlags.G1GC)) { if (G1GCPatterns.G1_MEMORY_SUMMARY.parse(line) != null) { if (line.contains("Metaspace")) { - getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); - getDiary().setTrue(SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.JDK80); } else { - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setFalse(SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.JDK80); } } } @@ -516,10 +519,10 @@ else if (youngCountAfterFirstCMSCycle > 1) private void checkForGCCause(GCLogTrace trace) { if (trace.gcCause(3, 0) == GCCause.METADATA_GENERATION_THRESHOLD) { - getDiary().setTrue(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); - getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); } else if ((trace.gcCause(3, 0) == GCCause.G1_EVACUATION_PAUSE) || (trace.gcCause(3, 0) == GCCause.G1_HUMONGOUS_ALLOCATION)) { - getDiary().setTrue(SupportedFlags.GC_CAUSE); + diary.setTrue(SupportedFlags.GC_CAUSE); } } @@ -541,59 +544,59 @@ private void version(String line) { String value = trace.getGroup(1).trim(); if ("CMS Perm".equals(value)) { - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setFalse(SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.JDK80); } else if ("PS Perm".equals(value)) { - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setFalse(SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.JDK80); } else if ("Perm".equals(value)) { - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setFalse(SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.JDK80); } else if ("PSPermGen".equals(value)) { - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setFalse(SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.JDK80); } else if ("Metaspace".equals(value)) { - getDiary().setTrue(SupportedFlags.JDK80); - getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); } } else if (META_SPACE_RECORD.parse(line) != null) { - getDiary().setTrue(SupportedFlags.JDK80); - getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); } // maybe we'll get lucky, app server often call System.gc() after startup. if (line.contains("(System)")) { - getDiary().setTrue(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); - getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); + diary.setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); } else if (line.contains("(System.gc()")) { - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); - getDiary().setTrue(SupportedFlags.GC_CAUSE); - } else if (getDiary().isGenerationalKnown() && getDiary().isGenerational()) { + diary.setFalse(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.GC_CAUSE); + } else if (diary.isGenerationalKnown() && diary.isGenerational()) { if ((trace = PREFIX.parse(line)) != null) { - if ((trace.getGroup(3) == null) && getDiary().isTrue(SupportedFlags.GC_DETAILS)) { - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setFalse(SupportedFlags.JDK80, SupportedFlags.GC_CAUSE); + if ((trace.getGroup(3) == null) && diary.isTrue(SupportedFlags.GC_DETAILS)) { + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.JDK80, SupportedFlags.GC_CAUSE); } else if (trace.gcCause(3, 0) != GCCause.GCCAUSE_NOT_SET) { - getDiary().setTrue(SupportedFlags.GC_CAUSE); - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.GC_CAUSE); + diary.setFalse(SupportedFlags.PRE_JDK70_40); } } else if ((trace = FULL_PREFIX.parse(line)) != null) { - if ((trace.getGroup(3) == null) && getDiary().isTrue(SupportedFlags.GC_DETAILS)) { - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); + if ((trace.getGroup(3) == null) && diary.isTrue(SupportedFlags.GC_DETAILS)) { + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); } else if (trace.gcCause(3, 0) != GCCause.GCCAUSE_NOT_SET) { - getDiary().setTrue(SupportedFlags.GC_CAUSE); - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.GC_CAUSE); + diary.setFalse(SupportedFlags.PRE_JDK70_40); } } - } else if (getDiary().isG1GCKnown() && getDiary().isG1GC()) { + } else if (diary.isG1GCKnown() && diary.isG1GC()) { if ((trace = G1GC_PREFIX.parse(line)) != null) { - if (getDiary().isTrue(SupportedFlags.GC_DETAILS) && (trace.gcCause() == GCCause.GCCAUSE_NOT_SET)) { - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setFalse(SupportedFlags.JDK80); + if (diary.isTrue(SupportedFlags.GC_DETAILS) && (trace.gcCause() == GCCause.GCCAUSE_NOT_SET)) { + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.JDK80); } else { //we can't say much else unless we look for 8.0 specific details - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setFalse(SupportedFlags.PRE_JDK70_40); } } } @@ -612,73 +615,73 @@ private void details(String line) { if ((trace = PREFIX.parse(line)) != null) { String cause = trace.getGroup(3); if (cause != null) { - getDiary().setTrue(SupportedFlags.GC_CAUSE); + diary.setTrue(SupportedFlags.GC_CAUSE); if ("(System)".equals(cause)) { - getDiary().setFalse(SupportedFlags.JDK80); - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setTrue(SupportedFlags.PRE_JDK70_40); + diary.setFalse(SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70); + diary.setTrue(SupportedFlags.PRE_JDK70_40); } else if ("(System.gc())".equals(cause)) { - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setFalse(SupportedFlags.PRE_JDK70_40); } else { - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setFalse(SupportedFlags.PRE_JDK70_40); } } else { if ((trace = G1GC_PREFIX.parse(line)) != null) { cause = trace.getGroup(3); if (cause == null) - getDiary().setTrue(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.PRE_JDK70_40); else - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setFalse(SupportedFlags.PRE_JDK70_40); } else { - getDiary().setFalse(SupportedFlags.GC_CAUSE); + diary.setFalse(SupportedFlags.GC_CAUSE); } } - getDiary().setFalse(SupportedFlags.GC_CAUSE); + diary.setFalse(SupportedFlags.GC_CAUSE); } else if (line.contains("promotion failure size =")) - getDiary().setTrue(SupportedFlags.PRINT_PROMOTION_FAILURE); + diary.setTrue(SupportedFlags.PRINT_PROMOTION_FAILURE); else if (FLS_HEADER.parse(line) != null) { - getDiary().setTrue(SupportedFlags.PRINT_FLS_STATISTICS); + diary.setTrue(SupportedFlags.PRINT_FLS_STATISTICS); } //old G1 log file // [GC Worker Start (ms): 12053551.6 12053551.6 12053551.6 12053551.6 12053551.7 12053551.7 12053551.7 12053551.7 12053551.7 12053551.7 12053551.7 12053551.7 //new G1 log file // [GC Worker Start (ms): Min: 76.3, Avg: 76.3, Max: 76.4, Diff: 0.1] - if (getDiary().isG1GC()) { + if (diary.isG1GC()) { if (line.startsWith("[GC Worker Start (ms): ")) if (line.startsWith("[GC Worker Start (ms): Min: ")) - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setFalse(SupportedFlags.PRE_JDK70_40); else - getDiary().setTrue(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.PRE_JDK70_40); } else if (line.contains("AdaptiveSizePolicy::")) { - getDiary().setTrue(SupportedFlags.ADAPTIVE_SIZING); + diary.setTrue(SupportedFlags.ADAPTIVE_SIZING); } //if we've seen a collection and Print Reference GC hasn't been set... - if (youngCollectionCount > 1 && !getDiary().isStateKnown(SupportedFlags.PRINT_REFERENCE_GC)) { - getDiary().setFalse(SupportedFlags.PRINT_REFERENCE_GC); + if (youngCollectionCount > 1 && !diary.isStateKnown(SupportedFlags.PRINT_REFERENCE_GC)) { + diary.setFalse(SupportedFlags.PRINT_REFERENCE_GC); } GCLogTrace gcLogTrace; if ((gcLogTrace = MEMORY_SUMMARY_RULE.parse(line)) != null) { if (gcLogTrace.next() != null) - getDiary().setTrue(SupportedFlags.GC_DETAILS); + diary.setTrue(SupportedFlags.GC_DETAILS); } // if we've seen a statement than this is false if (line.startsWith("{Heap before GC invocations=")) - getDiary().setTrue(SupportedFlags.PRINT_HEAP_AT_GC); + diary.setTrue(SupportedFlags.PRINT_HEAP_AT_GC); else if (collectionCount > 1) - getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC); + diary.setFalse(SupportedFlags.PRINT_HEAP_AT_GC); if ((trace = TenuredPatterns.TENURING_SUMMARY.parse(line)) != null) { //we have seen at least one good tenuring summary without an age breakdown if ((tenuringSummary > 0) && (!ageTableDetected)) { - getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1); } else - getDiary().setTrue(SupportedFlags.TENURING_DISTRIBUTION); + diary.setTrue(SupportedFlags.TENURING_DISTRIBUTION); // if calculated tenuring threshold == 0 we won't get an age breakdown so delay evaluation if (trace.getIntegerGroup(2) > 0) @@ -687,47 +690,47 @@ else if (collectionCount > 1) //If the MaxTenuringThreshold is set to be greater than 15 then we a configuration bug to report on. if (trace.getIntegerGroup(3) > 15) { maxTenuringThreshold = trace.getIntegerGroup(3); - getDiary().setTrue(SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); + diary.setTrue(SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); } else - getDiary().setFalse(SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); + diary.setFalse(SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); } else if (TenuredPatterns.TENURING_AGE_BREAKDOWN.parse(line) != null) { ageTableDetected = true; - getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); - } else if (getDiary().isTenuringDistributionKnown() && collectionCount > 1 && youngCollectionCount > 1) { - getDiary().setFalse(SupportedFlags.TENURING_DISTRIBUTION); + diary.setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); + } else if (diary.isTenuringDistributionKnown() && collectionCount > 1 && youngCollectionCount > 1) { + diary.setFalse(SupportedFlags.TENURING_DISTRIBUTION); } if (line.contains("G1Ergonomics")) { if (line.contains("CSet Construction") || line.contains("Heap Sizing")) { collectionCount++; - getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.ADAPTIVE_SIZING); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.G1GC, SupportedFlags.ADAPTIVE_SIZING); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); } } if (REFERENCE_PROCESSING_BLOCK.parse(line) != null) - getDiary().setTrue(SupportedFlags.PRINT_REFERENCE_GC); + diary.setTrue(SupportedFlags.PRINT_REFERENCE_GC); if (line.startsWith("Concurrent RS processed")) { collectionCount++; - getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.RSET_STATS); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.G1GC, SupportedFlags.RSET_STATS); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); } else if (line.contains(" (cardTable: ")) { - getDiary().setTrue(SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.CMS_DEBUG_LEVEL_1); } if (collectionCount > 1) { - if (!getDiary().isCMSDebugLevel1Known()) { - getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); + if (!diary.isCMSDebugLevel1Known()) { + diary.setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); } - if (!getDiary().isAdaptiveSizingKnown()) { - getDiary().setFalse(SupportedFlags.ADAPTIVE_SIZING); + if (!diary.isAdaptiveSizingKnown()) { + diary.setFalse(SupportedFlags.ADAPTIVE_SIZING); } - if (!getDiary().isRSetStatsKnown()) - getDiary().setFalse(SupportedFlags.RSET_STATS); + if (!diary.isRSetStatsKnown()) + diary.setFalse(SupportedFlags.RSET_STATS); } } @@ -766,47 +769,47 @@ private void evaluateCommandLineFlags(String[] commandLineFlags) { if (supportedFlag != null) { switch (supportedFlag) { case PrintGCApplicationStoppedTime: - getDiary().setState(SupportedFlags.APPLICATION_STOPPED_TIME, flagTurnedOn); + diary.setState(SupportedFlags.APPLICATION_STOPPED_TIME, flagTurnedOn); break; case PrintGCApplicationConcurrentTime: - getDiary().setState(SupportedFlags.APPLICATION_CONCURRENT_TIME, flagTurnedOn); + diary.setState(SupportedFlags.APPLICATION_CONCURRENT_TIME, flagTurnedOn); break; case PrintGCTimeStamps: break; case PrintGCDetails: - getDiary().setState(SupportedFlags.GC_DETAILS, flagTurnedOn); - if (flagTurnedOn && (getDiary().isJDK80() || getDiary().isUnifiedLogging())) - getDiary().setTrue(SupportedFlags.GC_CAUSE); + diary.setState(SupportedFlags.GC_DETAILS, flagTurnedOn); + if (flagTurnedOn && (diary.isJDK80() || diary.isUnifiedLogging())) + diary.setTrue(SupportedFlags.GC_CAUSE); break; case PrintGCCause: - getDiary().setState(SupportedFlags.GC_CAUSE, flagTurnedOn); + diary.setState(SupportedFlags.GC_CAUSE, flagTurnedOn); break; case PrintTenuringDistribution: - getDiary().setState(SupportedFlags.TENURING_DISTRIBUTION, flagTurnedOn); + diary.setState(SupportedFlags.TENURING_DISTRIBUTION, flagTurnedOn); break; case PrintAdaptiveSizePolicy: - getDiary().setState(SupportedFlags.ADAPTIVE_SIZING, flagTurnedOn); + diary.setState(SupportedFlags.ADAPTIVE_SIZING, flagTurnedOn); break; case PrintReferenceGC: - getDiary().setState(SupportedFlags.PRINT_REFERENCE_GC, flagTurnedOn); + diary.setState(SupportedFlags.PRINT_REFERENCE_GC, flagTurnedOn); break; case PrintHeapAtGC: - getDiary().setState(SupportedFlags.PRINT_HEAP_AT_GC, flagTurnedOn); + diary.setState(SupportedFlags.PRINT_HEAP_AT_GC, flagTurnedOn); break; case PrintPromotionFailure: - getDiary().setTrue(SupportedFlags.PRINT_PROMOTION_FAILURE); + diary.setTrue(SupportedFlags.PRINT_PROMOTION_FAILURE); case PrintFLSStatistics: - getDiary().setTrue(SupportedFlags.PRINT_FLS_STATISTICS); + diary.setTrue(SupportedFlags.PRINT_FLS_STATISTICS); default: //shouldn't be able to get here.... @@ -852,7 +855,7 @@ private void evaluateCommandLineFlags(String[] commandLineFlags) { } evaluateGCLogFlags(); - getDiary().setFalse(SupportedFlags.APPLICATION_STOPPED_TIME, SupportedFlags.APPLICATION_CONCURRENT_TIME, SupportedFlags.GC_DETAILS, + diary.setFalse(SupportedFlags.APPLICATION_STOPPED_TIME, SupportedFlags.APPLICATION_CONCURRENT_TIME, SupportedFlags.GC_DETAILS, SupportedFlags.GC_CAUSE, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_REFERENCE_GC, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.PRINT_PROMOTION_FAILURE, SupportedFlags.PRINT_FLS_STATISTICS); } @@ -885,26 +888,26 @@ private void parseJVMVersion(String versionString) { if (matcher.find()) { switch (matcher.group(2).charAt(0)) { case '7': - getDiary().setTrue(SupportedFlags.JDK70); - getDiary().setFalse(SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.JDK80); if (matcher.group(3) == null) - getDiary().setTrue(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.PRE_JDK70_40); try { int minorVersion = Integer.parseInt(matcher.group(3).substring(1)); if (minorVersion < 41) - getDiary().setTrue(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.PRE_JDK70_40); else - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setFalse(SupportedFlags.PRE_JDK70_40); } catch (NumberFormatException nfe) { - getDiary().setTrue(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.PRE_JDK70_40); } break; case '8': - getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); - getDiary().setTrue(SupportedFlags.JDK80, SupportedFlags.GC_CAUSE); // doesn't matter so much but may only be true for later versions of 8 + diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.JDK80, SupportedFlags.GC_CAUSE); // doesn't matter so much but may only be true for later versions of 8 break; case '9': - getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80); break; default: } @@ -962,64 +965,64 @@ private void evaluateGCLogFlags() { case 4354: case 6146: case 16384: - getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1); break; case 1024: // parallel/serial case 1026: - getDiary().setTrue(SupportedFlags.PARALLELGC); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.PARALLELGC); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1); break; case 1: // DefNew/Serial - getDiary().setTrue(SupportedFlags.DEFNEW, SupportedFlags.SERIAL); - getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.DEFNEW, SupportedFlags.SERIAL); + diary.setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1); break; case 8: //ParNew/Serial Deprecated case 4101: - getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.SERIAL); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.PARNEW, SupportedFlags.SERIAL); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1); break; case 16: //ParNew/CMS case 24: case 272: - getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); + diary.setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); break; case 2064: //DefNew/CMS Deprecated - getDiary().setTrue(SupportedFlags.DEFNEW, SupportedFlags.CMS); - getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); + diary.setTrue(SupportedFlags.DEFNEW, SupportedFlags.CMS); + diary.setFalse(SupportedFlags.PARNEW, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); break; case 48: //ParNew/iCMS case 56: - getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); + diary.setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); break; case 2096: //DefNew/iCMS - getDiary().setTrue(SupportedFlags.DEFNEW, SupportedFlags.CMS, SupportedFlags.ICMS); - getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); + diary.setTrue(SupportedFlags.DEFNEW, SupportedFlags.CMS, SupportedFlags.ICMS); + diary.setFalse(SupportedFlags.PARNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); break; case 6208: case 64: //G1GC - getDiary().setTrue(SupportedFlags.G1GC); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.G1GC); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); break; case 4096: - getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); - getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC); + diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); break; case 4160: - getDiary().setTrue(SupportedFlags.G1GC); - getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); + diary.setTrue(SupportedFlags.G1GC); + diary.setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1); break; default: @@ -1028,21 +1031,21 @@ private void evaluateGCLogFlags() { } private void setGCCause(String gcCause) { - if (gcCause == null && getDiary().isPrintGCDetails()) { - getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); + if (gcCause == null && diary.isPrintGCDetails()) { + diary.setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); } else if (gcCause != null) { if (GCCauses.get(gcCause) == GCCause.GCCAUSE_NOT_SET) { - if (getDiary().isPrintGCDetailsKnown() && getDiary().isPrintGCDetails()) { - getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); - getDiary().setTrue(SupportedFlags.JDK70); + if (diary.isPrintGCDetailsKnown() && diary.isPrintGCDetails()) { + diary.setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); + diary.setTrue(SupportedFlags.JDK70); } } else { if (gcCause.contains("System.gc()") || !gcCause.contains("System")) { - getDiary().setTrue(SupportedFlags.GC_CAUSE); - getDiary().setFalse(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.GC_CAUSE); + diary.setFalse(SupportedFlags.PRE_JDK70_40); } else { - getDiary().setTrue(SupportedFlags.PRE_JDK70_40); - getDiary().setTrue(SupportedFlags.JDK70); + diary.setTrue(SupportedFlags.PRE_JDK70_40); + diary.setTrue(SupportedFlags.JDK70); } } } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/UnifiedDiarizer.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/UnifiedDiarizer.java index 1f0c4639..a2dc68a1 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/UnifiedDiarizer.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/UnifiedDiarizer.java @@ -41,9 +41,11 @@ public class UnifiedDiarizer implements Diarizer { public UnifiedDiarizer() {} public Diary getDiary() { + fillInKnowns(); return diary; } + @Override public boolean isUnified() { return true; @@ -62,19 +64,18 @@ public int getMaxTenuringThreshold() { @Override public boolean hasJVMEvents() { - return getDiary().isApplicationStoppedTime() || - getDiary().isApplicationRunningTime() || - getDiary().isTLABData(); + return diary.isApplicationStoppedTime() || + diary.isApplicationRunningTime() || + diary.isTLABData(); } - @Override - public void fillInKnowns() { - getDiary().setFalse(ADAPTIVE_SIZING); + private void fillInKnowns() { + diary.setFalse(ADAPTIVE_SIZING); } @Override public boolean completed() { - return getDiary().isComplete() || lineCount < 1; + return diary.isComplete() || lineCount < 1; } @Override @@ -84,11 +85,11 @@ public boolean diarize(String line) { return false; lineCount--; extractDecorators(line); - if (!getDiary().isCollectorKnown()) + if (!diary.isCollectorKnown()) discoverCollector(line); - if (!getDiary().isDetailsKnown()) + if (!diary.isDetailsKnown()) discoverDetails(line); - if (!getDiary().isJVMEventsKnown()) + if (!diary.isJVMEventsKnown()) discoverJVMEvents(line); if ((CPU_BREAKOUT.parse(line) != null) || line.contains("gc,start")) stopTheWorldEvents++; @@ -109,40 +110,40 @@ private void extractDecorators(String line) { if (decorators.getLogLevel().isPresent()) { UnifiedLoggingLevel logLevel = decorators.getLogLevel().get(); if (decorators.tagsContain("gc,age")) - getDiary().setTrue(TENURING_DISTRIBUTION); + diary.setTrue(TENURING_DISTRIBUTION); else if (decorators.tagsContain("ref") && logLevel.isGreaterThanOrEqualTo(UnifiedLoggingLevel.debug)) - getDiary().setTrue(PRINT_REFERENCE_GC); + diary.setTrue(PRINT_REFERENCE_GC); else if (decorators.tagsContain("gc,phases") && logLevel.isGreaterThanOrEqualTo(UnifiedLoggingLevel.debug)) - getDiary().setTrue(GC_DETAILS); + diary.setTrue(GC_DETAILS); else if ( decorators.tagsContain("gc,ergo")) - getDiary().setTrue(ADAPTIVE_SIZING); + diary.setTrue(ADAPTIVE_SIZING); if (decorators.tagsContain("safepoint")) - getDiary().setTrue(APPLICATION_STOPPED_TIME, APPLICATION_CONCURRENT_TIME); + diary.setTrue(APPLICATION_STOPPED_TIME, APPLICATION_CONCURRENT_TIME); - if (getDiary().isZGC()) { + if (diary.isZGC()) { if (decorators.tagsContain("task")) - getDiary().setTrue(GC_DETAILS); + diary.setTrue(GC_DETAILS); else if (decorators.tagsContain("heap")) - getDiary().setTrue(PRINT_HEAP_AT_GC); + diary.setTrue(PRINT_HEAP_AT_GC); else if (decorators.tagsContain("tlab")) - getDiary().setTrue(TLAB_DATA); + diary.setTrue(TLAB_DATA); else if (decorators.tagsContain("gc,start") && line.contains("Garbage Collection (")) - getDiary().setTrue(GC_CAUSE); + diary.setTrue(GC_CAUSE); else if (decorators.tagsContain("gc,heap")) { if (line.contains("Heap before GC")) - getDiary().setTrue(PRINT_HEAP_AT_GC); - getDiary().setTrue(GC_DETAILS); + diary.setTrue(PRINT_HEAP_AT_GC); + diary.setTrue(GC_DETAILS); } else if (decorators.tagsContain("gc,ref")) - getDiary().setTrue(PRINT_REFERENCE_GC); + diary.setTrue(PRINT_REFERENCE_GC); else if (decorators.tagsContain("gc,heap") && decorators.getLogLevel().get() == UnifiedLoggingLevel.debug) - getDiary().setTrue(PRINT_HEAP_AT_GC); - } else if (getDiary().isShenandoah()) { + diary.setTrue(PRINT_HEAP_AT_GC); + } else if (diary.isShenandoah()) { if (decorators.tagsContain("gc,task") || decorators.tagsContain("gc,start")) - getDiary().setTrue(GC_DETAILS); + diary.setTrue(GC_DETAILS); else if (decorators.tagsContain("gc,ergo")) - getDiary().setTrue(ADAPTIVE_SIZING); + diary.setTrue(ADAPTIVE_SIZING); else if (decorators.tagsContain("gc") && line.contains("Trigger")) - getDiary().setTrue(GC_CAUSE); + diary.setTrue(GC_CAUSE); } } } @@ -170,42 +171,42 @@ private void extractTagsAndLevels(Decorators decorators) { private void discoverCollector(String line) { if ( ZGC_TAG.parse(line) != null || CYCLE_START.parse(line) != null) { - getDiary().setTrue(ZGC); - getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, G1GC, RSET_STATS, SHENANDOAH, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TENURING_DISTRIBUTION, MAX_TENURING_THRESHOLD_VIOLATION, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS); + diary.setTrue(ZGC); + diary.setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, G1GC, RSET_STATS, SHENANDOAH, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TENURING_DISTRIBUTION, MAX_TENURING_THRESHOLD_VIOLATION, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS); return; } if ( SHENANDOAH_TAG.parse(line) != null) { - getDiary().setTrue(SHENANDOAH); - getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, G1GC, RSET_STATS, ZGC, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TENURING_DISTRIBUTION, MAX_TENURING_THRESHOLD_VIOLATION, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS, PRINT_HEAP_AT_GC); + diary.setTrue(SHENANDOAH); + diary.setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, G1GC, RSET_STATS, ZGC, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TENURING_DISTRIBUTION, MAX_TENURING_THRESHOLD_VIOLATION, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS, PRINT_HEAP_AT_GC); return; } if (G1_TAG.parse(line) != null || line.contains("G1 Evacuation Pause") || (line.contains("Humongous regions: "))) { - getDiary().setTrue(G1GC); - getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, ZGC, SHENANDOAH, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS); + diary.setTrue(G1GC); + diary.setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, ZGC, SHENANDOAH, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS); return; } if (CMS_TAG.parse(line) != null || PARNEW_TAG.parse(line) != null || line.contains("ParNew")) { - getDiary().setTrue(PARNEW, CMS); - getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS); + diary.setTrue(PARNEW, CMS); + diary.setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS); return; } if (PARALLEL_TAG.parse(line) != null || line.contains("ParOldGen") || line.contains("PSYoungGen")) { - getDiary().setTrue(PARALLELGC, PARALLELOLDGC, GC_CAUSE); - getDiary().setFalse(DEFNEW, SERIAL, PARNEW, CMS, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS); + diary.setTrue(PARALLELGC, PARALLELOLDGC, GC_CAUSE); + diary.setFalse(DEFNEW, SERIAL, PARNEW, CMS, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS); return; } if (SERIAL_TAG.parse(line) != null || line.contains("DefNew")) { - getDiary().setTrue(DEFNEW, SERIAL, GC_CAUSE); - getDiary().setFalse(PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS); + diary.setTrue(DEFNEW, SERIAL, GC_CAUSE); + diary.setFalse(PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS); return; } } @@ -218,26 +219,26 @@ private void discoverDetails(String line) { //todo: RSET_STATS for G1 not looked for... if (G1_COLLECTION.parse(line) != null) { - getDiary().setTrue(GC_CAUSE); + diary.setTrue(GC_CAUSE); } else if (CYCLE_START.parse(line) != null) { - getDiary().setTrue(GC_CAUSE); + diary.setTrue(GC_CAUSE); } if (stopTheWorldEvents > CYCLES_TO_EXAMINE_BEFORE_GIVING_UP) - getDiary().setFalse(ADAPTIVE_SIZING, GC_CAUSE, TLAB_DATA, PRINT_REFERENCE_GC, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS, RSET_STATS, PRINT_HEAP_AT_GC); + diary.setFalse(ADAPTIVE_SIZING, GC_CAUSE, TLAB_DATA, PRINT_REFERENCE_GC, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS, RSET_STATS, PRINT_HEAP_AT_GC); } private void discoverJVMEvents(String line) { if (stopTheWorldEvents > CYCLES_TO_EXAMINE_FOR_SAFEPOINT) { - getDiary().setFalse(APPLICATION_STOPPED_TIME, APPLICATION_CONCURRENT_TIME); + diary.setFalse(APPLICATION_STOPPED_TIME, APPLICATION_CONCURRENT_TIME); } } @Override public DateTimeStamp getTimeOfFirstEvent() { - return getDiary().getTimeOfFirstEvent(); + return diary.getTimeOfFirstEvent(); } } \ No newline at end of file diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java index 7d22b90c..76c06e4f 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java @@ -159,6 +159,40 @@ public void testThat2CMFDoNotHappen() { } } + @Test + public void basicLogTest() { + String[] lines = { + "9.089: [GC 17024K->1751K(2095040K), 0.0308466 secs", + "21.464: [GC 18775K->3413K(2095040K), 0.0431857 secs", + "29.141: [GC 20437K->2693K(2095040K), 0.0045342 secs", + "33.155: [GC 19717K->3974K(2095040K), 0.0115157 secs", + "36.966: [GC 20998K->5300K(2095040K), 0.0269848 secs", + "40.351: [GC 22324K->4799K(2095040K), 0.0071082 secs", + "43.373: [GC 21823K->5862K(2095040K), 0.0105840 secs", + "46.464: [GC 22886K->6859K(2095040K), 0.0289186 secs", + "49.185: [GC 23883K->8778K(2095040K), 0.0303312 secs", + "51.113: [GC 25802K->8674K(2095040K), 0.0119712 secs", + "54.218: [GC 25698K->9720K(2095040K), 0.0098641 secs", + "57.311: [GC 26744K->11179K(2095040K), 0.0212367 secs", + "60.249: [GC 28203K->11381K(2095040K), 0.0087235 secs", + "62.602: [GC 28405K->12469K(2095040K), 0.0258451 secs", + "65.245: [GC 29493K->13720K(2095040K), 0.0188990 secs", + "67.554: [GC 30744K->15788K(2095040K), 0.0296812 secs", + "69.923: [GC 32812K->16455K(2095040K), 0.0232905 secs", + "72.100: [GC 33479K->17013K(2095040K), 0.0100926 secs", + "74.262: [GC 34037K->18471K(2095040K), 0.0247458 secs", + "76.203: [GC 35495K->20250K(2095040K), 0.0270935 secs", + "78.319: [GC 37274K->21322K(2095040K), 0.0234226 secs", + "80.257: [GC 38346K->21604K(2095040K), 0.0097355 secs", + "82.298: [GC 38628K->22911K(2095040K), 0.0228335 secs", + "84.129: [GC 39935K->23835K(2095040K), 0.0203206 secs" + }; + + List jvmEvents = feedParser(lines); + assertEquals(24, jvmEvents.size()); + assertEquals(9.089,getParser().diary.getTimeOfFirstEvent().toSeconds()); + } + @Override protected Diarizer diarizer() { diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/ParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/ParserTest.java index a4ae4f89..9813e801 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/ParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/ParserTest.java @@ -44,6 +44,10 @@ public void setUp() { */ protected abstract GCLogParser parser(); + GCLogParser getParser() { + return this.parser; + } + /** * Parser runs in its own thread so start one for it and then feed it the lines to be parsed * diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java index bc83b558..b63ec483 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java @@ -32,8 +32,8 @@ protected GCLogParser parser() { @Test public void testNewDecoratorCombination() { String[] lines = { + "[2023-12-06T07:32:54.113+0000][25ms] Using G1", "[2023-12-06T07:32:54.117+0000][29ms] Version: 17.0.2+8-86 (release)", - "[2023-12-06T07:32:54.117+0000][25ms] Using G1", "[2023-12-06T07:32:54.117+0000][29ms] CPUs: 2 total, 2 available", "[2023-12-06T07:32:54.117+0000][29ms] Memory: 6531M", "[2023-12-06T07:32:54.117+0000][29ms] Large Page Support: Disabled", @@ -55,7 +55,7 @@ public void testNewDecoratorCombination() { "[2023-12-06T07:32:54.359+0000][270ms] GC(0) Using 2 workers of 2 for evacuation", "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Pre Evacuate Collection Set: 0.1ms", "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Merge Heap Roots: 0.0ms", - "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Evacuate Collection Set: 5.2ms", + "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Evacuate Collection Set: 5.2ms:", "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Post Evacuate Collection Set: 3.9ms", "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Other: 0.2ms", "[2023-12-06T07:32:54.368+0000][280ms] GC(0) Eden regions: 6->0(8)", @@ -71,6 +71,7 @@ public void testNewDecoratorCombination() { "[2023-12-06T07:32:54.701+0000][612ms] GC(1) Pre Evacuate Collection Set: 0.1ms", "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Merge Heap Roots: 0.0ms", "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Evacuate Collection Set: 4.3ms", + "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Post Evacuate Collection Set: 0.3ms", "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Other: 0.1ms", "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Eden regions: 8->0(60)", "[2023-12-06T07:32:54.701+0000][613ms] GC(1) Survivor regions: 1->2(2)", diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/diary/LogDiaryTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/diary/LogDiaryTest.java index 11a77ebd..7cf36fb1 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/diary/LogDiaryTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/diary/LogDiaryTest.java @@ -13,6 +13,7 @@ import java.io.File; import java.io.IOException; import java.util.Objects; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.fail; @@ -32,7 +33,6 @@ protected Diarizer getJVMConfiguration(GCLogFile gcLogFile) throws IOException { filter(completed -> completed). findFirst(); - jvmConfiguration.fillInKnowns(); return jvmConfiguration; } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java index 43cfb18c..e925e55f 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ParserTest.java @@ -111,8 +111,6 @@ private Diarizer getJVMConfiguration(GCLogFile gcLogFile) { map(jvmConfiguration::diarize). filter(completed -> completed). findFirst(); - - jvmConfiguration.fillInKnowns(); return jvmConfiguration; } catch (IOException e) { fail(e.getMessage()); From ac853e70b29676ee14d5ca2273d60aa8bbd38104 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Mon, 25 Mar 2024 19:35:11 -0700 Subject: [PATCH 18/34] refactor: remove no longer needed print statement --- .../gctoolkit/integration/aggregation/CMSCycleAggregator.java | 1 - .../com/microsoft/gctoolkit/integration/CMSEventsTest.java | 4 ++-- .../microsoft/gctoolkit/parser/GenerationalHeapParser.java | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java index b37455ef..04dde4e7 100644 --- a/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java +++ b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java @@ -26,7 +26,6 @@ public void count(InitialMark event) { } public void count(CMSRemark event) { - System.out.println(event.toString()); if ( event.equals(lastRemark)) return; lastRemark = event; aggregation().remark(); diff --git a/IT/src/test/java/com/microsoft/gctoolkit/integration/CMSEventsTest.java b/IT/src/test/java/com/microsoft/gctoolkit/integration/CMSEventsTest.java index 6a8f6227..ba5af07a 100644 --- a/IT/src/test/java/com/microsoft/gctoolkit/integration/CMSEventsTest.java +++ b/IT/src/test/java/com/microsoft/gctoolkit/integration/CMSEventsTest.java @@ -51,8 +51,8 @@ public void analyze(String gcLogFile) { machine.getAggregation(CMSCycleAggregation.class).ifPresent(cmsCycleCounts -> { Assertions.assertEquals( 1, cmsCycleCounts.getInitialMark(), "Initial Mark events count"); - //Assertions.assertEquals( 1, cmsCycleCounts.getRemark(), "Remark events count"); - //Assertions.assertEquals( 4, cmsCycleCounts.getConcurrentEvent(), "concurrent phase events count"); + Assertions.assertEquals( 1, cmsCycleCounts.getRemark(), "Remark events count"); + Assertions.assertEquals( 5, cmsCycleCounts.getConcurrentEvent(), "concurrent phase events count"); }); } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java index b6ff0f89..9e2610ca 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java @@ -2056,6 +2056,8 @@ private void log(String line) { // if (line.startsWith("}")) return; // if (line.startsWith("Heap")) return; // if (line.startsWith("[Times: user")) return; + if ( line.startsWith("Metaspace used")) return; + if ( line.startsWith("class space used")) return; if (line.startsWith("par new generation total")) return; // if (line.startsWith("concurrent mark-sweep generation total")) return; if (line.startsWith("concurrent-mark-sweep perm gen total")) return; From 39afac31bdf1c50b78e5c5275cd534c1369c91e5 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Tue, 26 Mar 2024 07:59:50 -0700 Subject: [PATCH 19/34] refactor: tidy from review --- .../gctoolkit/integration/aggregation/CMSCycleAggregator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java index 04dde4e7..d4d4f9a7 100644 --- a/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java +++ b/IT/src/main/java/com/microsoft/gctoolkit/integration/aggregation/CMSCycleAggregator.java @@ -7,6 +7,7 @@ import com.microsoft.gctoolkit.event.generational.CMSRemark; import com.microsoft.gctoolkit.event.generational.InitialMark; + @Aggregates({EventSource.GENERATIONAL}) public class CMSCycleAggregator extends Aggregator { From 12a17da38bd150f26a12e3ea19434e8f51d88915 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Wed, 10 Apr 2024 17:42:18 -0700 Subject: [PATCH 20/34] refactor: support date stamps only for preunified logs --- .../gctoolkit/parser/AbstractLogTrace.java | 34 +- .../gctoolkit/parser/CMSPatterns.java | 3 +- .../parser/CMSTenuredPoolParser.java | 15 +- .../gctoolkit/parser/G1GCPatterns.java | 12 +- .../gctoolkit/parser/G1GCTokens.java | 2 +- .../gctoolkit/parser/GCLogParser.java | 62 +- .../gctoolkit/parser/GCLogTrace.java | 20 +- .../parser/GenerationalHeapParser.java | 535 +++++++++--------- .../gctoolkit/parser/GenericTokens.java | 49 +- .../parser/PreUnifiedG1GCParser.java | 50 +- .../parser/PreUnifiedGCLogParser.java | 39 +- .../gctoolkit/parser/PreUnifiedTokens.java | 15 +- .../gctoolkit/parser/UnifiedG1GCParser.java | 3 +- .../parser/jvm/PreUnifiedDiarizer.java | 40 +- ...ncurrentMarkSweepPhaseParserRulesTest.java | 71 ++- .../gctoolkit/parser/G1GCParserRulesTest.java | 44 +- .../PreUnifiedCMSTenuredParserTest.java | 5 +- .../PreUnifiedGenerationalParserTest.java | 5 +- .../parser/patterns/G1GCPatternsTest.java | 29 +- .../parser/patterns/GCCausePatternTest.java | 95 ++++ .../parser/unittests/CMSParNewParserTest.java | 10 +- .../unittests/ICMSParNewParserTest.java | 13 +- .../unittests/PreunifiedG1GCParserTest.java | 22 +- 23 files changed, 635 insertions(+), 538 deletions(-) rename parser/src/test/java/com/microsoft/gctoolkit/parser/{patterns => }/PreUnifiedCMSTenuredParserTest.java (98%) rename parser/src/test/java/com/microsoft/gctoolkit/parser/{patterns => }/PreUnifiedGenerationalParserTest.java (98%) create mode 100644 parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/GCCausePatternTest.java diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java index dc238e74..63512616 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java @@ -64,13 +64,12 @@ public double getTimeStamp() { if (!matcher.find()) { return MISSING_TIMESTAMP_SENTINEL; } - return getDoubleGroup(2); + return convertToDouble(matcher.group(1)); } public String getDateStamp() { - Matcher matcher = DATE_TIME_STAMP_RULE.matcher(trace.group(0)); - if (matcher.find()) { - return matcher.group(1); + if (trace.find()) { + return trace.group(4); } else { return null; } @@ -96,16 +95,23 @@ public DateTimeStamp getDateTimeStamp() { * @return the nth date timestamp pairing */ public DateTimeStamp getDateTimeStamp(int nth) { - Matcher matcher = DATE_TIME_STAMP_RULE.matcher(trace.group(0)); - for (int i = 0; i < nth - 1; i++) - if (!matcher.find()) - break; - if (matcher.find()) { - if (matcher.group(2) == null) { - return new DateTimeStamp(matcher.group(1)); - } - return new DateTimeStamp(matcher.group(1), convertToDouble(matcher.group(2))); - } + Matcher matcher; + if ( nth > 1) { + matcher = DATE_TIME_STAMP_RULE.matcher(trace.group(0)); + for (int i = 0; i < nth; i++) + if (!matcher.find()) + break; + } else + matcher = trace; + + //if (matcher.find()) { + String timeStamp = ( matcher.group(3) == null) ? matcher.group(4) : matcher.group(3); + String dateStamp = ( matcher.group(2) == null) ? matcher.group(5) : matcher.group(2); + if (timeStamp != null) { + return new DateTimeStamp(dateStamp, convertToDouble(timeStamp)); + } else if ( dateStamp != null) + return new DateTimeStamp(dateStamp); + //} return new DateTimeStamp(MISSING_TIMESTAMP_SENTINEL); } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java index 87fb14c9..33cf032c 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java @@ -41,6 +41,7 @@ public interface CMSPatterns extends SharedPatterns { /********** CMS Phase records **********/ //3.307: [GC [1 CMS-initial-mark: 0K(18874368K)] 302009K(20761856K), 0.0994470 secs] [Times: user=0.20 sys=0.00, real=0.10 secs] //12.986: [GC[1 CMS-initial-mark: 33532K(62656K)] 49652K(81280K), 0.0014191 secs] [Times: user=0.00 sys=0.00, real=0.00 secs] + //40.971: [GC (CMS Initial Mark) [1 CMS-initial-mark: 2692K(5376K)] 38078K(354944K), 0.0147940 secs] [Times: user=0.01 sys=0.00, real=0.02 secs] GCParseRule INITIAL_MARK = new GCParseRule("INITIAL_MARK", GC_PREFIX + "\\[1 CMS-initial-mark: " + OCCUPANCY_CONFIGURED + "\\] " + OCCUPANCY_CONFIGURED_PAUSE + "\\]"); GCParseRule CONCURRENT_PHASE_START = new GCParseRule("CONCURRENT_PHASE_START", "^" + CMS_PHASE_START); @@ -309,7 +310,7 @@ public interface CMSPatterns extends SharedPatterns { GCParseRule RESCAN_CARDS_DETAILS = new GCParseRule("RESCAN_CARDS_DETAILS", "^, " + PAUSE_TIME + "\\]" + WEAK_REF_BLOCK + " " + REMARK_BLOCK); //, 0.2020511 secs]34.132: [weak refs processing, 0.0000108 secs]34.132: [class unloading, 0.0026382 secs]34.135: [scrub symbol & string tables, 0.0026847 secs] [1 CMS-remark: 0K(4194304K)] 889188K(5872064K), 0.2081265 secs] - GCParseRule RESCAN_SPLIT_UNLOADING_STRING = new GCParseRule("RESCAN_SPLIT_UNLOADING_STRING", "^, " + TIMESTAMP + "\\]" + WEAK_REF_BLOCK + CLASS_UNLOADING_BLOCK + STRING_AND_SYMBOL_SCRUB_BLOCK + " " + REMARK_BLOCK); + GCParseRule RESCAN_SPLIT_UNLOADING_STRING = new GCParseRule("RESCAN_SPLIT_UNLOADING_STRING", "^, " + PAUSE_TIME + "\\]" + WEAK_REF_BLOCK + CLASS_UNLOADING_BLOCK + STRING_AND_SYMBOL_SCRUB_BLOCK + " " + REMARK_BLOCK); //, 1.2157670 secs]42102.265: [weak refs processing, 0.1038920 secs]Work queue overflow (benign) (pmc_rm=1542597, kac=0) GCParseRule RESCAN_OVERFLOW_DETAILS = new GCParseRule("RESCAN_OVERFLOW_DETAILS", "^, " + PAUSE_TIME + "\\]" + WEAK_REF_BLOCK + "Work queue overflow"); diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java index 241d8660..b5453cf0 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java @@ -73,8 +73,8 @@ else if ((trace = EndOfFile.parse(line)) != null) { */ private void initialMark(GCLogTrace trace) { InitialMark initialMark = new InitialMark(trace.getDateTimeStamp(), GCCause.UNKNOWN_GCCAUSE, trace.getDoubleGroup(trace.groupCount())); - MemoryPoolSummary tenured = trace.getOccupancyWithMemoryPoolSizeSummary(4); - MemoryPoolSummary heap = trace.getOccupancyWithMemoryPoolSizeSummary(8); + MemoryPoolSummary tenured = trace.getOccupancyWithMemoryPoolSizeSummary(7); + MemoryPoolSummary heap = trace.getOccupancyWithMemoryPoolSizeSummary(11); initialMark.add(heap.minus(tenured), tenured, heap); publish(initialMark); } @@ -90,6 +90,7 @@ private void endOfConcurrentPhase(GCLogTrace trace) { } private void endConcurrentPrecleanWithReferenceProcessing(GCLogTrace trace) { + trace.notYetImplemented(); try { publish(new ConcurrentPreClean(startOfPhase, trace.getDoubleGroup(14) - startOfPhase.getTimeStamp(), trace.getDoubleGroup(16), trace.getDoubleGroup(17))); } catch (Throwable t) { @@ -98,9 +99,9 @@ private void endConcurrentPrecleanWithReferenceProcessing(GCLogTrace trace) { } private void endOfConcurrentPhase(GCLogTrace trace, DateTimeStamp timeStamp) { - String phase = trace.getGroup(3); - double cpuTime = trace.getDoubleGroup(4); - double wallTime = trace.getDoubleGroup(5); + String phase = trace.getGroup(6); + double cpuTime = trace.getDoubleGroup(7); + double wallTime = trace.getDoubleGroup(8); double duration = timeStamp.getTimeStamp() - startOfPhase.getTimeStamp(); if ("mark".equals(phase)) publish(new ConcurrentMark(startOfPhase, duration, cpuTime, wallTime)); @@ -118,8 +119,8 @@ else if ("reset".equals(phase)) private void abortPrecleanDueToTime(GCLogTrace trace) { try { - double cpuTime = trace.getDoubleGroup(4); - double wallClock = trace.getDoubleGroup(5); + double cpuTime = trace.getDoubleGroup(7); + double wallClock = trace.getDoubleGroup(8); publish(new AbortablePreClean(startOfPhase, trace.getDateTimeStamp().getTimeStamp() - startOfPhase.getTimeStamp(), cpuTime, wallClock, true)); } catch (Exception e) { LOG.warning("concurrent phase end choked on " + trace); diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCPatterns.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCPatterns.java index 64817b5c..cbc111c8 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCPatterns.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCPatterns.java @@ -222,15 +222,17 @@ public interface G1GCPatterns extends G1GCTokens { //Concurrent phase //549.246: [GC concurrent-root-region-scan-start] + //657.426: [GC concurrent-root-region-scan-end, 0.0052220 secs] //549.246: [GC concurrent-mark-start] - GCParseRule G1_CONCURRENT_START = new GCParseRule("G1_CONCURRENT_START", "^" + GC_PREFIX + "concurrent-(.+)-start\\]"); - GCParseRule G1_CONCURRENT_START_WITHOUT_PREFIX = new GCParseRule("G1_CONCURRENT_START_WITHOUT_PREFIX", "^" + "concurrent-(.+)-start\\]"); + GCParseRule G1_CONCURRENT_START = new GCParseRule("G1_CONCURRENT_START", "^" + GC_PREFIX + "concurrent-(.+)-start\\]$"); + GCParseRule G1_CONCURRENT_START_WITHOUT_PREFIX = new GCParseRule("G1_CONCURRENT_START_WITHOUT_PREFIX", "^" + "concurrent-(.+)-start\\]$"); //549.246: [GC concurrent-root-region-scan-end, 0.0000700 secs] //549.251: [GC concurrent-mark-end, 0.0055240 secs] GCParseRule G1_CONCURRENT_END = new GCParseRule("G1_CONCURRENT_END", "^" + GC_PREFIX + "concurrent-(.+)-end, " + PAUSE_TIME); GCParseRule G1_CORRUPTED_CONCURRENT_END = new GCParseRule("G1_CORRUPTED_CONCURRENT_END", "^\\[GC concurrent-(.+)-end, " + PAUSE_TIME); - GCParseRule G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END = new GCParseRule("G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END", DATE_TIMESTAMP + DATE_STAMP + "\\[GC concurrent-(.+)-end, " + PAUSE_TIME); + //GCParseRule G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END = new GCParseRule("G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END", "^" + DATE_TIMESTAMP + DATE_STAMP + "\\[GC concurrent-(.+)-end, " + PAUSE_TIME); + GCParseRule G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END = new GCParseRule("G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END", "^" + DATE_TIMESTAMP + GC_PREFIX + "concurrent-(.+)-end, " + PAUSE_TIME); GCParseRule G1_FLOATING_CONCURRENT_PHASE_START = new GCParseRule("G1_FLOATING_CONCURRENT_PHASE_START", "^\\[GC concurrent(.+)\\]$"); GCParseRule G1_FULL_INTERRUPTS_CONCURRENT_CYCLE = new GCParseRule("G1_FULL_INTERRUPTS_CONCURRENT_CYCLE", FULL_GC_PREFIX + GC_PREFIX + " concurrent-(.+)-end, " + PAUSE_TIME); //Strange Full GC corruption @@ -263,8 +265,6 @@ public interface G1GCPatterns extends G1GCTokens { GCParseRule CORRUPTED_CONCURRENT_START_V4 = new GCParseRule("CORRUPTED_CONCURRENT_START_V4", DATE_TIMESTAMP + DATE_STAMP + "\\[Full GC " + GC_CAUSE + GC_PREFIX + "concurrent-(.+)-start\\]"); //2018-02-08T20:21:26.246+0000: 2018-02-08T20:21:26.246+0000879136.850: [Full GC (Metadata GC Threshold) : 879136.850: [GC concurrent-root-region-scan-start] GCParseRule CORRUPTED_CONCURRENT_START_V5 = new GCParseRule("CORRUPTED_CONCURRENT_START_V5", DATE_STAMP + DATE + FULL_GC_PREFIX + ": " + GC_PREFIX + "concurrent-(.+)-start\\]"); - //2018-02-08T20:27:32.297+0000: 879502.901: 2018-02-08T20:27:32.297+0000: 879502.901: [GC concurrent-root-region-scan-end, 0.0000375 secs] - GCParseRule CORRUPTED_CONCURRENT_START_V6 = new GCParseRule("CORRUPTED_CONCURRENT_START_V6", DATE_TIMESTAMP + GC_PREFIX + "concurrent-(.+)-end, " + PAUSE_TIME); //2018-02-08T20:31:33.633+0000: 2018-02-08T20:31:33.633+0000: 879744.238879744.238: [Full GC (Metadata GC Threshold) : [GC concurrent-root-region-scan-start] GCParseRule CORRUPTED_CONCURRENT_START_V7 = new GCParseRule("CORRUPTED_CONCURRENT_START_V7", DATE_STAMP + DATE_STAMP + INTEGER + DECIMAL_POINT + "\\d{3}" + TIMESTAMP + "\\[Full GC " + GC_CAUSE + ": \\[GC concurrent-(.+)-start\\]"); //2018-02-08T20:31:54.234+0000: 2018-02-08T20:31:54.234+0000879764.839: 879764.839: [Full GC (Metadata GC Threshold) : [GC concurrent-root-region-scan-start] @@ -312,7 +312,7 @@ public interface G1GCPatterns extends G1GCTokens { //tag secs may or may not be there so lets not be greedy. GCParseRule YOUNG_WITH_CONCURRENT_END = new GCParseRule("YOUNG_WITH_CONCURRENT_END", G1GC_PREFIX + "\\((young|mixed)\\)" + GC_PREFIX + "concurrent-(.+)-end, " + "(-?" + REAL_NUMBER + ")"); //100081.540: [Full GC100081.540: [GC concurrent-root-region-scan-start] - GCParseRule FULLGC_WITH_CONCURRENT_PHASE = new GCParseRule("FULLGC_WITH_CONCURRENT_PHASE", FULL_GC_PREFIX + GC_PREFIX + "concurrent-(.+)-start\\]"); + GCParseRule FULLGC_WITH_CONCURRENT_PHASE = new GCParseRule("FULLGC_WITH_CONCURRENT_PHASE", "^" + FULL_GC_PREFIX + GC_PREFIX + "concurrent-(.+)-start\\]"); //28983.087: [GC pause (young)28984.641: [SoftReference, 0 refs, 0.0000060 secs]28984.641: [WeakReference, 651 refs, 0.0001330 secs]28984.641: [FinalReference, 4358 refs, 0.0198770 secs]28984.661: [PhantomReference, 20 refs, 0.0000100 secs]28984.661: [JNI Weak Reference, 0.0002120 secs]2014-10-21T16:31:20.256-0500: 28984.853: [GC concurrent-mark-end, 0.8720730 sec] (to-space overflow), 1.88262800 secs] GCParseRule YOUNG_REFERENCE_WITH_CONCURRENT_END = new GCParseRule("YOUNG_REFERENCE_WITH_CONCURRENT_END", G1GC_PREFIX + "\\((young||mixed)\\)( \\(initial-mark\\))?" + REFERENCE_RECORDS + GC_PREFIX + " concurrent-(.+)-end, " + PAUSE_TIME + "\\]"); diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCTokens.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCTokens.java index fd89aed1..a89b3dee 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCTokens.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCTokens.java @@ -5,7 +5,7 @@ public interface G1GCTokens extends SharedPatterns { - String G1GC_PREFIX = DATE_TIMESTAMP + "\\[GC pause " + GC_CAUSE; + String G1GC_PREFIX = "^" + DATE_TIMESTAMP + "\\[GC pause " + GC_CAUSE; String G1_SURVIVOR_FROM_TO = REAL_VALUE + "([B,K,M,G])->" + REAL_VALUE + "([B,K,M,G])"; String G1_OCCUPANCY_CONFIGURED = REAL_VALUE + "([B,K,M,G])\\(" + REAL_VALUE + "([B,K,M,G])\\)"; diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java index f6223a84..ac07ba62 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java @@ -82,8 +82,8 @@ else if (now.before(getClock())) { * not been recorded. In this case, JVMTermination time should advanced * to after the previous event has ended. * - * @param channel - * @param event + * @param channel to publish to + * @param event to be published */ public void publish(ChannelName channel, JVMEvent event) { lastDuration = event.getDuration(); @@ -108,13 +108,11 @@ boolean hasPrintGCDetails() { // todo: mixes aggregator with parsing. premature optimization... MemoryPoolSummary getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(GCLogTrace trace, int offset) { - MemoryPoolSummary summary = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(offset); - return summary; + return trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(offset); } MemoryPoolSummary getTotalOccupancyWithTotalHeapSizeSummary(GCLogTrace trace, int offset) { - MemoryPoolSummary summary = trace.getOccupancyWithMemoryPoolSizeSummary(offset); - return summary; + return trace.getOccupancyWithMemoryPoolSizeSummary(offset); } GCLogTrace extractReferenceBlock(String line, GCParseRule rule) { @@ -126,24 +124,24 @@ ReferenceGCSummary extractPrintReferenceGC(String line) { ReferenceGCSummary summary = new ReferenceGCSummary(); GCLogTrace trace; if ((trace = extractReferenceBlock(line, SOFT_REFERENCE)) != null) - summary.addSoftReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration()); - + summary.addSoftReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(6), trace.getDuration()); if ((trace = extractReferenceBlock(line, WEAK_REFERENCE)) != null) - summary.addWeakReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration()); + summary.addWeakReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(6), trace.getDuration()); if ((trace = extractReferenceBlock(line, FINAL_REFERENCE)) != null) - summary.addFinalReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration()); + summary.addFinalReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(6), trace.getDuration()); if ((trace = extractReferenceBlock(line, PHANTOM_REFERENCE)) != null) { - if (trace.groupNotNull(4)) - summary.addPhantomReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getIntegerGroup(4), trace.getDuration()); + if (trace.groupNotNull(7)) { + summary.addPhantomReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(6), trace.getIntegerGroup(7), trace.getDuration()); + } else - summary.addPhantomReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration()); + summary.addPhantomReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(6), trace.getDuration()); } if ((trace = extractReferenceBlock(line, JNI_REFERENCE)) != null) { - if (trace.groupNotNull(3)) - summary.addJNIWeakReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration()); + if (trace.groupNotNull(6)) + summary.addJNIWeakReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(6), trace.getDuration()); else summary.addJNIWeakReferences(trace.getDateTimeStamp(), trace.getDuration()); } @@ -165,20 +163,26 @@ MemoryPoolSummary extractPermOrMetaspaceRecord(String line) { MemoryPoolSummary metaDataPool = null; if ((trace = PERM_SPACE_RECORD.parse(line)) != null) { String type = trace.getGroup(1).trim(); - if ("CMS Perm".equals(type)) - metaDataPool = extractPermGenRecord(trace); - else if ("PS Perm".equals(type)) - metaDataPool = extractPermGenRecord(trace); - else if ("PSPermGen".equals(type)) - metaDataPool = extractPermGenRecord(trace); - else if ("Perm".equals(type)) - metaDataPool = extractPermGenRecord(trace); - else if ("Metaspace".equals(type)) { - if (trace.getGroup(2) != null) { - metaDataPool = new MetaspaceRecord(trace.toKBytes(2), trace.toKBytes(4), trace.toKBytes(6)); - } else { - metaDataPool = new MetaspaceRecord(trace.toKBytes(4), trace.toKBytes(4), trace.toKBytes(6)); - } + switch (type) { + case "CMS Perm": + metaDataPool = extractPermGenRecord(trace); + break; + case "PS Perm": + metaDataPool = extractPermGenRecord(trace); + break; + case "PSPermGen": + metaDataPool = extractPermGenRecord(trace); + break; + case "Perm": + metaDataPool = extractPermGenRecord(trace); + break; + case "Metaspace": + if (trace.getGroup(2) != null) { + metaDataPool = new MetaspaceRecord(trace.toKBytes(2), trace.toKBytes(4), trace.toKBytes(6)); + } else { + metaDataPool = new MetaspaceRecord(trace.toKBytes(4), trace.toKBytes(4), trace.toKBytes(6)); + } + break; } } else if ((trace = META_SPACE_RECORD.parse(line)) != null) { int index = (trace.getGroup(1) == null) ? 1 : 3; diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogTrace.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogTrace.java index c29ffe68..c2d566ac 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogTrace.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogTrace.java @@ -57,11 +57,11 @@ public GCCause gcCause(int base, int offset) { } public GCCause gcCause(int offset) { - return gcCause(3, offset); + return gcCause(6, offset); } public GCCause gcCause() { - return gcCause(3, 0); + return gcCause(6, 0); } public double getPauseTime() { @@ -182,7 +182,7 @@ public MemoryPoolSummary getOccupancyBeforeAfterWithMemoryPoolSizeSummary(int of long size = toKBytes(offset + 4); return new MemoryPoolSummary(before, size, after, size); } catch (NumberFormatException numberFormatException) { - LOGGER.fine("Unable to calculate generational memory pool summary."); + LOGGER.warning("Unable to calculate generational memory pool summary."); notYetImplemented(); } @@ -203,6 +203,20 @@ public MemoryPoolSummary getOccupancyWithMemoryPoolSizeSummary(int offset) { return null; } + public MemoryPoolSummary getOccupancyWithMemoryPoolSizeSummary() { + + try { + long occupancy = toKBytes(1); + long size = toKBytes(3); + return new MemoryPoolSummary(occupancy, size, occupancy, size); + } catch (NumberFormatException numberFormatException) { + LOGGER.fine("Unable to calculate generational memory pool occupancy summary."); + notYetImplemented(); + } + + return null; + } + public MetaspaceRecord getMetaSpaceRecord(int offset) { try { long before = toKBytes(offset); diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java index 9e2610ca..a24dae6f 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java @@ -159,23 +159,23 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim parseRules.put(INITIAL_MARK, this::initialMark); parseRules.put(SCAVENGE_BEFORE_REMARK, this::scavengeBeforeRemark); parseRules.put(SCAVENGE_BEFORE_REMARK_TENURING, this::scavengeBeforeRemarkTenuring); - parseRules.put(PARALLEL_REMARK_WEAK_REF, this::remarkAt12); - parseRules.put(PARALLEL_REMARK_CLASS_UNLOADING, this::remarkAt21); + parseRules.put(PARALLEL_REMARK_WEAK_REF, this::recordRemark); + parseRules.put(PARALLEL_REMARK_CLASS_UNLOADING, this::recordRemark); parseRules.put(REMARK_PARNEW_PROMOTION_FAILED, this::remarkParNewPromotionFailed); parseRules.put(PARALLEL_REMARK_STRING_SYMBOL, this::parallelRemarkStringSymbolClause); - parseRules.put(PARALLEL_REMARK_WEAK_CLASS_SYMBOL_STRING, this::remarkAt21); - parseRules.put(PARALLEL_REMARK_WEAK_STRING, this::remarkAt15); + parseRules.put(PARALLEL_REMARK_WEAK_CLASS_SYMBOL_STRING, this::recordRemark); + parseRules.put(PARALLEL_REMARK_WEAK_STRING, this::recordRemark); parseRules.put(PARALLEL_RESCAN, this::parallelRescan); - parseRules.put(REMARK, this::remarkAt11); - parseRules.put(PARALLEL_RESCAN_V2, this::remarkAt11); - parseRules.put(PARALLEL_RESCAN_WEAK_CLASS_SCRUB, this::remarkAt13); + parseRules.put(REMARK, this::recordRemark); + parseRules.put(PARALLEL_RESCAN_V2, this::recordRemark); + parseRules.put(PARALLEL_RESCAN_WEAK_CLASS_SCRUB, this::recordRemark); //, 0.1127040 secs]220.624: [weak refs processing, 0.1513820 secs] [1 CMS-remark: 10541305K(16777216K)] 10742883K(18664704K), 0.7371020 secs] //todo: this was capturing records that is shouldn't have so the rule was modified.. now does it work??? Needs through testing now that order of evaluation will change - parseRules.put(SERIAL_REMARK_SCAN_BREAKDOWNS, this::remarkAt15); - parseRules.put(REMARK_DETAILS, this::remarkAt1); + parseRules.put(SERIAL_REMARK_SCAN_BREAKDOWNS, this::recordRemark); + parseRules.put(REMARK_DETAILS, this::recordRemark); parseRules.put(REMARK_REFERENCE_PROCESSING, this::recordRemarkWithReferenceProcessing); parseRules.put(TENURING_DETAILS, this::tenuringDetails); - parseRules.put(RESCAN_WEAK_CLASS_SYMBOL_STRING, this::remarkAt11); + parseRules.put(RESCAN_WEAK_CLASS_SYMBOL_STRING, this::rescanWeakClassSymbolString); parseRules.put(CONCURRENT_MODE_FAILURE_DETAILS, this::concurrentModeFailureDetails); parseRules.put(CONCURRENT_MODE_FAILURE_DETAILS_META, this::concurrentModeFailureDetails); parseRules.put(PARNEW_DETAILS_CONCURRENT_MODE_FAILURE_PERM, this::parNewDetailsConcurrentModeFailure); @@ -198,8 +198,8 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim parseRules.put(iCMS_PARNEW_DEFNEW_TENURING_DETAILS, this::iCMSParNewDefNewTenuringDetails); parseRules.put(iCMS_FULL, this::iCMSFullGC); parseRules.put(iCMS_PROMOTION_FAILED, this::iCMSPromotionFailed); - parseRules.put(iCMS_PROMOTION_FAILED_PERM, this::iCMSPromotionFailedPermMeta); - parseRules.put(iCMS_PROMOTION_FAILED_META, this::iCMSPromotionFailedPermMeta); +// parseRules.put(iCMS_PROMOTION_FAILED_PERM, this::iCMSPromotionFailedPermMeta); // todo: this rule and the next appear to be a duplication of iCMS_PARNEW_PROMOTION_FAILURE +// parseRules.put(iCMS_PROMOTION_FAILED_META, this::iCMSPromotionFailedPermMeta); parseRules.put(iCMS_MISLABELED_FULL, this::iCMSMislabeledFull); parseRules.put(iCMS_FULL_AFTER_CONCURRENT_MODE_FAILURE, this::iCMSFullAfterConcurrentModeFailure); parseRules.put(iCMS_FULL_AFTER_CONCURRENT_MODE_FAILURE_META, this::iCMSFullAfterConcurrentModeFailure); @@ -210,7 +210,7 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim parseRules.put(FULL_PARNEW_CMF_META, this::fullParNewConcurrentModeFailureMeta); parseRules.put(FULL_PARNEW_CMF_PERM, this::fullParNewConcurrentModeFailurePerm); parseRules.put(PARNEW_CONCURRENT_MODE_FAILURE_PERM, this::parNewConcurrentModeFailurePerm); - parseRules.put(PARNEW_CONCURRENT_MODE_FAILURE_META, this::parNewConcurrentModeFailureMeta); + parseRules.put(PARNEW_CONCURRENT_MODE_FAILURE_META, this::parNewConcurrentModeFailurePerm); parseRules.put(PS_FULL_GC_V2_PERM, this::psFullGCV2Perm); parseRules.put(PS_FULL_GC_PERM, this::psFullGCPerm); parseRules.put(CMS_FULL_PERM, this::cmsFullPermOrMeta); @@ -275,7 +275,7 @@ public class GenerationalHeapParser extends PreUnifiedGCLogParser implements Sim //this rule must be evaluated before CONCURRENT_PHASE_END_BLOCK parseRules.put(ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE, this::abortPrecleanDueToTime); - parseRules.put(CONCURRENT_PHASE_CARDS, this::abortPreclean); + parseRules.put(CONCURRENT_PHASE_CARDS, this::abortPrecleanWithCards); parseRules.put(PRECLEAN_REFERENCE, this::endConcurrentPrecleanWithReferenceProcessing); parseRules.put( new GCParseRule("FLOATING_CPU_BREAKOUT", "^" + CPU_SUMMARY), this::noop); parseRules.put(new GCParseRule("END_OF_DATA_SENTINEL", END_OF_DATA_SENTINEL), this::endOfFile); @@ -370,7 +370,7 @@ public void endOfFile(GCLogTrace trace, String line) { public void defNew(GCLogTrace trace, String line) { DefNew defNew = new DefNew(getClock(), trace.gcCause(), trace.getDuration()); - defNew.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(7), this.getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 14)); + defNew.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(13), this.getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 20)); defNew.add(extractCPUSummary(line)); publish(defNew); } @@ -378,8 +378,8 @@ public void defNew(GCLogTrace trace, String line) { //2019-10-22T23:41:21.852+0000: 21.912: [GC (GCLocker Initiated GC) 2019-10-22T23:41:21.853+0000: 21.912: [DefNew2019-10-22T23:41:21.914+0000: 21.974: [SoftReference, 0 refs, 0.0000842 secs]2019-10-22T23:41:21.914+0000: 21.974: [WeakReference, 76 refs, 0.0000513 secs]2019-10-22T23:41:21.914+0000: 21.974: [FinalReference, 91635 refs, 0.0396861 secs]2019-10-22T23:41:21.954+0000: 22.014: [PhantomReference, 0 refs, 3 refs, 0.0000444 secs]2019-10-22T23:41:21.954+0000: 22.014: [JNI Weak Reference, 0.0000281 secs]: 419520K->19563K(471936K), 0.1019514 secs] 502104K->102148K(2044800K), 0.1020469 secs] [Times: user=0.09 sys=0.01, real=0.10 secs] public void defNewDetails(GCLogTrace trace, String line) { DefNew defNew = new DefNew(getClock(), trace.gcCause(), trace.getDuration()); - MemoryPoolSummary heap = this.getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 37); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(30); + MemoryPoolSummary heap = this.getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 58); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(51); defNew.add(young, heap.minus(young), heap); defNew.add(extractCPUSummary(line)); defNew.add(extractPrintReferenceGC(line)); @@ -405,8 +405,8 @@ public void serialFull(GCLogTrace trace, String line) { //62.616: [GC 62.616: [ParNew: 5033216K->129451K(5662336K), 0.2536590 secs] 5097075K->193310K(24536704K), 0.2538510 secs] //48.021: [GC48.021: [ParNew: 306686K->34046K(306688K), 0.3196120 secs] 1341473K->1125818K(8669952K), 0.3197540 secs] public void parNew(GCLogTrace trace, String line) { - ParNew collection = new ParNew(getClock(), trace.gcCause(1), trace.getDoubleGroup(20)); - collection.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(7), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 14)); + ParNew collection = new ParNew(getClock(), trace.gcCause(1), trace.getDuration()); + collection.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(13), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 20)); collection.add(extractCPUSummary(line)); publish(collection); } @@ -536,13 +536,13 @@ private void noop(GCLogTrace trace, String line) { //public static final ParseRule PARNEW_CONCURRENT_MODE_END = new ParseRule (GC_PREFIX + PARNEW_BLOCK + TIMESTAMP + "\\[CMS" + CMS_PHASE_END + "(?: " + CPU_BREAKDOWN + ")?$"); //public static final String CMS_PHASE_END = DATE_TIMESTAMP + "\\[CMS-concurrent-(.+): " + CPU_WALLCLOCK + "\\]"; public void parNewConcurrentModeEnd(GCLogTrace trace, String line) { - concurrentPhaseEnd(trace,line,14); + concurrentPhaseEnd(trace,line,23); garbageCollectionTypeForwardReference = GarbageCollectionTypes.ParNew; scavengeTimeStamp = getClock(); gcCauseForwardReference = trace.gcCause(); - scavengeDurationForwardReference = trace.getDoubleGroup(12); + scavengeDurationForwardReference = trace.getDoubleGroup(18); parNewForwardReference = new ParNew(scavengeTimeStamp, garbageCollectionTypeForwardReference, gcCauseForwardReference, scavengeDurationForwardReference); - youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6); + youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(12); } public void parNewCardTable(GCLogTrace trace, String line) { @@ -554,7 +554,7 @@ public void parNewCardTable(GCLogTrace trace, String line) { //89.260: [GC 89.260: [ParNew: 19135K->19135K(19136K), 0.0000156 secs]89.260: [CMS: 105875K->107775K(107776K), 0.5703972 secs] 125011K->116886K(126912K), [CMS Perm : 15589K->15584K(28412K)], 0.5705219 secs] public void parNewToConcurrentModeFailure(GCLogTrace trace, String line) { ConcurrentModeFailure collection = new ConcurrentModeFailure(getClock(), trace.gcCause(), trace.getDoubleGroup(trace.groupCount())); - collection.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6), trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(15), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 22)); + collection.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(12), trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(24), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 31)); collection.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); collection.add(extractCPUSummary(line)); publish(collection); @@ -574,8 +574,8 @@ public void parNewToPsudoConcurrentModeFailure(GCLogTrace trace, String line) { public void parNewReference(GCLogTrace trace, String line) { ParNew gcEvent = new ParNew(trace.getDateTimeStamp(), trace.gcCause(), trace.getDuration()); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(30); - MemoryPoolSummary heap = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(37); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(51); + MemoryPoolSummary heap = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(58); gcEvent.add(young, heap.minus(young), heap); gcEvent.addReferenceGCSummary(extractPrintReferenceGC(line)); gcEvent.add(extractCPUSummary(line)); @@ -612,9 +612,9 @@ public void defNewReference(GCLogTrace trace, String line) { //GenerationalHeapParser, not implemented: 2015-07-06T15:18:19.465-0700: 483260.591: [GC (Allocation Failure) 483260.592: [ParNew (promotion failed): 2146944K->2146944K(2146944K), 0.9972460 secs] 20182045K->20340243K(20732992K), 0.9975730 secs public void parNewPromotionFailed(GCLogTrace trace, String line) { - ParNewPromotionFailed collection = new ParNewPromotionFailed(getClock(), GCCause.PROMOTION_FAILED, trace.getDoubleGroup(trace.groupCount())); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6); - MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 13); + ParNewPromotionFailed collection = new ParNewPromotionFailed(getClock(), GCCause.PROMOTION_FAILED, trace.getDuration()); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(12); + MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 19); collection.add(young, heap); collection.add(extractCPUSummary(line)); publish(collection); @@ -666,14 +666,13 @@ public void parNewPromotionFailedTenuring(GCLogTrace trace, String line) { * @param line The GC log line being parsed */ public void parNewPromotionFailedInConcurrentMarkSweepPhase(GCLogTrace trace, String line) { - concurrentPhaseEnd(trace,line,12); - int offset = (trace.groupCount() == 16) ? 2 : 0; + concurrentPhaseEnd(trace,line,15); + int offset = (trace.groupCount() == 23) ? 0 : 2; scavengeTimeStamp = getClock(); garbageCollectionTypeForwardReference = GarbageCollectionTypes.ParNewPromotionFailed; gcCauseForwardReference = GCCause.PROMOTION_FAILED; - youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(5 - offset); - scavengeDurationForwardReference = trace.getDoubleGroup(11 - offset); - scavengeCPUSummaryForwardReference = extractCPUSummary(line); + youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(8 - offset); + scavengeDurationForwardReference = trace.getDoubleGroup(14 - offset); } //364.157: [GC 364.157: [ParNew: 235968K->235968K(235968K), 0.0000430 secs]364.157: [CMSbailing out to foreground collection @@ -689,30 +688,30 @@ public void concurrentMarkSweepBailingToForeground(GCLogTrace trace, String line scavengeTimeStamp = getClock(); garbageCollectionTypeForwardReference = GarbageCollectionTypes.ParNew; gcCauseForwardReference = trace.gcCause(); - youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6); + youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(12); scavengeDurationForwardReference = trace.getDoubleGroup(trace.groupCount() - 1); } //14921.473: [GC14921.473: [ParNew (promotion failed): 609300K->606029K(629120K), 0.1794470 secs]14921.652: [CMS: 1224499K->1100531K(1398144K), 3.4546330 secs] 1829953K->1100531K(2027264K), [CMS Perm : 148729K->148710K(262144K)], 3.6342950 secs] //84.977: [GC 84.977: [ParNew (promotion failed): 17024K->19136K(19136K), 0.0389515 secs]85.016: [CMS: 107077K->107775K(107776K), 0.2868956 secs] 109258K->107895K(126912K), [CMS Perm : 10680K->10674K(21248K)], 0.3260017 secs] [Times: user=0.37 sys=0.00, real=0.32 secs] public void promotionFailedToFull(GCLogTrace trace, String line) { - ParNewPromotionFailed youngCollection = new ParNewPromotionFailed(getClock(), GCCause.PROMOTION_FAILED, trace.getDoubleGroup(12)); - MemoryPoolSummary youngGen = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6); - MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(15); - MemoryPoolSummary totalHeap = this.getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 22); + double youngDuration = trace.getDoubleGroup(18); + ParNewPromotionFailed youngCollection = new ParNewPromotionFailed(getClock(), GCCause.PROMOTION_FAILED, youngDuration); + MemoryPoolSummary youngGen = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(12); + MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(24); + MemoryPoolSummary totalHeap = this.getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 31); // This calculation is a best guess as the data is improperly recorded and details are lost youngCollection.add(youngGen, new MemoryPoolSummary(tenured.getOccupancyBeforeCollection(), tenured.getOccupancyBeforeCollection(), tenured.getSizeAfterCollection()), new MemoryPoolSummary(youngGen.getOccupancyBeforeCollection() + tenured.getOccupancyBeforeCollection(), youngGen.getOccupancyAfterCollection() + tenured.getOccupancyBeforeCollection(), youngGen.getSizeAfterCollection() + tenured.getSizeAfterCollection())); + publish(youngCollection); FullGC collection; - if (trace.getGroup(13) != null) - collection = new FullGC(new DateTimeStamp(trace.getGroup(13), trace.getDoubleGroup(14)), GCCause.PROMOTION_FAILED, trace.getDoubleGroup(trace.groupCount())); - else - collection = new FullGC(new DateTimeStamp(trace.getDoubleGroup(14)), GCCause.PROMOTION_FAILED, trace.getDoubleGroup(trace.groupCount())); - + DateTimeStamp fullStart = trace.getDateTimeStamp(3); + double duration = trace.getDuration() - youngDuration; + collection = new FullGC(fullStart, GCCause.PROMOTION_FAILED, duration); collection.add(totalHeap.minus(tenured), tenured, totalHeap); collection.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); collection.add(extractCPUSummary(line)); @@ -737,18 +736,20 @@ public void plabSummary(GCLogTrace trace, String line) { public void parNewPromotionFailedTimeAbortPreclean(GCLogTrace trace, String line) { garbageCollectionTypeForwardReference = GarbageCollectionTypes.ParNewPromotionFailed; gcCauseForwardReference = GCCause.PROMOTION_FAILED; - fullGCTimeStamp = getClock(); - scavengeTimeStamp = trace.getDateTimeStamp(2); - youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6); - scavengeDurationForwardReference = trace.getDoubleGroup(12); + scavengeDurationForwardReference = trace.getDoubleGroup((2 * TIME_DATE_OFFSET) + 6); + fullGCTimeStamp = getClock().add(scavengeDurationForwardReference); + scavengeTimeStamp = trace.getDateTimeStamp(); + youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(2 * TIME_DATE_OFFSET); } + // todo: doesn't seem to be test coverage for this public void parNewPromotionFailedConcurrentPhase(GCLogTrace trace, String line) { scavengeTimeStamp = trace.getDateTimeStamp(); garbageCollectionTypeForwardReference = GarbageCollectionTypes.ParNew; gcCauseForwardReference = trace.gcCause(); - scavengeDurationForwardReference = trace.getDoubleGroup(12); - youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6); + scavengeDurationForwardReference = trace.getDoubleGroup(18); + youngMemoryPoolSummaryForwardReference = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(12); + concurrentPhaseEnd(trace,line,19); } // ParNew with tenuring, no concurrent-mode-failure @@ -759,12 +760,15 @@ public void corruptedParNewConcurrentPhase(GCLogTrace trace, String line) { scavengeTimeStamp = trace.getDateTimeStamp(); gcCauseForwardReference = trace.gcCause(); garbageCollectionTypeForwardReference = GarbageCollectionTypes.ParNew; + concurrentPhaseEnd(trace,line,11); } + //(promotion failed): 118016K->118016K(118016K), 0.0288030 secs]17740.440: [CMS (concurrent mode failure): 914159K->311550K(917504K), 0.5495730 secs] 985384K->311550K(1035520K), + // [CMS Perm : 65977K->65950K(131072K)], 0.5785090 secs] [Times: user=0.67 sys=0.01, real=0.58 secs] public void corruptedParNewBody(GCLogTrace trace, String line) { ParNewPromotionFailed parNewPromotionFailed = new ParNewPromotionFailed(scavengeTimeStamp, GCCause.PROMOTION_FAILED, trace.getPauseTime() - trace.getDoubleGroup(16)); - MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 17); - MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(10); + MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 20); + MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(13); MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1); MemoryPoolSummary parNewHeap = new MemoryPoolSummary(heap.getOccupancyBeforeCollection(), heap.getSizeAfterCollection(), heap.getOccupancyBeforeCollection(), heap.getSizeAfterCollection()); @@ -779,16 +783,6 @@ public void corruptedParNewBody(GCLogTrace trace, String line) { publish(concurrentModeFailure); } - /* - private void startOfConcurrentPhase(GCLogTrace trace, String line) { - startOfConcurrentPhase = trace.getDateTimeStamp(); - } - private void endOfConcurrentPhase(GCLogTrace trace, String line) { - DateTimeStamp endOfPhase = trace.getDateTimeStamp(); - endOfConcurrentPhase(trace, endOfPhase); - } - */ - public void concurrentPhaseStart(GCLogTrace trace, String line) { startOfConcurrentPhase = trace.getDateTimeStamp(); inConcurrentPhase = true; @@ -800,9 +794,9 @@ public void concurrentPhaseEnd(GCLogTrace trace, String line) { public void concurrentPhaseEnd(GCLogTrace trace, String line, int offset) { try { - double cpuTime = trace.getDoubleGroup(4 + offset); - double wallClock = trace.getDoubleGroup(5 + offset); - switch (trace.getGroup(3 + offset)) { + double cpuTime = trace.getDoubleGroup(7 + offset); + double wallClock = trace.getDoubleGroup(8 + offset); + switch (trace.getGroup(6 + offset)) { case "mark": publish(new ConcurrentMark(startOfConcurrentPhase, wallClock, cpuTime, wallClock)); break; @@ -821,7 +815,6 @@ public void concurrentPhaseEnd(GCLogTrace trace, String line, int offset) { default: LOGGER.warning("concurrent phase not recognized end statement -> " + trace); } - } catch (Exception e) { LOGGER.warning("concurrent phase end throws " + e.getMessage() + " for " + trace); } @@ -831,7 +824,7 @@ public void concurrentPhaseEnd(GCLogTrace trace, String line, int offset) { //todo: Initial-mark rule maybe too greedy??? public void initialMark(GCLogTrace trace, String line) { InitialMark initialMark = new InitialMark(getClock(), trace.gcCause(), trace.getDoubleGroup(trace.groupCount())); - initialMark.add(trace.getOccupancyWithMemoryPoolSizeSummary(4), getTotalOccupancyWithTotalHeapSizeSummary(trace, 8)); + initialMark.add(trace.getOccupancyWithMemoryPoolSizeSummary(7), getTotalOccupancyWithTotalHeapSizeSummary(trace, 11)); initialMark.add(extractCPUSummary(line)); publish(initialMark); } @@ -839,30 +832,22 @@ public void initialMark(GCLogTrace trace, String line) { //2399564.221: [GC[YG occupancy: 143736 K (2052672 K)]2399564.221: [GC 2399564.222: [ParNew: 143736K->43849K(2052672K), 10.4702845 secs] 2214360K->2115172K(6193728K), 10.4716661 secs //4981160.497: [GC[YG occupancy: 77784 K (153344 K)]4981160.498: [Rescan (parallel) , 0.1474240 secs]2014-06-10T17:50:11.873-0700: 4981160.645: [weak refs processing, 0.0000430 secs]2014-06-10T17:50:11.873-0700: 4981160.645: [scrub string table, 0.0009790 secs] [1 CMS-remark: 1897500K(1926784K)] 1975285K(2080128K), 0.1486260 secs] [Times: user=0.29 sys=0.00, real=0.15 secs] public void scavengeBeforeRemark(GCLogTrace trace, String line) { - ParNew parNew = new ParNew(new DateTimeStamp(trace.getGroup(6), trace.getDoubleGroup(7)), trace.gcCause(), trace.getDoubleGroup(trace.groupCount())); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(11); - MemoryPoolSummary tenured = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 18); + DateTimeStamp parNewStart = trace.getDateTimeStamp(2); + ParNew parNew = new ParNew(parNewStart, GCCause.UNKNOWN_GCCAUSE, trace.getDuration()); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(20); + MemoryPoolSummary tenured = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 27); parNew.add(young, tenured.minus(young), tenured); parNew.add(extractCPUSummary(line)); - parNewForwardReference = parNew; + publish(parNew); } public void scavengeBeforeRemarkTenuring(GCLogTrace trace, String line) { - scavengeTimeStamp = new DateTimeStamp(trace.getGroup(6), trace.getDoubleGroup(7)); + scavengeTimeStamp = trace.getDateTimeStamp(2); garbageCollectionTypeForwardReference = GarbageCollectionTypes.ParNew; } - //todo: collapse all of these methods with either a reparse or a location qualifier - public void remarkAt21(GCLogTrace trace, String line) { - recordRemark(trace, line, 21, trace.gcCause()); - } - public void parallelRemarkStringSymbolClause(GCLogTrace trace, String line) { - recordRemark(trace, line, 18, trace.gcCause()); - } - - public void remarkAt15(GCLogTrace trace, String line) { - recordRemark(trace, line, 15, trace.gcCause()); + recordRemark(trace, line, trace.gcCause()); } //todo: outstanding question.. is ParNew scavenge before remark pause time included in the remark pause time @@ -873,11 +858,11 @@ public void parallelRescan(GCLogTrace trace, String line) { collection = new CMSRemark(remarkTimeStamp, GCCause.CMS_FINAL_REMARK, trace.getDoubleGroup(trace.groupCount())); remarkTimeStamp = null; } else { - collection = new CMSRemark(getClock(), GCCause.CMS_FINAL_REMARK, trace.getDoubleGroup(trace.groupCount())); + collection = new CMSRemark(getClock(), GCCause.CMS_FINAL_REMARK, trace.getDuration()); } - MemoryPoolSummary tenured = trace.getOccupancyWithMemoryPoolSizeSummary(10); - MemoryPoolSummary heap = getTotalOccupancyWithTotalHeapSizeSummary(trace, 14); + MemoryPoolSummary tenured = trace.getOccupancyWithMemoryPoolSizeSummary(19); + MemoryPoolSummary heap = getTotalOccupancyWithTotalHeapSizeSummary(trace, 23); collection.add(heap.minus(tenured), tenured, heap); recordRescanStepTimes(collection, line); collection.addReferenceGCSummary(extractPrintReferenceGC(line)); @@ -889,20 +874,18 @@ public void parallelRescan(GCLogTrace trace, String line) { } } - public void remarkAt12(GCLogTrace trace, String line) { - recordRemark(trace, line, 12, trace.gcCause()); - } - - public void remarkAt11(GCLogTrace trace, String line) { - recordRemark(trace, line, 11, trace.gcCause(2)); - } - - public void remarkAt13(GCLogTrace trace, String line) { - recordRemark(trace, line, 13, trace.gcCause()); + public void rescanWeakClassSymbolString(GCLogTrace trace, String line) { + CMSRemark remark = new CMSRemark(getClock(), GCCause.CMS_FINAL_REMARK, trace.getDuration()); + MemoryPoolSummary tenured = new MemoryPoolSummary(trace.toKBytes(20), trace.toKBytes(22), trace.toKBytes(20), trace.toKBytes(22)); + MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(24), trace.toKBytes(26), trace.toKBytes(24), trace.toKBytes(26)); + remark.add(youngMemoryPoolSummaryForwardReference, tenured, heap); + recordRescanStepTimes(remark, line); + remark.add(extractCPUSummary(line)); + publish(remark); } - public void remarkAt1(GCLogTrace trace, String line) { - recordRemark(trace, line, 1, trace.gcCause()); + public void recordRemark(GCLogTrace trace, String line) { + recordRemark(trace, line, trace.gcCause()); } public void remarkParNewPromotionFailed(GCLogTrace trace, String line) { @@ -919,7 +902,7 @@ public void tenuringDetails(GCLogTrace trace, String line) { MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1); MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 8); if (garbageCollectionTypeForwardReference == GarbageCollectionTypes.ParNew) { - ParNew collection = new ParNew(scavengeTimeStamp, gcCauseForwardReference, trace.getDoubleGroup(14)); + ParNew collection = new ParNew(scavengeTimeStamp, gcCauseForwardReference, trace.getDuration()); collection.add(young, heap); collection.add(referenceGCForwardReference); collection.add(extractCPUSummary(line)); @@ -991,20 +974,37 @@ public void concurrentModeFailureDetails(GCLogTrace trace, String line) { //: 439257K->439257K(471872K), 0.5857972 secs]898606.202: [CMS (concurrent mode failure): 1040282K->1048575K(1048576K), 28.6064288 secs] 1478173K->1055740K(1520448K), 29.1931179 secs] //: 189422K->15827K(471872K), 0.1282129 secs]131348.814: [CMS (concurrent mode failure): 927657K->279012K(1572864K), 23.8067389 secs] 1116483K->279012K(2044736K), [CMS Perm : 131071K->27250K(131072K)], 23.9363976 secs] //this one is tricky, the GC is caused by CMS Perm filling up. Take heap before GC as heap after GC for the ParNew + + /** + * Internal memory pools have to be derived for the ParNew. + * The following calculations were used to derive all pool sizes for the log entry below + * : 189422K->15827K(471872K), 0.1282129 secs]131348.814: [CMS (concurrent mode failure): 927657K->279012K(1572864K), 23.8067389 secs] 1116483K->279012K(2044736K), [CMS Perm : 131071K->27250K(131072K)], 23.9363976 secs] + * total heap for young = 189422K + 927657K -> 1116483K (2044736K), @ indexes 1 + 13 -> 20 (24) + * total heap for CMF = 1116483K->279012K(2044736K) 20 -> 22 (24) + * tenured for CMF = 927657K->279012K(1572864K) 13 -> 15 (17) + * young for CMF = 15827K -> 0K (471872K) 3 -> 0K (5) + * @param trace representing the log line + * @param line the text for the log line (can be extracted from the trace using trace.group(0); + * + */ public void parNewDetailsConcurrentModeFailure(GCLogTrace trace, String line) { ParNew parNew = new ParNew(scavengeTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getDoubleGroup(7)); - MemoryPoolSummary summary = new MemoryPoolSummary(trace.toKBytes(24), trace.toKBytes(28), trace.toKBytes(24), trace.toKBytes(28)); - parNew.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1), summary); + MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(1) + trace.toKBytes(13), trace.toKBytes(24), trace.toKBytes(20), trace.toKBytes(24)); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1); + parNew.add(young, heap); parNew.add(extractCPUSummary(line)); publish(parNew, false); if ( inConcurrentPhase) { // todo: publish closing out concurrent phase LOGGER.warning("concurrent phase not closed"); } + ConcurrentModeFailure collection = new ConcurrentModeFailure(fullGCTimeStamp, gcCauseForwardReference, trace.getDuration()); collection.add( - new MemoryPoolSummary(trace.toKBytes(3), trace.toKBytes(5), 0L, trace.toKBytes(28) - trace.toKBytes(21)), - trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(17), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 24)); + new MemoryPoolSummary( + trace.toKBytes(3), trace.toKBytes(5), 0L, trace.toKBytes(5)), + trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(13), + trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(20)); collection.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); collection.add(extractCPUSummary(line)); publish(collection); @@ -1031,15 +1031,15 @@ public void parNewDetailsPromotionFailedWithConcurrentMarkSweepPhase(GCLogTrace public void parNewDetailsWithConcurrentModeFailure(GCLogTrace trace, String line) { ParNewPromotionFailed collection = new ParNewPromotionFailed(scavengeTimeStamp, gcCauseForwardReference, trace.getDoubleGroup(7)); MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1); - MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(17), trace.toKBytes(21), trace.toKBytes(17), trace.toKBytes(21)); + MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(20), trace.toKBytes(24), trace.toKBytes(20), trace.toKBytes(24)); MemoryPoolSummary tenured = heap.minus(young); collection.add(young, tenured, heap); collection.add(extractCPUSummary(line)); publish(collection, false); ConcurrentModeFailure fullCollection = new ConcurrentModeFailure(getClock(), gcCauseForwardReference, trace.getDoubleGroup(trace.groupCount())); - tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(10); - heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 17); + tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(13); + heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 20); young = heap.minus(tenured); fullCollection.add(young, tenured, heap); fullCollection.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); @@ -1112,7 +1112,7 @@ else if (garbageCollectionTypeForwardReference == GarbageCollectionTypes.FullGC) //(concurrent mode failure): 1696350K->613432K(2015232K), 22.0030939 secs] 1772267K->613432K(2097088K) icms_dc=100 , 22.3467890 secs] public void iCMSConcurrentModeFailure(GCLogTrace trace, String line) { if (garbageCollectionTypeForwardReference == GarbageCollectionTypes.FullGC || garbageCollectionTypeForwardReference == GarbageCollectionTypes.ConcurrentModeFailure) { - ConcurrentModeFailure failure = new ConcurrentModeFailure(fullGCTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getDoubleGroup(trace.groupCount())); + ConcurrentModeFailure failure = new ConcurrentModeFailure(fullGCTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getDuration()); failure.add(getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 8)); failure.recordDutyCycle(trace.getIntegerGroup(21)); failure.add(extractCPUSummary(line)); @@ -1152,18 +1152,18 @@ public void iCMSConcurrentModeFailureDuringParNewDefNewDetails(GCLogTrace trace, { ParNewPromotionFailed collection = new ParNewPromotionFailed(scavengeTimeStamp, trace.getDoubleGroup(7)); MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1); - MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(17), trace.toKBytes(21), trace.toKBytes(17), trace.toKBytes(21)); + MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(20), trace.toKBytes(24), trace.toKBytes(20), trace.toKBytes(24)); collection.add(young, heap.minus(young), heap); collection.add(extractCPUSummary(line)); publish(collection); } { - ConcurrentModeFailure collection = new ConcurrentModeFailure(getClock(), GCCause.CMS_FAILURE, trace.getDoubleGroup(trace.groupCount())); - MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(10); - MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 17); + ConcurrentModeFailure collection = new ConcurrentModeFailure(getClock(), GCCause.CMS_FAILURE, trace.getDuration()); + MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(13); + MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 20); collection.add(heap.minus(tenured), tenured, heap); - collection.recordDutyCycle(trace.getIntegerGroup(23)); + collection.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); collection.add(extractCPUSummary(line)); publish(collection); } @@ -1174,7 +1174,7 @@ public void fullGCInterruptsConcurrentPhase(GCLogTrace trace, String line) { fullGCTimeStamp = getClock(); garbageCollectionTypeForwardReference = GarbageCollectionTypes.FullGC; gcCauseForwardReference = trace.gcCause(); - endOfConcurrentPhase(trace,trace.getDateTimeStamp(3), 5); + endOfConcurrentPhase(trace,trace.getDateTimeStamp(3), 11); } public void fullGCReferenceConcurrentModeFailure(GCLogTrace trace, String line) { @@ -1199,8 +1199,9 @@ public void fullParNewStart(GCLogTrace trace, String line) { } public void iCMSParNew(GCLogTrace trace, String line) { - ParNew parNew = new ParNew(getClock(), trace.gcCause(), trace.getDoubleGroup(trace.groupCount())); - parNew.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 13)); + int offset = TIME_DATE_OFFSET * 2; + ParNew parNew = new ParNew(getClock(), trace.gcCause(), trace.getDuration()); + parNew.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(offset), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, offset + 7)); parNew.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); parNew.add(extractCPUSummary(line)); publish(parNew); @@ -1209,44 +1210,44 @@ public void iCMSParNew(GCLogTrace trace, String line) { //Full GC with ParNew promotion failed. //"445909.040: [GC 445909.040: [ParNew (1: promotion failure size = 4629668) (promotion failed): 460096K->460096K(460096K), 0.8467130 secs]445909.887: [CMS: 6252252K->2709964K(7877440K), 23.9213270 secs] 6637399K->2709964K(8337536K), [CMS Perm : 1201406K->68157K(2097152K)] icms_dc=0 , 24.7685320 secs] public void iCMSParNewPromotionFailureRecord(GCLogTrace trace, String line) { - ParNewPromotionFailed parNewPromotionFailed = new ParNewPromotionFailed(getClock(), GCCause.PROMOTION_FAILED, trace.getDoubleGroup(13)); - MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(23), trace.toKBytes(27), trace.toKBytes(23), trace.toKBytes(27)); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(7); + ParNewPromotionFailed parNewPromotionFailed = new ParNewPromotionFailed(getClock(), GCCause.PROMOTION_FAILED, trace.getDoubleGroup(19)); + MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(32), trace.toKBytes(36), trace.toKBytes(32), trace.toKBytes(36)); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(13); MemoryPoolSummary tenured = heap.minus(young); parNewPromotionFailed.add(young, tenured, heap); - parNewPromotionFailed.recordDutyCycle(trace.getIntegerGroup(36)); + parNewPromotionFailed.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 2)); publish(parNewPromotionFailed); //Synthesis a Full GC DateTimeStamp fullGCStart = trace.getDateTimeStamp(3); fullGCStart = getClock().add(fullGCStart.getTimeStamp() - getClock().getTimeStamp()); - FullGC fullGC = new FullGC(fullGCStart, GCCause.PROMOTION_FAILED, trace.getDoubleGroup(22)); - heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 23); - tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(16); - young = new MemoryPoolSummary(0L, trace.toKBytes(11), 0L, trace.toKBytes(11)); + FullGC fullGC = new FullGC(fullGCStart, GCCause.PROMOTION_FAILED, trace.getDoubleGroup(31)); + heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 32); + tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(25); + young = new MemoryPoolSummary(0L, trace.toKBytes(17), 0L, trace.toKBytes(17)); fullGC.add(young, tenured, heap); fullGC.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); - fullGC.recordDutyCycle(trace.getIntegerGroup(36)); + fullGC.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 2)); publish(fullGC); } public void iCMSParNewPromotionFailure(GCLogTrace trace, String line) { - ParNewPromotionFailed parNewPromotionFailed = new ParNewPromotionFailed(getClock(), GCCause.PROMOTION_FAILED, trace.getDoubleGroup(12)); - MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(22), trace.toKBytes(26), trace.toKBytes(22), trace.toKBytes(26)); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6); + double parnewDuration = trace.getDoubleGroup((2 * TIME_DATE_OFFSET) + 6); + ParNewPromotionFailed parNewPromotionFailed = new ParNewPromotionFailed(getClock(), GCCause.PROMOTION_FAILED, parnewDuration); + MemoryPoolSummary heap = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary((3 * TIME_DATE_OFFSET) + 13); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(2 * TIME_DATE_OFFSET); MemoryPoolSummary tenured = heap.minus(young); - parNewPromotionFailed.add(young, tenured, heap); - parNewPromotionFailed.recordDutyCycle(trace.getIntegerGroup(35)); + parNewPromotionFailed.add(young, tenured, new MemoryPoolSummary(heap.getOccupancyBeforeCollection(),heap.getSizeBeforeCollection(),heap.getOccupancyBeforeCollection(),heap.getSizeBeforeCollection())); + parNewPromotionFailed.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); publish(parNewPromotionFailed); //Synthesis a Full GC DateTimeStamp fullGCStart = trace.getDateTimeStamp(3); - ConcurrentModeFailure fullGC = new ConcurrentModeFailure(fullGCStart, GCCause.PROMOTION_FAILED, trace.getDoubleGroup(trace.groupCount() - 1)); - heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 22); - tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(15); - young = new MemoryPoolSummary(0L, trace.toKBytes(10), 0L, trace.toKBytes(10)); + ConcurrentModeFailure fullGC = new ConcurrentModeFailure(fullGCStart, GCCause.PROMOTION_FAILED, trace.getDuration() - parnewDuration); + tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary((3 * TIME_DATE_OFFSET) + 6); + young = new MemoryPoolSummary(young.getOccupancyBeforeCollection(), young.getSizeBeforeCollection(), 0L, young.getSizeAfterCollection()); fullGC.add(young, tenured, heap); fullGC.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); - fullGC.recordDutyCycle(trace.getIntegerGroup(35)); + fullGC.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); publish(fullGC); } @@ -1255,13 +1256,13 @@ public void iCMSParNewPromotionFailure(GCLogTrace trace, String line) { public void fullGCiCMS(GCLogTrace trace, String line) { FullGC collection; if (trace.gcCause() == GCCause.JAVA_LANG_SYSTEM) - collection = new SystemGC(getClock(), trace.gcCause(), trace.getDoubleGroup(trace.groupCount())); + collection = new SystemGC(getClock(), trace.gcCause(), trace.getDuration()); else - collection = new FullGC(getClock(), trace.gcCause(), trace.getDoubleGroup(trace.groupCount())); - MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(5); - MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 12); + collection = new FullGC(getClock(), trace.gcCause(), trace.getDuration()); + MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(2 + TIME_DATE_OFFSET); + MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, (TIME_DATE_OFFSET) + 9); collection.add(heap.minus(tenured), tenured, heap); - collection.recordDutyCycle(trace.getIntegerGroup(25)); + collection.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); collection.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); collection.add(extractCPUSummary(line)); publish(collection); @@ -1270,20 +1271,20 @@ public void fullGCiCMS(GCLogTrace trace, String line) { //: 81792K->0K(81856K), 0.0431036 secs] 90187K->11671K(2097088K) icms_dc=5 , 0.0435503 secs] public void iCMSParNewDefNewTenuringDetails(GCLogTrace trace, String line) { if (GarbageCollectionTypes.ParNew == garbageCollectionTypeForwardReference) { - ParNew collection = new ParNew(scavengeTimeStamp, gcCauseForwardReference, trace.getDoubleGroup(trace.groupCount())); + ParNew collection = new ParNew(scavengeTimeStamp, gcCauseForwardReference, trace.getDuration()); collection.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 8)); collection.recordDutyCycle(trace.getIntegerGroup(14)); collection.add(extractCPUSummary(line)); publish(collection); } else if (GarbageCollectionTypes.ParNewPromotionFailed == garbageCollectionTypeForwardReference) { - ParNewPromotionFailed collection = new ParNewPromotionFailed(scavengeTimeStamp, gcCauseForwardReference, trace.getDoubleGroup(trace.groupCount())); + ParNewPromotionFailed collection = new ParNewPromotionFailed(scavengeTimeStamp, gcCauseForwardReference, trace.getDuration()); MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1); MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 8); collection.add(young, heap.minus(young), heap); collection.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); publish(collection); } else if (GarbageCollectionTypes.FullGC == garbageCollectionTypeForwardReference) { - FullGC collection = new FullGC(fullGCTimeStamp, gcCauseForwardReference, trace.getDoubleGroup(trace.groupCount())); + FullGC collection = new FullGC(fullGCTimeStamp, gcCauseForwardReference, trace.getDuration()); MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1); MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 8); collection.add(young, heap.minus(young), heap); @@ -1298,11 +1299,11 @@ public void iCMSParNewDefNewTenuringDetails(GCLogTrace trace, String line) { //35305.590: [GC 35305.590: [ParNew (promotion failed): 10973K->3147K(153344K), 0.0295730 secs] 1763339K->1763817K(2080128K) icms_dc=32 , 0.0297680 secs] public void iCMSPromotionFailed(GCLogTrace trace, String line) { - ParNewPromotionFailed collection = new ParNewPromotionFailed(getClock(), trace.getDoubleGroup(trace.groupCount())); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(5); - MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 12); + ParNewPromotionFailed collection = new ParNewPromotionFailed(getClock(), trace.getDuration()); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(TIME_DATE_OFFSET + 2); + MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, TIME_DATE_OFFSET + 9); collection.add(young, heap.minus(young), heap); - collection.recordDutyCycle(trace.getIntegerGroup(18)); + collection.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); publish(collection); } @@ -1311,65 +1312,35 @@ private void iCMSFullGC(GCLogTrace trace, String line) { FullGC collection; GCCause cause = trace.gcCause(); if (cause == GCCause.JAVA_LANG_SYSTEM) - collection = new SystemGC(getClock(), trace.getDoubleGroup(trace.groupCount())); + collection = new SystemGC(getClock(), trace.getDuration()); else if ((cause == GCCause.UNKNOWN_GCCAUSE) || (cause == GCCause.GCCAUSE_NOT_SET)) { - collection = new FullGC(getClock(), cause, trace.getDoubleGroup(trace.groupCount())); + collection = new FullGC(getClock(), cause, trace.getDuration()); } else { LOGGER.warning("Unable to parse -> " + trace); return; } - MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6); - MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 13); + MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(12); + MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 19); collection.add(heap.minus(tenured), tenured, heap); collection.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); collection.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); publish(collection); } - - //72825.712: [GC72825.712: [ParNew (promotion failed): 153344K->153344K(153344K), 0.3895590 secs]72826.102: [CMS: 1831960K->1554399K(1926784K), 8.3796720 secs] 1914705K->1554399K(2080128K), [CMS Perm : 131704K->130876K(222768K)] icms_dc=42 , 8.7694530 secs] - //1207,770: [GC 1207,770: [ParNew: 629120K->629120K(629120K), 0,0000240 secs]1207,770: [CMS: 1270289K->696493K(1398144K), 4,3217660 secs] 1899409K->696493K(2027264K), [CMS Perm : 216443K->214985K(360736K)] icms_dc=24 , 4,3221610 secs] [Times: user=4,31 sys=0,01, real=4,32 secs] - /* - 2013-01-25T10:12:13.318+0100: 1207,770: [GC 1207,770: [ParNew: 629120K->629120K(629120K), 0,0000240 secs]1207,770: [CMS: 1270289K->696493K(1398144K), 4,3217660 secs] 1899409K->696493K(2027264K), [CMS Perm : 216443K->214985K(360736K)] icms_dc=24 , 4,3221610 secs] [Times: user=4,31 sys=0,01, real=4,32 secs] - ParNew promotion failed outside of concurrent cycle - */ - public void iCMSPromotionFailedPermMeta(GCLogTrace trace, String line) { - double gcDuration = trace.getDoubleGroup(20); - double parNewDuration = trace.getDoubleGroup(11); - ParNewPromotionFailed parNew = new ParNewPromotionFailed(trace.getDateTimeStamp(), GCCause.UNKNOWN_GCCAUSE, parNewDuration); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(5); - MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(14); - MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 21); - parNew.add(young, - new MemoryPoolSummary(tenured.getOccupancyBeforeCollection(), tenured.getSizeAfterCollection(), tenured.getOccupancyBeforeCollection(), tenured.getSizeAfterCollection()), - new MemoryPoolSummary(heap.getOccupancyBeforeCollection(), heap.getSizeAfterCollection(), heap.getOccupancyBeforeCollection(), heap.getSizeAfterCollection())); - parNew.add(extractCPUSummary(line)); - publish(parNew); - - FullGC full = new FullGC(trace.getDateTimeStamp(3), GCCause.PROMOTION_FAILED, gcDuration); - full.add(heap.minus(tenured), tenured, heap); - full.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); - full.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); - full.add(extractCPUSummary(line)); - publish(full); - } - //91057.643: [GC 91057.643: [ParNew: 153344K->153344K(153344K), 0.0000250 secs]91057.643: [CMS: 1895319K->1754894K(1926784K), 9.6910090 secs] 2048663K->1754894K(2080128K), [CMS Perm : 135228K->134975K(230652K)] icms_dc=65 , 9.6912470 secs] public void iCMSMislabeledFull(GCLogTrace trace, String line) { - double gcDuration = trace.getDoubleGroup(trace.groupCount()); - double parNewDuration = trace.getDoubleGroup(12); + double parNewDuration = trace.getDoubleGroup((TIME_DATE_OFFSET * 2) + 6); ParNew parNew = new ParNew(trace.getDateTimeStamp(2), trace.gcCause(), parNewDuration); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6); - MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(15); - MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 22); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(TIME_DATE_OFFSET * 2); + MemoryPoolSummary tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary((TIME_DATE_OFFSET * 2) + 12); + MemoryPoolSummary heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, (TIME_DATE_OFFSET * 2) + 19); parNew.add(young, new MemoryPoolSummary(tenured.getOccupancyBeforeCollection(), tenured.getSizeAfterCollection(), tenured.getOccupancyBeforeCollection(), tenured.getSizeAfterCollection()), new MemoryPoolSummary(heap.getOccupancyBeforeCollection(), heap.getSizeAfterCollection(), heap.getOccupancyBeforeCollection(), heap.getSizeAfterCollection())); - parNew.add(young, tenured, heap); publish(parNew); - FullGC full = new FullGC(trace.getDateTimeStamp(3), trace.gcCause(), gcDuration); + FullGC full = new FullGC(trace.getDateTimeStamp(3), trace.gcCause(), trace.getDuration()); full.add(heap.minus(tenured), tenured, heap); full.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); full.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); @@ -1379,9 +1350,9 @@ public void iCMSMislabeledFull(GCLogTrace trace, String line) { //10110.232: [Full GC 10110.232: [CMS (concurrent mode failure): 2715839K->1659203K(2752512K), 39.3188254 secs] 2740830K->1659203K(3106432K), [CMS Perm : 206079K->204904K(402264K)] icms_dc=100 , 39.3199631 secs] [Times: user=39.37 sys=0.02, real=39.32 secs] public void iCMSFullAfterConcurrentModeFailure(GCLogTrace trace, String line) { - ConcurrentModeFailure failure = new ConcurrentModeFailure(getClock(), GCCause.UNKNOWN_GCCAUSE, trace.getDoubleGroup(trace.groupCount())); - failure.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 13)); - failure.recordDutyCycle(trace.getIntegerGroup(26)); + ConcurrentModeFailure failure = new ConcurrentModeFailure(getClock(), GCCause.UNKNOWN_GCCAUSE, trace.getDuration()); + failure.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(TIME_DATE_OFFSET * 2), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, (TIME_DATE_OFFSET * 2) + 7)); + failure.recordDutyCycle(trace.getIntegerGroup(trace.groupCount() - 1)); failure.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); failure.add(extractCPUSummary(line)); publish(failure); @@ -1409,11 +1380,11 @@ public void psFullGCMeta(GCLogTrace trace, String line) { GCCause cause = trace.gcCause(); FullGC collection; if (cause == GCCause.JAVA_LANG_SYSTEM || cause == GCCause.HEAP_DUMP) { - collection = new SystemGC(getClock(), cause, trace.getDoubleGroup(trace.groupCount())); + collection = new SystemGC(getClock(), cause, trace.getDuration()); } else { - collection = new PSFullGC(getClock(), cause, trace.getDoubleGroup(trace.groupCount())); + collection = new PSFullGC(getClock(), cause, trace.getDuration()); } - collection.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(5), trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(11), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 17)); + collection.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(TIME_DATE_OFFSET + 2), trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(TIME_DATE_OFFSET + 8), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, TIME_DATE_OFFSET + 14)); collection.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); collection.add(extractCPUSummary(line)); publish(collection); @@ -1431,47 +1402,23 @@ public void psFullGCV2Meta(GCLogTrace trace, String line) { } public void parNewConcurrentModeFailurePerm(GCLogTrace trace, String line) { - double totalPause = trace.getPauseTime(); - double concurrentModeFailurePause = trace.getDoubleGroup(20); + double concurrentModeFailurePause = trace.getDoubleGroup(26); + double parNewPause = trace.getDoubleGroup(18); double startTimeGap = trace.getDoubleGroup(13) - getClock().getTimeStamp(); - ParNewPromotionFailed parNewPromotionFailed = new ParNewPromotionFailed(getClock(), trace.gcCause(), totalPause - concurrentModeFailurePause); - MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(21), trace.toKBytes(25), trace.toKBytes(21), trace.toKBytes(25)); - MemoryPoolSummary tenured = new MemoryPoolSummary(trace.toKBytes(14), trace.toKBytes(18), trace.toKBytes(14), trace.toKBytes(18)); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(6); - parNewPromotionFailed.add(young, tenured, heap); - publish(parNewPromotionFailed); - - ConcurrentModeFailure concurrentModeFailure = new ConcurrentModeFailure(getClock().add(startTimeGap), GCCause.PROMOTION_FAILED, concurrentModeFailurePause); + ParNewPromotionFailed parNewPromotionFailed = new ParNewPromotionFailed(getClock(), trace.gcCause(), parNewPause); + MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(27), trace.toKBytes(31), trace.toKBytes(27), trace.toKBytes(31)); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(12); - heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 21); - tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(14); - concurrentModeFailure.add(heap.minus(tenured), tenured, heap); - concurrentModeFailure.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); - concurrentModeFailure.add(extractCPUSummary(line)); - publish(concurrentModeFailure); - - } - - public void parNewConcurrentModeFailureMeta(GCLogTrace trace, String line) { - - double totalPause = trace.getPauseTime(); - double concurrentModeFailurePause = trace.getDoubleGroup(19); - double startTimeGap = trace.getDoubleGroup(12) - getClock().getTimeStamp(); - - ParNewPromotionFailed parNewPromotionFailed = new ParNewPromotionFailed(getClock(), trace.gcCause(), totalPause - concurrentModeFailurePause); - MemoryPoolSummary heap = new MemoryPoolSummary(trace.toKBytes(20), trace.toKBytes(24), trace.toKBytes(20), trace.toKBytes(24)); - MemoryPoolSummary tenured = new MemoryPoolSummary(trace.toKBytes(13), trace.toKBytes(17), trace.toKBytes(13), trace.toKBytes(17)); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(5); - parNewPromotionFailed.add(young, tenured, heap); + parNewPromotionFailed.add(young, heap.minus(young), heap); publish(parNewPromotionFailed); ConcurrentModeFailure concurrentModeFailure = new ConcurrentModeFailure(getClock().add(startTimeGap), GCCause.PROMOTION_FAILED, concurrentModeFailurePause); - heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 20); - tenured = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(13); - concurrentModeFailure.add(heap.minus(tenured), tenured, heap); + heap = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 27); + young = new MemoryPoolSummary( trace.toKBytes(14), trace.toKBytes(16), 0, trace.toKBytes(16)); + concurrentModeFailure.add(young, heap.minus(young), heap); concurrentModeFailure.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); concurrentModeFailure.add(extractCPUSummary(line)); publish(concurrentModeFailure); @@ -1515,11 +1462,11 @@ public void psFullGCPerm(GCLogTrace trace, String line) { GCCause cause = trace.gcCause(); FullGC collection; if (cause == GCCause.JAVA_LANG_SYSTEM || cause == GCCause.HEAP_DUMP) { - collection = new SystemGC(getClock(), cause, trace.getDoubleGroup(trace.groupCount())); + collection = new SystemGC(getClock(), cause, trace.getDuration()); } else { - collection = new PSFullGC(getClock(), cause, trace.getDoubleGroup(trace.groupCount())); + collection = new PSFullGC(getClock(), cause, trace.getDuration()); } - collection.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(5), trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(11), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 17)); + collection.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(8), trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(14), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 20)); collection.addPermOrMetaSpaceRecord(extractPermOrMetaspaceRecord(line)); collection.add(extractCPUSummary(line)); publish(collection); @@ -1560,8 +1507,8 @@ public void parNewNoDetails(GCLogTrace trace, String line) { } public void youngNoDetails(GCLogTrace trace, String line) { - YoungGC youngGC = new YoungGC(getClock(), GarbageCollectionTypes.Young, GCCause.UNKNOWN_GCCAUSE, trace.getDoubleGroup(trace.groupCount())); - youngGC.add(getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 3)); + YoungGC youngGC = new YoungGC(getClock(), GarbageCollectionTypes.Young, GCCause.UNKNOWN_GCCAUSE, trace.getDuration()); + youngGC.add(getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 6)); youngGC.add(extractCPUSummary(line)); publish(youngGC); } @@ -1570,13 +1517,13 @@ public void cmsNoDetails(GCLogTrace trace, String line) { if (expectRemark) { expectRemark = false; CMSRemark remark = new CMSRemark(getClock(), trace.getDoubleGroup(trace.groupCount())); - remark.add(getTotalOccupancyWithTotalHeapSizeSummary(trace, 3)); + remark.add(getTotalOccupancyWithTotalHeapSizeSummary(trace, 6)); remark.add(extractCPUSummary(line)); publish(remark); } else { expectRemark = true; InitialMark initialMark = new InitialMark(getClock(), trace.getDoubleGroup(trace.groupCount())); - initialMark.add(getTotalOccupancyWithTotalHeapSizeSummary(trace, 3)); + initialMark.add(getTotalOccupancyWithTotalHeapSizeSummary(trace, 6)); initialMark.add(extractCPUSummary(line)); publish(initialMark); } @@ -1585,8 +1532,8 @@ public void cmsNoDetails(GCLogTrace trace, String line) { // 29.975: [Full GC 155252K->44872K(205568K), 0.3398460 secs] public void fullNoGCDetails(GCLogTrace trace, String line) { expectRemark = false; - FullGC fullGC = new FullGC(getClock(), GCCause.UNKNOWN_GCCAUSE, trace.getDoubleGroup(trace.groupCount())); - fullGC.add(getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 3)); + FullGC fullGC = new FullGC(getClock(), GCCause.UNKNOWN_GCCAUSE, trace.getDuration()); + fullGC.add(getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 6)); publish(fullGC); } @@ -1756,7 +1703,7 @@ public void psFullReferencePhase(GCLogTrace trace, String line) { } public void psDetailsWithTenuring(GCLogTrace trace, String line) { - PSYoungGen collection = new PSYoungGen(scavengeTimeStamp, gcCauseForwardReference, trace.getDoubleGroup(trace.groupCount())); + PSYoungGen collection = new PSYoungGen(scavengeTimeStamp, gcCauseForwardReference, trace.getDuration()); collection.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(2), getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 8)); if (referenceGCForwardReference != null) collection.add(referenceGCForwardReference); @@ -1867,11 +1814,11 @@ public void concurrentPhaseYields(GCLogTrace trace, String line) { private void precleanTimedoutWithCards(GCLogTrace trace, String line) { abortPrecleanDueToTime = true; - endOfConcurrentPhase(trace, trace.getDateTimeStamp(), 3); + GCLogTrace concurrentPhase = new GCParseRule("X",CMS_PHASE_END).parse(line); + endOfConcurrentPhase(concurrentPhase, concurrentPhase.getDateTimeStamp(), 0); } private void shouldCollectConcurrent(GCLogTrace trace, String line) { - trace.notYetImplemented(); LOGGER.warning("Not Yet Implemented -> " + trace); } @@ -1892,15 +1839,15 @@ private void adaptivePolicySizeBody(GCLogTrace trace, String line) { } public void remarkSplitByDebug(GCLogTrace trace, String line) { - long youngOccupancy = trace.getLongGroup(4); - long youngConfiguredSize = trace.getLongGroup(5); + long youngOccupancy = trace.getLongGroup(7); + long youngConfiguredSize = trace.getLongGroup(8); youngMemoryPoolSummaryForwardReference = new MemoryPoolSummary(youngOccupancy, youngConfiguredSize, youngOccupancy, youngConfiguredSize); } public void scavengeBeforeRemarkReference(GCLogTrace trace, String line) { - parNewForwardReference = new ParNew(new DateTimeStamp(trace.getGroup(6), trace.getDoubleGroup(7)), trace.gcCause(5), trace.getDoubleGroup(48)); - MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(35); - MemoryPoolSummary tenured = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 42); + parNewForwardReference = new ParNew(trace.getDateTimeStamp(), trace.gcCause(), trace.getDoubleGroup(trace.groupCount() - 3)); + MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(59); + MemoryPoolSummary tenured = getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 66); parNewForwardReference.add(young, tenured.minus(young), tenured); parNewForwardReference.add(extractPrintReferenceGC(line)); parNewForwardReference.add(extractCPUSummary(line)); @@ -1982,7 +1929,7 @@ public void splitRemarkReferenceWithWeakReferenceSplitBug(GCLogTrace trace, Stri collection.addReferenceGCSummary(extractPrintReferenceGC(line)); GCLogTrace weakReferenceFragment = weakReferenceFragmentRule.parse(line); if (weakReferenceFragment != null) { - collection.getReferenceGCSummary().addWeakReferences(trace.getDateTimeStamp(4), 0, weakReferenceFragment.getPauseTime()); + collection.getReferenceGCSummary().addWeakReferences(trace.getDateTimeStamp(2), 0, weakReferenceFragment.getPauseTime()); } collection.add(extractCPUSummary(line)); publish(collection); @@ -1995,19 +1942,25 @@ public void adaptiveSizePolicyStop(GCLogTrace trace, String line) { //************************************ end of direct support for parsing rules - private void recordRemark(GCLogTrace trace, String line, int offset, GCCause gcCause) { - CMSRemark collection; - if (gcCause == GCCause.UNKNOWN_GCCAUSE) - collection = new CMSRemark(getClock(), trace.getDoubleGroup(trace.groupCount())); - else - collection = new CMSRemark(getClock(), gcCause, trace.getDoubleGroup(trace.groupCount())); + private final GCParseRule cmsRemarkBlockRule = new GCParseRule("CMS_REMARK_BLOCK", REMARK_BLOCK); + private void recordRemark(GCLogTrace trace, String line, GCCause gcCause) { - MemoryPoolSummary tenured = trace.getOccupancyWithMemoryPoolSizeSummary(offset); - MemoryPoolSummary heap = getTotalOccupancyWithTotalHeapSizeSummary(trace, offset + 4); - collection.add(heap.minus(tenured), tenured, heap); - recordRescanStepTimes(collection, line); - collection.add(extractCPUSummary(line)); - publish(collection); + CMSRemark collection = (gcCause == GCCause.UNKNOWN_GCCAUSE) + ? new CMSRemark(getClock(), trace.getDuration()) + : new CMSRemark(getClock(), gcCause, trace.getDuration()); + + // stabilizes memory pool sizing information + GCLogTrace remarkBlock = cmsRemarkBlockRule.parse(line); + if ( remarkBlock == null) { + LOGGER.warning("Internal error, unable to parse CMS information from " + line); + } else { + MemoryPoolSummary tenured = remarkBlock.getOccupancyWithMemoryPoolSizeSummary(); + MemoryPoolSummary heap = remarkBlock.getOccupancyWithMemoryPoolSizeSummary(); + collection.add(heap.minus(tenured), tenured, heap); + recordRescanStepTimes(collection, line); + collection.add(extractCPUSummary(line)); + publish(collection); + } } //124.310: [GC (CMS Final Remark) [YG occupancy: 670314 K (737280 K)]124.310: [Rescan (parallel) , 0.1178559 secs]124.428: [weak refs processing124.428: [SoftReference, 0 refs, 0.0000046 secs]124.428: [WeakReference, 0 refs, 0.0000028 secs]124.428: [FinalReference, 0 refs, 0.0000026 secs]124.428: [PhantomReference, 0 refs, 0 refs, 0.0000032 secs]124.428: [JNI Weak Reference, 0.0000094 secs], 0.0000365 secs]124.428: [class unloading, 0.0056223 secs]124.434: [scrub symbol table, 0.0028174 secs]124.437: [scrub string table, 0.0021200 secs][1 CMS-remark: 2896239K(3375104K)] 3566554K(4112384K), 0.1290335 secs] [Times: user=0.94 sys=0.01, real=0.12 secs] @@ -2082,8 +2035,8 @@ private void log(String line) { //this rule must be evaluated before CONCURRENT_PHASE_END_BLOCK private void abortPrecleanDueToTime(GCLogTrace trace, String line) { try { - double cpuTime = trace.getDoubleGroup(4); - double wallClock = trace.getDoubleGroup(5); + double cpuTime = trace.getDoubleGroup(7); + double wallClock = trace.getDoubleGroup(8); publish(new AbortablePreClean(startOfConcurrentPhase, wallClock, cpuTime, wallClock, true)); abortPrecleanDueToTime = false; } catch (Exception e) { @@ -2091,10 +2044,10 @@ private void abortPrecleanDueToTime(GCLogTrace trace, String line) { } } - private void abortPreclean(GCLogTrace trace, String line) { + private void abortPrecleanWithCards(GCLogTrace trace, String line) { try { - double cpuTime = trace.getDoubleGroup(7); - double wallClock = trace.getDoubleGroup(8); + double cpuTime = trace.getDoubleGroup(10); + double wallClock = trace.getDoubleGroup(11); publish(new AbortablePreClean(startOfConcurrentPhase, wallClock, cpuTime, wallClock, false)); abortPrecleanDueToTime = false; } catch (Exception e) { @@ -2113,34 +2066,41 @@ private void endOfConcurrentPhase(GCLogTrace trace, String line) { private void endConcurrentPrecleanWithReferenceProcessing(GCLogTrace trace, String line) { GCLogTrace concurrentBlock = CONCURRENT_PHASE_END_BLOCK.parse(line); try { - publish(new ConcurrentPreClean(startOfConcurrentPhase, concurrentBlock.getDoubleGroup(8), concurrentBlock.getDoubleGroup(4), concurrentBlock.getDoubleGroup(5))); + publish(new ConcurrentPreClean(startOfConcurrentPhase, concurrentBlock.getDoubleGroup(11), concurrentBlock.getDoubleGroup(7), concurrentBlock.getDoubleGroup(8))); } catch (Throwable t) { - LOGGER.warning("concurrent phase choked on " + trace.toString()); + LOGGER.warning("Unable to extract data from " + trace.toString()); } } private void endOfConcurrentPhase(GCLogTrace trace, DateTimeStamp timeStamp, int offset) { - String phase = trace.getGroup(3 + offset); - double cpuTime = trace.getDoubleGroup(4 + offset); - double wallTime = trace.getDoubleGroup(5 + offset); + String phase = trace.getGroup(6 + offset); + double cpuTime = trace.getDoubleGroup(7 + offset); + double wallTime = trace.getDoubleGroup(8 + offset); double duration = timeStamp.toSeconds() - startOfConcurrentPhase.toSeconds(); // The wallTime measure is a very good estimate of duration. if ( duration == Double.NaN) duration = wallTime; - if ("mark".equals(phase)) - publish(new ConcurrentMark(startOfConcurrentPhase, duration, cpuTime, wallTime)); - else if ("preclean".equals(phase)) - publish(new ConcurrentPreClean(startOfConcurrentPhase, duration, cpuTime, wallTime)); - else if ("abortable-preclean".equals(phase)) { - publish(new AbortablePreClean(startOfConcurrentPhase, duration, cpuTime, wallTime, abortPrecleanDueToTime)); - abortPrecleanDueToTime = false; + + switch (phase) { + case "mark": + publish(new ConcurrentMark(startOfConcurrentPhase, duration, cpuTime, wallTime)); + break; + case "preclean": + publish(new ConcurrentPreClean(startOfConcurrentPhase, duration, cpuTime, wallTime)); + break; + case "abortable-preclean": + publish(new AbortablePreClean(startOfConcurrentPhase, duration, cpuTime, wallTime, abortPrecleanDueToTime)); + abortPrecleanDueToTime = false; + break; + case "sweep": + publish(new ConcurrentSweep(startOfConcurrentPhase, duration, cpuTime, wallTime)); + break; + case "reset": + publish(new ConcurrentReset(startOfConcurrentPhase, duration, cpuTime, wallTime)); + break; + default: + LOGGER.warning("concurrent phase choked on " + trace); } - else if ("sweep".equals(phase)) - publish( new ConcurrentSweep(startOfConcurrentPhase, duration, cpuTime, wallTime)); - else if ("reset".equals(phase)) - publish(new ConcurrentReset(startOfConcurrentPhase, duration, cpuTime, wallTime)); - else - LOGGER.warning("concurrent phase choked on " + trace); } public void logMissedFirstRecordForEvent(String line) { @@ -2204,3 +2164,28 @@ public void publishTo(JVMEventChannel bus) { super.publishTo(bus); } } + +/* + todo: this is a very old but bad corruption... + ParNew starts, the output is interrupted by the start of an abortable-preclean but then a CMF happens smashing everything + At the moment the abortable-preclean and the CMF is reported on, the ParNew is missed. + In all honesty, missing a single ParNew isn't going to change the conclusion of any analysis so I'm opt-ing to leave it. + 2014-06-10T22:56:26.242-0700: 4999535.014: [GC2014-06-10T22:56:26.242-0700: 4999535.014: [ParNew2014-06-10T22:56:26.256-0700: 4999535.028: [CMS-concurrent-abortable-preclean: 1.948/3.114 secs] [Times: user=1.93 sys=0.12, real=3.11 secs] + (promotion failed) +Desired survivor size 8716288 bytes, new threshold 6 (max 6) +- age 1: 1036320 bytes, 1036320 total +- age 2: 825248 bytes, 1861568 total +- age 3: 119024 bytes, 1980592 total +- age 4: 113784 bytes, 2094376 total +- age 5: 129024 bytes, 2223400 total +- age 6: 154976 bytes, 2378376 total +: 141769K->140729K(153344K), 0.3807730 secs]2014-06-10T22:56:26.623-0700: 4999535.395: [CMS + (concurrent mode failure): 1920816K->50773K(1926784K), 28.3938140 secs] 2062055K->50773K(2080128K), [CMS Perm : 48657K->41071K(262144K)], 28.7750370 secs] [Times: user=1.65 sys=0.03, real=28.78 secs] + + A normal record looks like this + 2014-06-10T23:00:18.728-0700: 4999767.501: [GC2014-06-10T23:00:18.728-0700: 4999767.501: [ParNew +Desired survivor size 8716288 bytes, new threshold 6 (max 6) +- age 1: 775312 bytes, 775312 total +- age 2: 497168 bytes, 1272480 total +: 138559K->1370K(153344K), 0.0215080 secs] 189332K->52143K(2080128K), 0.0217400 secs] [Times: user=0.04 sys=0.00, real=0.02 secs] + */ diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenericTokens.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenericTokens.java index e46a91c7..9567adc3 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenericTokens.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenericTokens.java @@ -19,58 +19,17 @@ public interface GenericTokens { //Time String TIME = "(-?" + REAL_NUMBER + ")"; String DURATION_MS = TIME + "\\s?ms"; + //0.0700188 String PAUSE_TIME = TIME + "\\s?(?:secs?|ms)"; String CONCURRENT_TIME = PAUSE_TIME; // Date values - String DATE = "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[\\+|\\-]\\d{4}"; + String DATE = "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[+|-]\\d{4}"; // Post 1.7.0_40 clauses //"(\\(.+?\\)\\s?)? ?"; this rule was too liberal -// Permanent Generation Full -// CMS Failure -// Last ditch collection -// promotion failed" -// Update Allocation Context Stats -// Missing GC Cause -// Concurrent Mark Stack Overflow -// young -// -// System.gc() -// FullGCAlot -// ScavengeAlot -// Allocation Profiler -// JvmtiEnv ForceGarbageCollection -// GCLocker Initiated GC -// Heap Inspection Initiated GC -// Heap Dump Initiated GC -// WhiteBox Initiated Young GC -// WhiteBox Initiated Concurrent Mark -// WhiteBox Initiated Full GC -// No GC -// Allocation Failure -// Tenured Generation Full -// Metadata GC Threshold -// Metadata GC Clear Soft References -// CMS Generation Full -// CMS Initial Mark -// CMS Final Remark -// CMS Concurrent Mark -// Old Generation Expanded On Last Scavenge -// Old Generation Too Full To Scavenge -// Ergonomics -// G1 Evacuation Pause -// G1 Humongous Allocation -// Diagnostic Command -// Timer -// Warmup -// Allocation Rate -// Allocation Stall -// Proactive -// ILLEGAL VALUE - last gc cause - ILLEGAL VALUE -// unknown GCCause - - String GC_CAUSE = "(\\([a-zA-Z\\. 1]+?\\(?\\){1,2})?\\s*"; + //"(\\([a-zA-Z\\. 1]+?\\(?\\){1,2})?\\s*"; this rule works, but I prefer the one below + String GC_CAUSE = "(\\([G1,A-Z,a-z, ,-,.gc\\(\\)]+\\))?\\s*"; // (Diagnostic Command) (System.gc()) String SAFE_POINT_CAUSE = "\"(.+)\""; diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedG1GCParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedG1GCParser.java index da3dbf57..f2796fd2 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedG1GCParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedG1GCParser.java @@ -296,8 +296,8 @@ void g1FullAdaptiveSizing(GCLogTrace trace, String line) { private void g1YoungSplitStart(GCLogTrace trace, String line) { timeStampForwardReference = getClock(); - gcCauseForwardReference = trace.gcCause(3, 0); - if ("(young)".equals(trace.getGroup(4))) + gcCauseForwardReference = trace.gcCause(); + if ("(young)".equals(trace.getGroup(7))) collectionTypeForwardReference = GarbageCollectionTypes.Young; else collectionTypeForwardReference = GarbageCollectionTypes.Mixed; @@ -344,6 +344,9 @@ private boolean ignoreFrequentlySeenButUnwantedLines(String line) { if (line.startsWith("Heap after GC invocations=")) return true; if (line.startsWith("OpenJDK")) return true; if (line.equals("}")) return true; + if (line.startsWith("Java HotSpot(TM)")) return true; + if (line.startsWith("CommandLine")) return true; + if (line.startsWith("Memory: ")) return true; return line.contains("Allocation failed. Thread"); } @@ -364,19 +367,18 @@ private void ignore(GCLogTrace trace, String line) { //26.893: [GC pause (G1 Evacuation Pause) (young) (to-space exhausted), 0.1709670 secs] //115.421: [GC pause (G1 Evacuation Pause) (young) (initial-mark) (to-space exhausted), 0.0476190 secs] private void processYoungGenCollection(GCLogTrace trace, String line) { - - boolean initialMark = trace.contains(5, "initial-mark"); + boolean initialMark = trace.contains(8, "initial-mark"); boolean tospaceExhausted = trace.contains(trace.groupCount() - 2, "to-space"); - if (trace.contains(4, "young")) { + if (trace.contains(7, "young")) { if (initialMark) forwardReference = new G1YoungInitialMark(trace.getDateTimeStamp(), trace.gcCause(3, 0), trace.getDoubleGroup(trace.groupCount())); else - forwardReference = new G1Young(trace.getDateTimeStamp(), trace.gcCause(3, 0), trace.getDoubleGroup(trace.groupCount())); + forwardReference = new G1Young(trace.getDateTimeStamp(), trace.gcCause(), trace.getDoubleGroup(trace.groupCount())); if (tospaceExhausted) ((G1Young) forwardReference).toSpaceExhausted(); - } else if (trace.contains(4, "mixed")) { - forwardReference = new G1Mixed(trace.getDateTimeStamp(), trace.gcCause(3, 0), trace.getDoubleGroup(trace.groupCount())); + } else if (trace.contains(7, "mixed")) { + forwardReference = new G1Mixed(trace.getDateTimeStamp(), trace.gcCause(), trace.getDoubleGroup(trace.groupCount())); if (tospaceExhausted) ((G1Mixed) forwardReference).toSpaceExhausted(); } else @@ -388,10 +390,6 @@ private void processYoungGenCollection(GCLogTrace trace, String line) { //1566.108: [GC pause (mixed) 7521K->5701K(13M), 0.0030090 secs] //549.243: [GC pause (young) (initial-mark) 9521K->7824K(13M), 0.0021590 secs] private void processYoung(GCLogTrace trace, String line) { - if (trace.getGroup(7) != null) - trace.notYetImplemented(); - if (trace.getGroup(8) != null) - trace.notYetImplemented(); MemoryPoolSummary summary = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(9); if ("young".equals(trace.getGroup(4))) { G1Young collection = null; @@ -558,7 +556,7 @@ private void g1FullWithConcurrentPhaseStart(GCLogTrace trace, String line) { timeStampForwardReference = getClock(); gcCauseForwardReference = trace.gcCause(); concurrentPhaseStartTimeStamp = trace.getDateTimeStamp(2); - if (!setGarbageCollectionTypeForwardReference(trace.getGroup(7))) + if (!setGarbageCollectionTypeForwardReference(trace.getGroup(13))) trace.notYetImplemented(); } @@ -695,8 +693,8 @@ private void g1gcFullGC(GCLogTrace trace, String line) { forwardReference = new G1SystemGC(trace.getDateTimeStamp(), trace.getPauseTime()); else forwardReference = new G1FullGCNES(trace.getDateTimeStamp(), trace.gcCause(), trace.getPauseTime()); - forwardReference.addMemorySummary(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(4)); - // not thrilled about this "hack".... but currently no way to differentiate between a full with details and a full withoutgit c + forwardReference.addMemorySummary(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(7)); + // not thrilled about this "hack".... but currently no way to differentiate between a full with details and a full without CPU summary if ((diary != null) && (!diary.isPrintGCDetails())) publish(forwardReference); } @@ -785,9 +783,9 @@ private void g1YoungSplitEnd(GCLogTrace trace, String line) { private void g1InitialMark(GCLogTrace trace, String line) { timeStampForwardReference = getClock(); gcCauseForwardReference = trace.gcCause(); - if ("young".equals(trace.getGroup(4))) + if ("young".equals(trace.getGroup(7))) collectionTypeForwardReference = GarbageCollectionTypes.G1GCYoungInitialMark; - else if (trace.contains(4, "mixed")) + else if (trace.contains(7, "mixed")) collectionTypeForwardReference = GarbageCollectionTypes.G1GCMixedInitialMark; else trace.notYetImplemented(); @@ -834,21 +832,19 @@ private void concurrentStringDedup(GCLogTrace trace, String line) { //6.298: [GC remark 6.298: [GC ref-proc, 0.0000570 secs], 0.0010940 secs] //2014-02-21T16:04:24.321-0100: 7.852: [GC remark 2014-02-21T16:04:24.322-0100: 7.853: [GC ref-proc, 0.0000640 secs], 0.0013310 secs] private void g1Remark(GCLogTrace trace, String line) { - G1Remark remark = new G1Remark(trace.getDateTimeStamp(), (trace.getGroup(7) != null) ? trace.getDoubleGroup(7) : 0.0d, trace.getDoubleGroup(trace.groupCount())); - publish(remark); + publish(new G1Remark(trace.getDateTimeStamp(), (trace.getGroup(13) != null) ? trace.getDoubleGroup(13) : 0.0d, trace.getDoubleGroup(trace.groupCount()))); } //2015-04-09T14:28:44.235+0100: 6.597: [GC remark 6.597: [Finalize Marking, 0.0091510 secs] 6.606: [GC ref-proc, 0.0014102 secs] 6.608: [Unloading, 0.0044869 secs], 0.0153351 secs] //public G1Remark( DateTimeStamp timeStamp, double referenceProcessingTimes, double finalizeMarking, double unloading, double duration) private void g1180Remark(GCLogTrace trace, String line) { - G1Remark remark = new G1Remark(trace.getDateTimeStamp(), trace.getDoubleGroup(8), trace.getDoubleGroup(5), trace.getDoubleGroup(11), trace.getDoubleGroup(trace.groupCount())); + G1Remark remark = new G1Remark(trace.getDateTimeStamp(), trace.getDoubleGroup(17), trace.getDoubleGroup(11), trace.getDoubleGroup(23), trace.getDoubleGroup(trace.groupCount())); publish(remark); } private void g1180RemarkRefDetails(GCLogTrace trace, String line) { - G1Remark remark = new G1Remark(trace.getDateTimeStamp(), trace.getDoubleGroup(32), trace.getDoubleGroup(5), trace.getDoubleGroup(trace.groupCount() - 1), trace.getDuration()); - ReferenceGCSummary summary = extractPrintReferenceGC(line); - remark.add(summary); + G1Remark remark = new G1Remark(trace.getDateTimeStamp(), trace.getDoubleGroup(56), trace.getDoubleGroup(11), trace.getDoubleGroup(trace.groupCount() - 1), trace.getDuration()); + remark.add(extractPrintReferenceGC(line)); publish(remark); } @@ -856,7 +852,7 @@ private void g1180RemarkRefDetails(GCLogTrace trace, String line) { //todo: capture memory summary private void g1Cleanup(GCLogTrace trace, String line) { G1Cleanup cleanup = new G1Cleanup(trace.getDateTimeStamp(), trace.getPauseTime()); - cleanup.addMemorySummary(getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 4)); + cleanup.addMemorySummary(getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 7)); publish(cleanup); } @@ -1113,11 +1109,11 @@ private void g1ConcurrentEnd(GCLogTrace trace, String line) { if (concurrentPhaseStartTimeStamp == null) concurrentPhaseStartTimeStamp = getClock(); - if ("root-region-scan".equals(trace.getGroup(4))) { + if ("root-region-scan".equals(trace.getGroup(7))) { publish(new ConcurrentScanRootRegion(concurrentPhaseStartTimeStamp, trace.getDuration())); - } else if ("mark".equals(trace.getGroup(4))) { + } else if ("mark".equals(trace.getGroup(7))) { publish(new G1ConcurrentMark(concurrentPhaseStartTimeStamp, trace.getDuration())); - } else if ("cleanup".equals(trace.getGroup(4))) { + } else if ("cleanup".equals(trace.getGroup(7))) { publish(new G1ConcurrentCleanup(concurrentPhaseStartTimeStamp, trace.getDuration())); } else trace.notYetImplemented(); diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedGCLogParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedGCLogParser.java index 13d3911a..4e5dfb3c 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedGCLogParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedGCLogParser.java @@ -2,10 +2,7 @@ // Licensed under the MIT License. package com.microsoft.gctoolkit.parser; -import com.microsoft.gctoolkit.event.CPUSummary; -import com.microsoft.gctoolkit.event.MemoryPoolSummary; import com.microsoft.gctoolkit.event.generational.CMSRemark; -import com.microsoft.gctoolkit.event.jvm.PermGenSummary; import java.util.logging.Level; import java.util.logging.Logger; @@ -32,47 +29,19 @@ void advanceClock(String record) { } } - // todo: mixes aggregator with parsing. premature optimization... - MemoryPoolSummary getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(GCLogTrace trace, int offset) { - MemoryPoolSummary summary = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(offset); - return summary; - } - - MemoryPoolSummary getTotalOccupancyWithTotalHeapSizeSummary(GCLogTrace trace, int offset) { - MemoryPoolSummary summary = trace.getOccupancyWithMemoryPoolSizeSummary(offset); - return summary; - } - - GCLogTrace extractReferenceBlock(String line, GCParseRule rule) { - return rule.parse(line); - } - - MemoryPoolSummary extractPermGenRecord(GCLogTrace trace) { - int index = (trace.getGroup(2) == null) ? 2 : 4; - return new PermGenSummary(trace.getLongGroup(index), trace.getLongGroup(4), trace.getLongGroup(6)); - } - void recordRescanStepTimes(CMSRemark collection, String line) { GCLogTrace clause; double unloading = 0.0d, symbolTable = 0.0d, stringTable = 0.0d, stringAndSymbolTable = 0.0d; if ((clause = CLASS_UNLOADING.parse(line)) != null) - unloading = clause.getDoubleGroup(2); + unloading = clause.getDoubleGroup(6); if ((clause = SYMBOL_TABLE_SCRUB.parse(line)) != null) - symbolTable = clause.getDoubleGroup(2); + symbolTable = clause.getDoubleGroup(6); if ((clause = STRING_TABLE_SCRUB.parse(line)) != null) - stringTable = clause.getDoubleGroup(2); + stringTable = clause.getDoubleGroup(6); if ((clause = STRING_AND_SYMBOL_SCRUB.parse(line)) != null) - stringAndSymbolTable = clause.getDoubleGroup(2); + stringAndSymbolTable = clause.getDoubleGroup(6); collection.addClassUnloadingAndStringTableProcessingDurations(unloading, symbolTable, stringTable, stringAndSymbolTable); } - - CPUSummary extractCPUSummary(String line) { - GCLogTrace trace; - if ((trace = CPU_BREAKDOWN.parse(line)) != null) { - return new CPUSummary(trace.getDoubleGroup(1), trace.getDoubleGroup(2), trace.getDoubleGroup(3)); - } - return null; - } } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedTokens.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedTokens.java index 42cb544c..856bbae3 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedTokens.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedTokens.java @@ -14,10 +14,12 @@ public interface PreUnifiedTokens extends GenericTokens { String MS_TIME_STAMP = TIME + " ms"; String DATE_STAMP = "(" + DATE + "): "; - String DATE_TIMESTAMP = "(?:" + DATE_STAMP + ")?" + TIMESTAMP; - //DECIMAL_POINT = "(?:\\.|,)"; - // String INTEGER = "\\d+"; - //"(?:(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[\\+|\\-]\\d{4}): )?(" + "\\d+(?:\\.|,)\\d{3}): (?:#\\d+: )?\\[GC ?(\\([a-zA-Z\\. 1]+?\\(?\\){1,2})?\\s*"; + // String dateStampPattern = "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}-\\d{4}: )"; + // String timeStampPattern = "(\\d+.\\d{3}: )"; + // + // Pattern combined = Pattern.compile(dateStampPattern + timeStampPattern + "|" + timeStampPattern + "|" + dateStampPattern); + int TIME_DATE_OFFSET = 6; + String DATE_TIMESTAMP = "(" + DATE_STAMP + TIMESTAMP + "|" + TIMESTAMP + "|" + DATE_STAMP + ")"; GCParseRule DATE_TIMESTAMP_RECORD = new GCParseRule("DATE_TIMESTAMP_RECORD", DATE_TIMESTAMP); // Memory values @@ -28,8 +30,9 @@ public interface PreUnifiedTokens extends GenericTokens { String FRACTIONAL_BEFORE_AFTER_CONFIGURED = FRACTIONAL_MEMORY_SIZE + "->" + FRACTIONAL_MEMORY_SIZE + "\\(" + FRACTIONAL_MEMORY_SIZE + "\\)"; // GC Prefixes - String GC_PREFIX = DATE_TIMESTAMP + "(?:#\\d+: )?\\[GC ?" + GC_CAUSE; - String FULL_GC_PREFIX = DATE_TIMESTAMP + "(?:#\\d+: )?\\[Full GC ?" + GC_CAUSE; + String GC_ID = "(?:#\\\\d+: )?"; + String GC_PREFIX = DATE_TIMESTAMP + GC_ID + "\\[GC ?" + GC_CAUSE; + String FULL_GC_PREFIX = DATE_TIMESTAMP + GC_ID + "\\[Full GC ?" + GC_CAUSE; // Pre 1.8.0 records // Update: no longer only pre 1.8.0, updated to fix Metaspace in CMF records diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java index 419c9028..55deebb6 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java @@ -191,7 +191,6 @@ private void parse(String line) { } final String lineAfterGcId = line.substring(end); - parseRules.stream() .map(Map.Entry::getKey) .map(rule -> new AbstractMap.SimpleEntry<>(rule, rule.parse(lineAfterGcId))) @@ -375,7 +374,7 @@ private void g1Collection(GCLogTrace trace, String line) { } } forwardReference.setGcType(gcType); - forwardReference.setGCCause(trace.gcCause(1)); + forwardReference.setGCCause(trace.gcCause(-2)); forwardReference.setStartTime(getClock()); } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java index d23815de..5a06bca8 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java @@ -293,7 +293,7 @@ private void collector(String line) { diary.setTrue(SupportedFlags.GC_DETAILS); diary.setFalse(SupportedFlags.TENURING_DISTRIBUTION); diary.setFalse(SupportedFlags.PRINT_HEAP_AT_GC); - setGCCause(trace.getGroup(3)); + setGCCause(trace.getGroup(6)); } else if (ParallelPatterns.PS_DETAILS_WITH_TENURING.parse(line) != null) { diary.setTrue(SupportedFlags.GC_DETAILS); @@ -308,33 +308,33 @@ private void collector(String line) { if ((trace = CMSPatterns.PARNEW.parse(line)) != null) { diary.setTrue(SupportedFlags.GC_DETAILS); diary.setFalse(SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); - setGCCause(trace.getGroup(4)); + setGCCause(trace.getGroup(7)); } else if ((trace = CMSPatterns.PARNEW_TENURING.parse(line)) != null) { diary.setTrue(SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION); diary.setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1); - setGCCause(trace.getGroup(3)); + setGCCause(trace.getGroup(6)); } else if ((trace = SimplePatterns.PARNEW_NO_DETAILS.parse(line)) != null) { diary.setFalse(SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); - setGCCause(trace.getGroup(3)); + setGCCause(trace.getGroup(6)); } else if ((trace = SimplePatterns.PARNEW_START.parse(line)) != null) { diary.setFalse(SupportedFlags.GC_DETAILS, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC); - setGCCause(trace.groupCount() > 2 ? trace.getGroup(3) : null); + setGCCause(trace.groupCount() > 5 ? trace.getGroup(6) : null); } else if ((trace = CMSPatterns.PARNEW_REFERENCE_SPLIT.parse(line)) != null) { diary.setTrue(SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.PRINT_REFERENCE_GC); - setGCCause(trace.getGroup(3)); + setGCCause(trace.getGroup(6)); } } else if ((trace = SerialPatterns.DEFNEW.parse(line)) != null) { collectionCount++; youngCollectionCount++; diary.setTrue(SupportedFlags.DEFNEW, SupportedFlags.GC_DETAILS); diary.setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.PARNEW, SupportedFlags.G1GC, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.RSET_STATS, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); - setGCCause(trace.getGroup(3)); + setGCCause(trace.getGroup(6)); } else if ((trace = SerialPatterns.DEFNEW_TENURING.parse(line)) != null) { collectionCount++; youngCollectionCount++; diary.setTrue(SupportedFlags.DEFNEW, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION); diary.setFalse(SupportedFlags.PARNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.RSET_STATS); - setGCCause(trace.getGroup(3)); + setGCCause(trace.getGroup(6)); } else if (SimplePatterns.PARNEW_NO_DETAILS.parse(line) != null) { collectionCount++; youngCollectionCount++; @@ -356,7 +356,7 @@ private void collector(String line) { simpleParallelOrParNewDetected = true; youngCollectionCount++; diary.setFalse(SupportedFlags.G1GC, SupportedFlags.RSET_STATS); - setGCCause(trace.getGroup(3)); + setGCCause(trace.getGroup(6)); } else if (SimplePatterns.CMS_NO_DETAILS.parse(line) != null) { // could be parallel or CMS.. look for Full GC but even that may be a trick collectionCount++; @@ -374,7 +374,7 @@ private void collector(String line) { } else if (trace.gcCause() == GCCause.METADATA_GENERATION_THRESHOLD) { diary.setTrue(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); - } else if ((trace.gcCause() == GCCause.G1_EVACUATION_PAUSE) || (trace.gcCause(3, 0) == GCCause.G1_HUMONGOUS_ALLOCATION)) { + } else if ((trace.gcCause() == GCCause.G1_EVACUATION_PAUSE) || (trace.gcCause() == GCCause.G1_HUMONGOUS_ALLOCATION)) { diary.setTrue(SupportedFlags.GC_CAUSE); diary.setFalse(SupportedFlags.PRE_JDK70_40); } @@ -383,7 +383,7 @@ private void collector(String line) { youngCollectionCount++; diary.setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS); diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); - if (trace.gcCause(3, 0) == GCCause.GCCAUSE_NOT_SET) { + if (trace.gcCause() == GCCause.GCCAUSE_NOT_SET) { diary.setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); diary.setTrue(SupportedFlags.JDK70); } else { @@ -403,7 +403,7 @@ private void collector(String line) { youngCollectionCount++; diary.setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION); diary.setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.PRINT_REFERENCE_GC); - if (trace.getGroup(3) != null) + if (trace.getGroup(6) != null) diary.setTrue(SupportedFlags.GC_CAUSE); else diary.setFalse(SupportedFlags.GC_CAUSE); @@ -518,10 +518,10 @@ else if (youngCountAfterFirstCMSCycle > 1) } private void checkForGCCause(GCLogTrace trace) { - if (trace.gcCause(3, 0) == GCCause.METADATA_GENERATION_THRESHOLD) { + if (trace.gcCause() == GCCause.METADATA_GENERATION_THRESHOLD) { diary.setTrue(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); - } else if ((trace.gcCause(3, 0) == GCCause.G1_EVACUATION_PAUSE) || (trace.gcCause(3, 0) == GCCause.G1_HUMONGOUS_ALLOCATION)) { + } else if ((trace.gcCause() == GCCause.G1_EVACUATION_PAUSE) || (trace.gcCause() == GCCause.G1_HUMONGOUS_ALLOCATION)) { diary.setTrue(SupportedFlags.GC_CAUSE); } } @@ -573,18 +573,18 @@ private void version(String line) { diary.setTrue(SupportedFlags.GC_CAUSE); } else if (diary.isGenerationalKnown() && diary.isGenerational()) { if ((trace = PREFIX.parse(line)) != null) { - if ((trace.getGroup(3) == null) && diary.isTrue(SupportedFlags.GC_DETAILS)) { + if ((trace.getGroup(6) == null) && diary.isTrue(SupportedFlags.GC_DETAILS)) { diary.setTrue(SupportedFlags.JDK70); diary.setFalse(SupportedFlags.JDK80, SupportedFlags.GC_CAUSE); - } else if (trace.gcCause(3, 0) != GCCause.GCCAUSE_NOT_SET) { + } else if (trace.gcCause() != GCCause.GCCAUSE_NOT_SET) { diary.setTrue(SupportedFlags.GC_CAUSE); diary.setFalse(SupportedFlags.PRE_JDK70_40); } } else if ((trace = FULL_PREFIX.parse(line)) != null) { - if ((trace.getGroup(3) == null) && diary.isTrue(SupportedFlags.GC_DETAILS)) { + if ((trace.getGroup(6) == null) && diary.isTrue(SupportedFlags.GC_DETAILS)) { diary.setTrue(SupportedFlags.JDK70); diary.setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80); - } else if (trace.gcCause(3, 0) != GCCause.GCCAUSE_NOT_SET) { + } else if (trace.gcCause() != GCCause.GCCAUSE_NOT_SET) { diary.setTrue(SupportedFlags.GC_CAUSE); diary.setFalse(SupportedFlags.PRE_JDK70_40); } @@ -613,7 +613,7 @@ private void details(String line) { GCLogTrace trace; if ((trace = PREFIX.parse(line)) != null) { - String cause = trace.getGroup(3); + String cause = trace.getGroup(6); if (cause != null) { diary.setTrue(SupportedFlags.GC_CAUSE); if ("(System)".equals(cause)) { @@ -627,7 +627,7 @@ private void details(String line) { } } else { if ((trace = G1GC_PREFIX.parse(line)) != null) { - cause = trace.getGroup(3); + cause = trace.getGroup(6); if (cause == null) diary.setTrue(SupportedFlags.PRE_JDK70_40); else diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepPhaseParserRulesTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepPhaseParserRulesTest.java index 403fb772..ef9b6c9e 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepPhaseParserRulesTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepPhaseParserRulesTest.java @@ -7,6 +7,8 @@ import java.util.logging.Logger; +import static com.microsoft.gctoolkit.parser.CommonTestHelper.captureTest; +import static com.microsoft.gctoolkit.parser.ICMSPatterns.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -17,15 +19,17 @@ public class ConcurrentMarkSweepPhaseParserRulesTest implements CMSPatterns { @Test public void testCMSParseRules() { - for (int i = 0; i < rules.length; i++) + assertEquals(rules.length,lines.length, "rules length != # of groups of lines to be test"); + for (int i = 0; i < rules.length; i++) { for (int j = 0; j < lines.length; j++) { - GCLogTrace trace = rules[i].parse(lines[j]); - if (trace != null) { - assertEquals(i, j, i + " captured " + j); + int captured = captureTest(rules[i], lines[j]); + if (i == j) { + assertEquals(captured, lines[j].length, i + " failed to captured it's lines"); } else { - assertTrue(i != j, i + " captured " + j); + assertEquals(0, captured, i + " captured " + j); } } + } assertTrue(true); } @@ -33,11 +37,11 @@ public void testCMSParseRules() { /* Code that is useful when testing individual records */ - @Test + //@Test public void testDebugCMSParseRules() { - int index = 0; - GCParseRule rule = rules[index]; - evaluate(rule, lines[index]); + int index = 7; + for (String line : lines[index]) + evaluate(rules[index], line); } @@ -45,6 +49,7 @@ private void evaluate(GCParseRule rule, String string) { GCLogTrace trace = rule.parse(string); assertNotNull(trace); + trace.notYetImplemented(); // Enable debugging by setting gctoolkit.debug to true GCToolKit.LOG_DEBUG_MESSAGE(() -> { StringBuilder sb = new StringBuilder("matches groups " + trace.groupCount()); @@ -60,12 +65,50 @@ private void evaluate(GCParseRule rule, String string) { //168027.437: [Rescan (parallel) , 0.0387024 secs]168027.478: [weak refs processing, 0.0700188 secs] [1 CMS-remark: 1396579K(2293760K)] 1403108K(3014656K), 0.2027973 secs] [Times: user=1.25 sys=0.06, real=0.20 secs] private GCParseRule[] rules = { SPLIT_REMARK, - PRECLEAN_REFERENCE + PRECLEAN_REFERENCE, + CORRUPTED_PARNEW_BODY, + PARNEW_DETAILS_WITH_CONCURRENT_MODE_FAILURE, + //new GCParseRule("PARNEW_DETAILS_WITH_CONCURRENT_MODE_FAILURE", "^: " + BEFORE_AFTER_CONFIGURED_PAUSE + "\\]" + CMS_TENURED_BLOCK), // + " " + BEFORE_AFTER_CONFIGURED + ", " + PERM_RECORD + ", " + PAUSE_TIME + "\\]"), + iCMS_PARNEW_PROMOTION_FAILURE_RECORD, //todo: iCMS test??? + iCMS_FULL, + iCMS_PARNEW_PROMOTION_FAILURE, + //iCMS_PARNEW_PROMOTION_FAILURE_RECORD //todo: matches the rule above... + PARNEW_CONCURRENT_MODE_FAILURE_PERM, + PARNEW_CONCURRENT_MODE_FAILURE_META //todo: no matching data }; - private String[] lines = { - "2015-08-04T19:50:56.691-0400: 168027.353: [GC[YG occupancy: 64589 K (720896 K)]2015-08-04T19:50:56.693-0400: 168027.355: [GC 168027.355: [ParNew", - //"168027.437: [Rescan (parallel) , 0.0387024 secs]168027.478: [weak refs processing, 0.0700188 secs] [1 CMS-remark: 1396579K(2293760K)] 1403108K(3014656K), 0.2027973 secs] [Times: user=1.25 sys=0.06, real=0.20 secs]" - "2016-10-06T08:48:07.320+0200: 2002,085: [Preclean SoftReferences, 0,0000050 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean WeakReferences, 0,0000283 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean FinalReferences, 0,0000036 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean PhantomReferences, 0,0000354 secs]2016-10-06T08:48:07.330+0200: 2002,095: [CMS-concurrent-preclean: 0,010/0,011 secs] [Times: user=0,01 sys=0,00, real=0,01 secs]" + private String[][] lines = { + { // 0 + "2015-08-04T19:50:56.691-0400: 168027.353: [GC[YG occupancy: 64589 K (720896 K)]2015-08-04T19:50:56.693-0400: 168027.355: [GC 168027.355: [ParNew" + }, + { // 1 + //168027.437: [Rescan (parallel) , 0.0387024 secs]168027.478: [weak refs processing, 0.0700188 secs] [1 CMS-remark: 1396579K(2293760K)] 1403108K(3014656K), 0.2027973 secs] [Times: user=1.25 sys=0.06, real=0.20 secs] + "2016-10-06T08:48:07.320+0200: 2002.085: [Preclean SoftReferences, 0.0000050 secs]2016-10-06T08:48:07.320+0200: 2002.085: [Preclean WeakReferences, 0.0000283 secs]2016-10-06T08:48:07.320+0200: 2002.085: [Preclean FinalReferences, 0.0000036 secs]2016-10-06T08:48:07.320+0200: 2002.085: [Preclean PhantomReferences, 0.0000354 secs]2016-10-06T08:48:07.330+0200: 2002.095: [CMS-concurrent-preclean: 0.010/0.011 secs] [Times: user=0.01 sys=0.00, real=0.01 secs]", + //"2016-10-06T08:48:07.320+0200: 2002,085: [Preclean SoftReferences, 0,0000050 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean WeakReferences, 0,0000283 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean FinalReferences, 0,0000036 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean PhantomReferences, 0,0000354 secs]2016-10-06T08:48:07.330+0200: 2002,095: [CMS-concurrent-preclean: 0,010/0,011 secs] [Times: user=0,01 sys=0,00, real=0,01 secs]" + }, + { // 2 + "(promotion failed): 118016K->118016K(118016K), 0.0288030 secs]17740.440: [CMS (concurrent mode failure): 914159K->311550K(917504K), 0.5495730 secs] 985384K->311550K(1035520K), [CMS Perm : 65977K->65950K(131072K)], 0.5785090 secs] [Times: user=0.67 sys=0.01, real=0.58 secs]" + }, + { // 3 + ": 19134K->19136K(19136K), 0.0493809 secs]236.955: [CMS: 107351K->79265K(107776K), 0.2540576 secs] 119733K->79265K(126912K), [CMS Perm : 14256K->14256K(24092K)], 0.3036551 secs]", + ": 1069879K->1069879K(1090560K), 0.3135220 secs]2014-09-19T06:07:23.135+0200: 73512.294: [CMS: 1613084K->823344K(2423488K), 3.5186340 secs] 2639961K->823344K(3514048K), [CMS Perm : 205976K->205949K(343356K)], 3.8323790 secs] [Times: user=4.44 sys=0.00, real=3.83 secs]" + }, + { // 4 + "445909.040: [GC 445909.040: [ParNew (1: promotion failure size = 4629668) (promotion failed): 460096K->460096K(460096K), 0.8467130 secs]445909.887: [CMS: 6252252K->2709964K(7877440K), 23.9213270 secs] 6637399K->2709964K(8337536K), [CMS Perm : 1201406K->68157K(2097152K)] icms_dc=0 , 24.7685320 secs]" + }, + { // 5 + "2013-06-06T14:12:49.554+0200: 534744,148: [Full GC (System) 534744,148: [CMS: 1513767K->410320K(3512768K), 2,1361260 secs] 1598422K->410320K(4126208K), [CMS Perm : 120074K->119963K(200424K)] icms_dc=0 , 2,1363550 secs]", + "619930,816: [Full GC 619930,816: [CMS: 3512768K->3512767K(3512768K), 17,8327610 secs] 4126207K->4073974K(4126208K), [CMS Perm : 120062K->120053K(201192K)] icms_dc=100 , 17,8329750 secs]" + }, + { // 6 + "2016-08-17T10:41:41.088-0500: 76388.020: [GC (Allocation Failure) 2016-08-17T10:41:41.088-0500: 76388.020: [ParNew (promotion failed): 428321K->431492K(471872K), 0.1049894 secs]2016-08-17T10:41:41.193-0500: 76388.125: [CMS: 466697K->186596K(524288K), 0.7357085 secs] 893715K->186596K(996160K), [Metaspace: 46524K->46524K(1097728K)] icms_dc=0 , 0.8408171 secs] [Times: user=0.91 sys=0.00, real=0.84 secs]", + "72825.712: [GC72825.712: [ParNew (promotion failed): 153344K->153344K(153344K), 0.3895590 secs]72826.102: [CMS: 1831960K->1554399K(1926784K), 8.3796720 secs] 1914705K->1554399K(2080128K), [Metaspace: 131704K->130876K(222768K)] icms_dc=42 , 8.7694530 secs]" + }, + { // 7 + "2015-02-04T17:36:07.103-0500: 199626.435: [GC 199626.435: [ParNew: 812672K->812672K(914240K), 0.0000400 secs]199626.435: [CMS (concurrent mode failure): 1071394K->1081343K(1081344K), 6.8504740 secs] 1884066K->1092775K(1995584K), [CMS Perm : 99417K->99411K(524288K)], 6.8510440 secs] [Times: user=6.63 sys=0.02, real=6.85 secs]", + "199626.435: [GC 199626.435: [ParNew: 812672K->812672K(914240K), 0.0000400 secs]199626.435: [CMS (concurrent mode failure): 1071394K->1081343K(1081344K), 6.8504740 secs] 1884066K->1092775K(1995584K), [CMS Perm : 99417K->99411K(524288K)], 6.8510440 secs] [Times: user=6.63 sys=0.02, real=6.85 secs]" + }, + { // 8 + } }; } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCParserRulesTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCParserRulesTest.java index b3c9f27e..644897a4 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCParserRulesTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCParserRulesTest.java @@ -18,7 +18,7 @@ public void testG1GCParseRules() { for (int j = 0; j < lines.length; j++) { int captured = CommonTestHelper.captureTest(rules[i], lines[j]); if (i == j) { - assertEquals(captured, lines[j].length, i + " failed to captured it's lines"); + assertEquals(lines[j].length, captured, i + " failed to captured it's lines"); } else { assertEquals(0, captured, i + " captured " + j); } @@ -59,7 +59,7 @@ private void evaluate(GCParseRule rule, String string, boolean dump) { G1_FULL_ADAPTIVE_SIZING, FULLGC_WITH_CONCURRENT_PHASE, G1_180_REMARK_REF_DETAILS, // 10 - G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END, + G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END, //captured by 32 G1_FLOATING_CONCURRENT_PHASE_START, PARALLEL_TIME, G1_PARALLEL_PHASE_SUMMARY, @@ -80,16 +80,15 @@ private void evaluate(GCParseRule rule, String string, boolean dump) { CORRUPTED_CONCURRENT_START_V3, FULL_WITH_CONCURRENT_END, // 30 CORRUPTED_CONCURRENT_START_V5, - CORRUPTED_CONCURRENT_START_V6, FULL_MISSING_TIMESTAMP_CONCURRENT_START, CORRUPTED_CONCURRENT_START_V7, - CORRUPTED_CONCURRENT_START_V8, // 35 - CORRUPTED_CONCURRENT_START_V9, + CORRUPTED_CONCURRENT_START_V8, + CORRUPTED_CONCURRENT_START_V9, // 35 FULL_GC_FRAGMENT, CORRUPTED_CONCURRENT_START_V2, CONCURRENT_START_V3, - CONCURRENT_START_V4, // 40 - CONCURRENT_START_V5, + CONCURRENT_START_V4, + CONCURRENT_START_V5, // 40 G1_CONCURRENT_ABORT }; @@ -124,14 +123,16 @@ private void evaluate(GCParseRule rule, String string, boolean dump) { }, { // 9 "100081.540: [Full GC100081.540: [GC concurrent-root-region-scan-start]", - "2018-01-29T17:34:24.984+0000: 5115.588: [Full GC (Metadata GC Threshold) 2018-01-29T17:34:24.984+0000: 5115.588: [GC concurrent-root-region-scan-start]" + "2018-01-29T17:34:24.984+0000: 5115.588: [Full GC (Metadata GC Threshold) 2018-01-29T17:34:24.984+0000: 5115.588: [GC concurrent-root-region-scan-start]", +// "2018-02-08T20:13:02.246+0000: 878632.850: 2018-02-08T20:13:02.246+0000: [Full GC (Metadata GC Threshold) 878632.850: [GC concurrent-root-region-scan-start]" }, { // 10 "2015-12-21T10:26:34.913-0500: 35835.169: [GC remark 35835.169: [Finalize Marking, 0.8985905 secs] 35836.068: [GC ref-proc35836.068: [SoftReference, 74 refs, 0.0002156 secs]35836.068: [WeakReference, 243 refs, 0.0001747 secs]35836.068: [FinalReference, 4154 refs, 0.0012360 secs]35836.070: [PhantomReference, 0 refs, 0 refs, 0.0002882 secs]35836.070: [JNI Weak Reference, 0.0000227 secs], 0.0026411 secs] 35836.071: [Unloading, 0.0048615 secs], 0.9142368 secs]", "2016-05-31T12:49:30.282-0400: 2328.858: [GC remark 2016-05-31T12:49:30.282-0400: 2328.858: [Finalize Marking, 0.0020922 secs] 2016-05-31T12:49:30.284-0400: 2328.860: [GC ref-proc2016-05-31T12:49:30.284-0400: 2328.860: [SoftReference, 0 refs, 0.0000754 secs]2016-05-31T12:49:30.284-0400: 2328.860: [WeakReference, 3894 refs, 0.0012230 secs]2016-05-31T12:49:30.285-0400: 2328.861: [FinalReference, 102 refs, 0.0002524 secs]2016-05-31T12:49:30.286-0400: 2328.861: [PhantomReference, 2 refs, 231 refs, 0.0004237 secs]2016-05-31T12:49:30.286-0400: 2328.862: [JNI Weak Reference, 0.0003428 secs], 0.0023558 secs] 2016-05-31T12:49:30.286-0400: 2328.862: [Unloading, 0.0457079 secs], 0.0556017 secs]" }, { // 11 - "2017-01-24T22:50:27.724-0500: 21438.188: 2017-01-24T22:50:27.724-0500: [GC concurrent-root-region-scan-end, 0.0303627 secs]" + "2017-01-24T22:50:27.724-0500: 21438.188: 2017-01-24T22:50:27.724-0500: [GC concurrent-root-region-scan-end, 0.0303627 secs]", + "2018-02-08T20:27:32.297+0000: 879502.901: 2018-02-08T20:27:32.297+0000: 879502.901: [GC concurrent-root-region-scan-end, 0.0000375 secs]" }, { // 12 "[GC concurrent-mark-start]" @@ -207,40 +208,37 @@ private void evaluate(GCParseRule rule, String string, boolean dump) { { // 30 ": [Full GC (Metadata GC Threshold) 2018-02-08T20:04:14.979+0000: 878105.583: [GC concurrent-root-region-scan-end, 0.0000770 secs]", }, - { // 32 + { // 31 "2018-02-08T20:21:26.246+0000: 2018-02-08T20:21:26.246+0000879136.850: [Full GC (Metadata GC Threshold) : 879136.850: [GC concurrent-root-region-scan-start]" }, - { // 33 - "2018-02-08T20:27:32.297+0000: 879502.901: 2018-02-08T20:27:32.297+0000: 879502.901: [GC concurrent-root-region-scan-end, 0.0000375 secs]" - }, - { // 34 + { // 32 "[Full GC (Metadata GC Threshold) 2018-02-08T20:27:32.297+0000: 879502.902: [GC concurrent-mark-start]" }, - { + { // 33 "2018-02-08T20:31:33.633+0000: 2018-02-08T20:31:33.633+0000: 879744.238879744.238: [Full GC (Metadata GC Threshold) : [GC concurrent-root-region-scan-start]" }, - { + { // 34 "2018-02-08T20:31:54.234+0000: 2018-02-08T20:31:54.234+0000879764.839: 879764.839: [Full GC (Metadata GC Threshold) : [GC concurrent-root-region-scan-start]" }, - { + { // 35 "2018-02-08T20:32:51.258+0000: 2018-02-08T20:32:51.258+0000: 879821.862: 879821.862[Full GC (Metadata GC Threshold) : [GC concurrent-root-region-scan-start]" }, - { + { // 36 "2018-02-08T20:35:03.232+0000: 879953.837: [Full GC (Metadata GC Threshold)" }, - { + { // 37 "2018-02-08T20:05:15.505+0000: 878166.1102018-02-08T20:05:15.505+0000: : 878166.110: [GC concurrent-root-region-scan-start]" }, - { + { // 38 "2018-02-08T20:32:54.965+0000: 2018-02-08T20:32:54.965+0000: 879825.569: [GC concurrent-root-region-scan-start]" }, - { + { // 39 "2018-02-08T20:33:01.419+0000: 2018-02-08T20:33:01.419+0000879832.023: [GC concurrent-root-region-scan-start]" }, - { + { // 40 "2018-02-08T20:34:59.393+0000: 2018-02-08T20:34:59.393+0000: 879949.998: 879949.998[GC concurrent-root-region-scan-start]" }, - { + { // 41 "27105.565: [GC concurrent-mark-abort]" } }; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedCMSTenuredParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/PreUnifiedCMSTenuredParserTest.java similarity index 98% rename from parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedCMSTenuredParserTest.java rename to parser/src/test/java/com/microsoft/gctoolkit/parser/PreUnifiedCMSTenuredParserTest.java index 69dcf3b1..0f5d6142 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedCMSTenuredParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/PreUnifiedCMSTenuredParserTest.java @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package com.microsoft.gctoolkit.parser.patterns; +package com.microsoft.gctoolkit.parser; import com.microsoft.gctoolkit.event.generational.AbortablePreClean; @@ -12,9 +12,6 @@ import com.microsoft.gctoolkit.event.generational.InitialMark; import com.microsoft.gctoolkit.event.jvm.JVMEvent; import com.microsoft.gctoolkit.jvm.Diarizer; -import com.microsoft.gctoolkit.parser.CMSTenuredPoolParser; -import com.microsoft.gctoolkit.parser.GCLogParser; -import com.microsoft.gctoolkit.parser.ParserTest; import com.microsoft.gctoolkit.parser.jvm.PreUnifiedDiarizer; import org.junit.jupiter.api.Test; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/PreUnifiedGenerationalParserTest.java similarity index 98% rename from parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java rename to parser/src/test/java/com/microsoft/gctoolkit/parser/PreUnifiedGenerationalParserTest.java index 02fb973b..a7139019 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/PreUnifiedGenerationalParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/PreUnifiedGenerationalParserTest.java @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package com.microsoft.gctoolkit.parser.patterns; +package com.microsoft.gctoolkit.parser; import com.microsoft.gctoolkit.event.GCCause; import com.microsoft.gctoolkit.event.generational.AbortablePreClean; @@ -20,9 +20,6 @@ import com.microsoft.gctoolkit.event.jvm.JVMEvent; import com.microsoft.gctoolkit.io.GCLogFile; import com.microsoft.gctoolkit.jvm.Diarizer; -import com.microsoft.gctoolkit.parser.GCLogParser; -import com.microsoft.gctoolkit.parser.GenerationalHeapParser; -import com.microsoft.gctoolkit.parser.ParserTest; import com.microsoft.gctoolkit.parser.jvm.PreUnifiedDiarizer; import org.junit.jupiter.api.Test; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/G1GCPatternsTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/G1GCPatternsTest.java index 702d6357..8a7bc7fe 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/G1GCPatternsTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/G1GCPatternsTest.java @@ -3,6 +3,7 @@ package com.microsoft.gctoolkit.parser.patterns; import com.microsoft.gctoolkit.parser.G1GCPatterns; +import com.microsoft.gctoolkit.parser.GCLogTrace; import com.microsoft.gctoolkit.parser.GCParseRule; import org.junit.jupiter.api.Test; @@ -10,6 +11,7 @@ import static com.microsoft.gctoolkit.parser.CommonTestHelper.captureTest; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; public class G1GCPatternsTest implements G1GCPatterns { @@ -21,20 +23,31 @@ public void testParallelParseRules() { for (int j = 0; j < lines.length; j++) { int captured = captureTest(rules[i], lines[j]); if (i == j) { - assertEquals(captured, lines[j].length, i + " failed to captured it's lines"); + assertEquals( lines[j].length, captured, rules[i].getName() + " failed to captured it's lines"); } else { - assertEquals(0, captured, i + " captured " + j); + if ( captured != 0) + for( int k = 0; k < lines[j].length; k++) { + rules[i].parse(lines[j][k]).notYetImplemented(); + System.out.println(lines[j][k]); + } + assertEquals(0, captured, rules[i].getName() + " captured data from group " + j); } } } + @Test + public void testRuleThatShouldFail() { + // missing timestamp + assertNull(G1_YOUNG_SPLIT_START.parse("[GC pause (G1 Evacuation Pause) (young)")); + } + /* Code that is useful when testing individual records */ // private boolean debugging = Boolean.getBoolean("microsoft.debug"); // // @Test // public void testDebugParallelParseRules() { -// int index = 10; +// int index = 1; // GCParseRule rule = rules[index]; // assertEquals(lines[index].length, captureTest( rule, lines[index])); // } @@ -52,7 +65,8 @@ public void testParallelParseRules() { SPLIT_CLEANUP, FULL_WITH_CONCURRENT_PHASE_START, DELAY_MIXED_GC, - START_MIXED_GC + START_MIXED_GC, + G1_YOUNG_SPLIT_START }; private String[][] lines = { @@ -94,7 +108,7 @@ public void testParallelParseRules() { "369310.802: [GC pause (young) 485M->240M(512M), 0.0558340 secs]", "369447.597: [GC pause (young) (initial-mark) 485M->239M(512M), 0.0719290 secs]", "369674.919: [GC pause (mixed) 482M->185M(512M), 0.0679470 secs]", - "0.583: [GC pause (G1 Evacuation Pause) (young) 24M->4561K(256M), 0.0047007 secs]" + "0.583: [GC pause (G1 Evacuation Pause) (young) 24M->4561K(256M), 0.0047007 secs]", }, { // FULL_GC, no details 1.8.0_102 "16.603: [Full GC (System.gc()) 14M->3334K(11M), 0.0230975 secs]" @@ -110,6 +124,11 @@ public void testParallelParseRules() { }, { "7.986: [G1Ergonomics (Mixed GCs) do not start mixed GCs, reason: reclaimable percentage not over threshold, candidate old regions: 3 regions, reclaimable: 1380376 bytes (2.53 %), threshold: 10.00 %]" + }, + { + "2017-03-09T13:45:26.322-0500: 4.764: [GC pause (G1 Evacuation Pause) (young)", + "4.764: [GC pause (G1 Evacuation Pause) (young)", + "2017-03-09T13:45:26.322-0500: [GC pause (G1 Evacuation Pause) (young)", } }; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/GCCausePatternTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/GCCausePatternTest.java new file mode 100644 index 00000000..50f3ed8e --- /dev/null +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/GCCausePatternTest.java @@ -0,0 +1,95 @@ +package com.microsoft.gctoolkit.parser.patterns; + +import com.microsoft.gctoolkit.parser.GCLogTrace; +import com.microsoft.gctoolkit.parser.GCParseRule; +import com.microsoft.gctoolkit.parser.GenericTokens; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class GCCausePatternTest implements GenericTokens { + + String[] gcCauses = { + "(System.gc())", + "(Diagnostic Command)", + "(FullGCALot)", + "(ScavengeAlot)", + "(Allocation Profiler)", + "(JvmtiEnv ForceGarbageCollection)", + "(GCLocker Initiated GC)", + "(Heap Inspection Initiated GC)", + "(Heap Dump Initiated GC)", + "(No GC)", + "(Allocation Failure)", + "(Tenured Generation Full)", + "(Metadata GC Threshold)", + "(Permanent Generation Full)", + "(CMS Generation Full)", + "(CMS Initial Mark)", + "(CMS Final Remark)", + "(CMS Concurrent Mark)", + "(CMS Failure)", + "(Old Generation Expanded On Last Scavenge)", + "(Old Generation Too Full To Scavenge)", + "(Ergonomics)", + "(G1 Evacuation Pause)", + "(G1 Humongous Allocation)", + "(Last ditch collection)", + "(ILLEGAL VALUE - last gc cause - ILLEGAL VALUE)", + "(unknown GCCause)", + "(promotion failed)", + "(Update Allocation Context Stats)", + "(Missing GC Cause)", + "(Concurrent Mark Stack Overflow)", + "(young)", + "(WhiteBox Initiated Young GC)", + "(WhiteBox Initiated Concurrent Mark)", + "(WhiteBox Initiated Full GC)", + "(Metadata GC Clear Soft References)", + "(Timer)", + "(Warmup)", + "(Allocation Rate)", + "(Allocation Stall)", + "(Proactive)", + "(G1 Preventive Collection)" + }; + + String[] extraText = { + "(System.gc()) xxx", + "(Diagnostic Command) xxx", + "(ScavengeAlot) xxx", + "(GCLocker Initiated GC) xxx", + "(Allocation Failure) xxx", + "(Metadata GC Threshold) xxx", + "(CMS Initial Mark) xxx", + "(CMS Final Remark) xxx", + "(CMS Concurrent Mark) xxx", + "(Old Generation Expanded On Last Scavenge) xxx", + "(Ergonomics) xxx", + "(G1 Evacuation Pause) xxx", + "(G1 Humongous Allocation) xxx", + "(ILLEGAL VALUE - last gc cause - ILLEGAL VALUE) xxx" + }; + + @Test + public void matchesAllGCCauses() { + String local = "\\([G1,A-Z,a-z, ,-,.gc\\(\\)]+\\)"; + //\([G1,A-Z,a-z, ,-]+\) + //GCParseRule cause = new GCParseRule("GC_CAUSE", local); + GCParseRule cause = new GCParseRule("GC_CAUSE", GC_CAUSE); + for (int i = 0; i < gcCauses.length; i++) { + assertNotNull(cause.parse(gcCauses[i])); + } + } + + @Test + public void onlyMatchCause() { + GCParseRule cause = new GCParseRule("GC_CAUSE", GC_CAUSE); + for (String s : extraText) { + GCLogTrace trace = cause.parse(s); + assertNotNull(trace); + assertFalse(trace.contains("xxx")); + } + } +} diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java index 8dd2f1bb..2a45703f 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/CMSParNewParserTest.java @@ -84,8 +84,8 @@ public void testForDetails() { private static final int[][] detailsCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 - {0, 0, 3668, 12, 0, 96, 0, 0, 22, 0, 0, 156, 68, 156, 75, 71, 68, 66}, //todo: these numbers don't seem to add up - {0, 0, 2021, 1, 0, 8, 0, 0, 0, 0, 0, 71, 70, 71, 71, 70, 70, 63}, + {0, 0, 3668, 20, 0, 96, 0, 0, 22, 0, 0, 156, 68, 156, 75, 71, 68, 66}, //todo: these numbers don't seem to add up + {0, 0, 2090, 1, 0, 8, 0, 0, 0, 0, 0, 71, 70, 71, 71, 70, 70, 63}, {0, 0, 24981, 0, 0, 0, 0, 0, 0, 0, 2, 334, 333, 0, 0, 0, 0, 0}, {0, 0, 2762, 0, 0, 0, 0, 0, 3, 0, 0, 50, 49, 50, 49, 49, 49, 49}, {0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1}, @@ -176,7 +176,7 @@ public void testForDetailsTenuring() { 13, 11, 8, - 8, + 9, 8, 9, 9, @@ -199,8 +199,8 @@ public void testForDetailsTenuring() { { 0, 0, 47, 1, 0, 22, 17, 0, 19, 0, 2, 210, 172, 210, 200, 178, 172, 172}, // todo: audit expected 210 - 22 = 188, not 172 { 0, 0, 3369, 6, 0, 6, 0, 0, 0, 0, 3, 17, 11, 17, 17, 15, 11, 11}, { 0, 0, 5305, 0, 0, 0, 0, 0, 0, 0, 0, 918, 917, 917, 917, 915, 917, 917}, - { 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 1402, 1402, 1402, 1375, 1402, 1402}, - { 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 15, 15, 2, 15, 15}, + { 0, 0, 1468, 0, 0, 1, 0, 0, 0, 0, 0, 1402, 1402, 1402, 1402, 1397, 1402, 1402}, + { 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 15, 15, 4, 15, 15}, { 0, 0, 46228, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1, 1, 1, 1, 1, 1}, { 0, 0, 47000, 0, 0, 0, 0, 0, 0, 0, 15, 2, 2, 2, 2, 2, 2, 2}, { 0, 0, 48222, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1, 1, 1, 1, 1, 1}, diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java index ad33b936..25210036 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/ICMSParNewParserTest.java @@ -2,12 +2,17 @@ // Licensed under the MIT License. package com.microsoft.gctoolkit.parser.unittests; +import com.microsoft.gctoolkit.parser.GCLogTrace; +import com.microsoft.gctoolkit.parser.GCParseRule; import com.microsoft.gctoolkit.parser.diary.TestLogFile; +import com.microsoft.gctoolkit.time.DateTimeStamp; import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.file.Path; +import static com.microsoft.gctoolkit.parser.CMSPatterns.*; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; public class ICMSParNewParserTest extends ParserTest { @@ -51,13 +56,13 @@ public void testForSimpleLogs() { }; private static final int[][] detailsCounts = { - // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + /* 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 */ { 0, 0, 12359, 41, 0, 41, 0, 0, 0, 0, 0, 179, 166, 179, 177, 166, 166, 138}, { 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 13, 13, 13, 13, 13, 13, 13}, { 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 11, 11, 11, 11, 11}, { 0, 0, 1374, 0, 0, 0, 1, 0, 0, 0, 0, 3, 2, 3, 2, 2, 2, 2}, - { 0, 0, 143543, 75, 0, 306, 0, 0, 7, 0, 1, 10272, 10015, 10271, 10039, 10018, 10015, 9969}, - { 0, 0, 72395, 43, 0, 173, 0, 0, 4, 0, 1, 5137, 4985, 5136, 4994, 4986, 4985, 4966} + { 0, 0, 143543, 75, 0, 306, 0, 0, 7, 0, 1, 10272, 10015, 10271, 10039, 10021, 10015, 9969}, + { 0, 0, 72395, 43, 0, 173, 0, 0, 4, 0, 1, 5137, 4985, 5136, 4994, 4988, 4985, 4966} }; @@ -97,7 +102,7 @@ public void testForDetailsTenuring() { { 0, 0, 67, 0, 0, 1, 0, 0, 1, 0, 0, 25, 24, 25, 24, 24, 24, 24}, { 0, 0, 6240, 0, 0, 0, 0, 0, 0, 0, 1, 2087, 2087, 2087, 2087, 2087, 2086, 2086}, { 0, 0, 417811, 83, 0, 84, 0, 0, 76, 0, 0, 7840, 7809, 7824, 7809, 7783, 7757, 7756}, - { 0, 0, 32439, 0, 0, 0, 0, 0, 0, 0, 0, 103, 103, 103, 103, 102, 102, 102}, + { 0, 0, 32439, 0, 0, 0, 0, 0, 0, 0, 0, 103, 103, 103, 103, 103, 102, 102}, { 0, 0, 18507, 0, 0, 0, 1, 0, 0, 0, 0, 6, 5, 6, 5, 5, 5, 5}, }; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/PreunifiedG1GCParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/PreunifiedG1GCParserTest.java index ef6b674d..5ef99de1 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/PreunifiedG1GCParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/PreunifiedG1GCParserTest.java @@ -94,7 +94,8 @@ public void testForDetailsTenuringLogs() { "180/neo4j-gc.log.20140625", "170_45/rolling/ahjapp02a_gclog/ahj_prod_gc.log.0", //this file has been corrupted and hence missed an entire record "170_45/g1_details_tenuring_cause.log", - "180/seeker.log" + "180/seeker.log", + "180/seeker-notimestamp.log" }; private static final int[] detailsTenuringNumberOfDifferentCollectors = { @@ -104,18 +105,20 @@ public void testForDetailsTenuringLogs() { 9, 9, 8, + 8, 8 }; private static final int[][] detailsTenuringCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - {794, 413, 157, 0, 0, 157, 0, 157, 157, 157, 0, 0, 157}, - {2194, 1308, 553, 0, 39, 553, 0, 523, 521, 523, 0, 0, 553}, - {16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {1432, 9, 4, 0, 3, 4, 0, 4, 4, 4, 0, 0, 4}, - {3019, 7, 3, 0, 4, 3, 0, 3, 3, 3, 0, 0, 3}, - {544, 100, 52, 0, 0, 52, 0, 52, 36, 52, 0, 0, 52}, - {1581, 708, 246, 0, 0, 246, 0, 246, 246, 246, 0, 0, 246} + { 794, 413, 157, 0, 0, 157, 0, 157, 157, 157, 0, 0, 157}, + { 2194, 1308, 553, 0, 39, 553, 0, 523, 521, 523, 0, 0, 553}, + { 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + { 1432, 9, 4, 0, 3, 4, 0, 4, 4, 4, 0, 0, 4}, + { 3019, 7, 3, 0, 4, 3, 0, 3, 3, 3, 0, 0, 3}, + { 544, 100, 52, 0, 0, 52, 0, 52, 36, 52, 0, 0, 52}, + { 1581, 708, 246, 0, 0, 246, 0, 246, 246, 246, 0, 0, 246}, + { 1581, 708, 246, 0, 0, 246, 0, 246, 246, 246, 0, 0, 246} }; @@ -136,6 +139,9 @@ public void testForDetailsReferenceLogs() { private static final String[] detailsReference = { "post17040/170_51_mastermind.log", + //todo: there is a bug in the pre 1.7.0_45 G1 Collector that causes a Full GC and the subsequent concurrent + // cycles (which should not be happening) to be missed. This fix is twisty to I've decided to leave things + // as is because no one should be using G1 in JDK 1.7. Happy to revisit if it becomes an issue. "post17040/gc_with_overflow.log" }; From 66e3466a024d955fbd27e17f981400a0371d8388 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 11 Apr 2024 13:29:28 -0700 Subject: [PATCH 21/34] refactor: bump version of gctoolkit testdata to support new tests --- gclogs/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gclogs/pom.xml b/gclogs/pom.xml index 095d300f..68b4a97d 100644 --- a/gclogs/pom.xml +++ b/gclogs/pom.xml @@ -17,7 +17,7 @@ pom - 1.0.8 + 1.0.9 From fcb4ebde71f7a7502d8c7789244c8fbf9ec4d2a6 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 11 Apr 2024 13:32:53 -0700 Subject: [PATCH 22/34] refactor: small code cleanups --- .../parser/GenerationalHeapParser.java | 3 --- .../parser/jvm/PreUnifiedDiarizer.java | 27 +++++++++---------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java index a24dae6f..c6edaefd 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java @@ -2077,9 +2077,6 @@ private void endOfConcurrentPhase(GCLogTrace trace, DateTimeStamp timeStamp, int double cpuTime = trace.getDoubleGroup(7 + offset); double wallTime = trace.getDoubleGroup(8 + offset); double duration = timeStamp.toSeconds() - startOfConcurrentPhase.toSeconds(); - // The wallTime measure is a very good estimate of duration. - if ( duration == Double.NaN) - duration = wallTime; switch (phase) { case "mark": diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java index 5a06bca8..1bf6beb6 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java @@ -543,21 +543,18 @@ private void version(String line) { if ((trace = ParallelPatterns.PERM_SPACE_RECORD.parse(line)) != null) { String value = trace.getGroup(1).trim(); - if ("CMS Perm".equals(value)) { - diary.setTrue(SupportedFlags.JDK70); - diary.setFalse(SupportedFlags.JDK80); - } else if ("PS Perm".equals(value)) { - diary.setTrue(SupportedFlags.JDK70); - diary.setFalse(SupportedFlags.JDK80); - } else if ("Perm".equals(value)) { - diary.setTrue(SupportedFlags.JDK70); - diary.setFalse(SupportedFlags.JDK80); - } else if ("PSPermGen".equals(value)) { - diary.setTrue(SupportedFlags.JDK70); - diary.setFalse(SupportedFlags.JDK80); - } else if ("Metaspace".equals(value)) { - diary.setTrue(SupportedFlags.JDK80); - diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); + switch (value) { + case "CMS Perm": + case "PS Perm": + case "Perm": + case "PSPermGen": + diary.setTrue(SupportedFlags.JDK70); + diary.setFalse(SupportedFlags.JDK80); + break; + case "Metaspace": + diary.setTrue(SupportedFlags.JDK80); + diary.setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40); + break; } } else if (META_SPACE_RECORD.parse(line) != null) { diary.setTrue(SupportedFlags.JDK80); From a90ec2519b0fb6cd785b230d051feef841e714ee Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Tue, 16 Apr 2024 09:14:47 -0700 Subject: [PATCH 23/34] Update parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java Co-authored-by: Martijn Verburg --- .../java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java index 63512616..bf795584 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java @@ -96,7 +96,7 @@ public DateTimeStamp getDateTimeStamp() { */ public DateTimeStamp getDateTimeStamp(int nth) { Matcher matcher; - if ( nth > 1) { + if (nth > 1) { matcher = DATE_TIME_STAMP_RULE.matcher(trace.group(0)); for (int i = 0; i < nth; i++) if (!matcher.find()) From 3e4127355ee9c2d702f05b5892b44efaf3d4336d Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 18 Apr 2024 11:41:30 -0700 Subject: [PATCH 24/34] tidy: cleanup from review and added more tests --- .../gctoolkit/parser/AbstractLogTrace.java | 16 +++++++--------- .../microsoft/gctoolkit/parser/CMSPatterns.java | 1 + .../gctoolkit/parser/CMSTenuredPoolParser.java | 2 +- .../microsoft/gctoolkit/parser/G1GCPatterns.java | 1 - .../microsoft/gctoolkit/parser/GCLogParser.java | 10 ++-------- .../gctoolkit/parser/GenerationalHeapParser.java | 1 - .../gctoolkit/parser/UnifiedG1GCParser.java | 1 + .../ConcurrentMarkSweepPhaseParserRulesTest.java | 10 +++------- .../gctoolkit/parser/G1GCParserRulesTest.java | 1 - 9 files changed, 15 insertions(+), 28 deletions(-) diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java index 63512616..149d695a 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java @@ -96,7 +96,7 @@ public DateTimeStamp getDateTimeStamp() { */ public DateTimeStamp getDateTimeStamp(int nth) { Matcher matcher; - if ( nth > 1) { + if (nth > 0) { matcher = DATE_TIME_STAMP_RULE.matcher(trace.group(0)); for (int i = 0; i < nth; i++) if (!matcher.find()) @@ -104,14 +104,12 @@ public DateTimeStamp getDateTimeStamp(int nth) { } else matcher = trace; - //if (matcher.find()) { - String timeStamp = ( matcher.group(3) == null) ? matcher.group(4) : matcher.group(3); - String dateStamp = ( matcher.group(2) == null) ? matcher.group(5) : matcher.group(2); - if (timeStamp != null) { - return new DateTimeStamp(dateStamp, convertToDouble(timeStamp)); - } else if ( dateStamp != null) - return new DateTimeStamp(dateStamp); - //} + String timeStamp = ( matcher.group(3) == null) ? matcher.group(4) : matcher.group(3); + String dateStamp = ( matcher.group(2) == null) ? matcher.group(5) : matcher.group(2); + if (timeStamp != null) { + return new DateTimeStamp(dateStamp, convertToDouble(timeStamp)); + } else if ( dateStamp != null) + return new DateTimeStamp(dateStamp); return new DateTimeStamp(MISSING_TIMESTAMP_SENTINEL); } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java index 33cf032c..927f8ab6 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSPatterns.java @@ -259,6 +259,7 @@ public interface CMSPatterns extends SharedPatterns { // 2015-02-04T17:36:07.103-0500: 199626.435: [GC 199626.435: [ParNew: 812672K->812672K(914240K), 0.0000400 secs]199626.435: [CMS (concurrent mode failure): 1071394K->1081343K(1081344K), 6.8504740 secs] 1884066K->1092775K(1995584K), [CMS Perm : 99417K->99411K(524288K)], 6.8510440 secs] [Times: user=6.63 sys=0.02, real=6.85 secs] // 199626.435: [GC 199626.435: [ParNew: 812672K->812672K(914240K), 0.0000400 secs]199626.435: [CMS (concurrent mode failure): 1071394K->1081343K(1081344K), 6.8504740 secs] 1884066K->1092775K(1995584K), [CMS Perm : 99417K->99411K(524288K)], 6.8510440 secs] [Times: user=6.63 sys=0.02, real=6.85 secs] + // 199626.435: [GC 199626.435: [ParNew: 812672K->812672K(914240K), 0.0000400 secs]199626.435: [CMS (concurrent mode failure): 1071394K->1081343K(1081344K), 6.8504740 secs] 1884066K->1092775K(1995584K), [Metaspace: 99417K->99411K(524288K)], 6.8510440 secs] [Times: user=6.63 sys=0.02, real=6.85 secs] GCParseRule PARNEW_CONCURRENT_MODE_FAILURE_PERM = new GCParseRule("PARNEW_CONCURRENT_MODE_FAILURE_PERM", GC_PREFIX + PARNEW_BLOCK + TIMESTAMP + "\\[CMS \\(concurrent mode failure\\): " + BEFORE_AFTER_CONFIGURED_PAUSE + "\\] " + BEFORE_AFTER_CONFIGURED + ", " + PERM_RECORD + ", " + PAUSE_TIME + "\\]"); GCParseRule PARNEW_CONCURRENT_MODE_FAILURE_META = new GCParseRule("PARNEW_CONCURRENT_MODE_FAILURE_META", GC_PREFIX + PARNEW_PROMOTION_FAILED_BLOCK + TIMESTAMP + "\\[CMS: " + BEFORE_AFTER_CONFIGURED_PAUSE + "\\] " + BEFORE_AFTER_CONFIGURED + ", " + META_RECORD + ", " + PAUSE_TIME + "\\]"); //4.327: [FUll GC 4.328: [ParNew: 196768K->180907K(471872K), 0.1321291 secs]4.460: [CMS (concurrent mode failure): 473195K->376198K(1048576K), 5.1817732 secs] 668966K->376198K(1520448K), [CMS Perm : 13108K->27169K(13172K)], 5.3146647 secs] diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java index b5453cf0..a5f6dd59 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/CMSTenuredPoolParser.java @@ -22,6 +22,7 @@ import java.util.Set; import java.util.logging.Logger; +@Deprecated public class CMSTenuredPoolParser extends PreUnifiedGCLogParser implements SimplePatterns, ICMSPatterns { private static final Logger LOG = Logger.getLogger(CMSTenuredPoolParser.class.getName()); @@ -90,7 +91,6 @@ private void endOfConcurrentPhase(GCLogTrace trace) { } private void endConcurrentPrecleanWithReferenceProcessing(GCLogTrace trace) { - trace.notYetImplemented(); try { publish(new ConcurrentPreClean(startOfPhase, trace.getDoubleGroup(14) - startOfPhase.getTimeStamp(), trace.getDoubleGroup(16), trace.getDoubleGroup(17))); } catch (Throwable t) { diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCPatterns.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCPatterns.java index cbc111c8..a981ff59 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCPatterns.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCPatterns.java @@ -231,7 +231,6 @@ public interface G1GCPatterns extends G1GCTokens { //549.251: [GC concurrent-mark-end, 0.0055240 secs] GCParseRule G1_CONCURRENT_END = new GCParseRule("G1_CONCURRENT_END", "^" + GC_PREFIX + "concurrent-(.+)-end, " + PAUSE_TIME); GCParseRule G1_CORRUPTED_CONCURRENT_END = new GCParseRule("G1_CORRUPTED_CONCURRENT_END", "^\\[GC concurrent-(.+)-end, " + PAUSE_TIME); - //GCParseRule G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END = new GCParseRule("G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END", "^" + DATE_TIMESTAMP + DATE_STAMP + "\\[GC concurrent-(.+)-end, " + PAUSE_TIME); GCParseRule G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END = new GCParseRule("G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END", "^" + DATE_TIMESTAMP + GC_PREFIX + "concurrent-(.+)-end, " + PAUSE_TIME); GCParseRule G1_FLOATING_CONCURRENT_PHASE_START = new GCParseRule("G1_FLOATING_CONCURRENT_PHASE_START", "^\\[GC concurrent(.+)\\]$"); GCParseRule G1_FULL_INTERRUPTS_CONCURRENT_CYCLE = new GCParseRule("G1_FULL_INTERRUPTS_CONCURRENT_CYCLE", FULL_GC_PREFIX + GC_PREFIX + " concurrent-(.+)-end, " + PAUSE_TIME); diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java index ac07ba62..de9855ca 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java @@ -162,17 +162,11 @@ MemoryPoolSummary extractPermOrMetaspaceRecord(String line) { GCLogTrace trace; MemoryPoolSummary metaDataPool = null; if ((trace = PERM_SPACE_RECORD.parse(line)) != null) { - String type = trace.getGroup(1).trim(); - switch (type) { + String recordType = trace.getGroup(1).trim(); + switch (recordType) { case "CMS Perm": - metaDataPool = extractPermGenRecord(trace); - break; case "PS Perm": - metaDataPool = extractPermGenRecord(trace); - break; case "PSPermGen": - metaDataPool = extractPermGenRecord(trace); - break; case "Perm": metaDataPool = extractPermGenRecord(trace); break; diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java index c6edaefd..3b323036 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java @@ -1402,7 +1402,6 @@ public void psFullGCV2Meta(GCLogTrace trace, String line) { } public void parNewConcurrentModeFailurePerm(GCLogTrace trace, String line) { - double totalPause = trace.getPauseTime(); double concurrentModeFailurePause = trace.getDoubleGroup(26); double parNewPause = trace.getDoubleGroup(18); double startTimeGap = trace.getDoubleGroup(13) - getClock().getTimeStamp(); diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java index 55deebb6..adfdcc83 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java @@ -373,6 +373,7 @@ private void g1Collection(GCLogTrace trace, String line) { break; } } + // The location for the gc cause is back 2 from where it typically is in other records. forwardReference.setGcType(gcType); forwardReference.setGCCause(trace.gcCause(-2)); forwardReference.setStartTime(getClock()); diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepPhaseParserRulesTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepPhaseParserRulesTest.java index ef9b6c9e..1bcf7b02 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepPhaseParserRulesTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/ConcurrentMarkSweepPhaseParserRulesTest.java @@ -39,7 +39,7 @@ public void testCMSParseRules() { //@Test public void testDebugCMSParseRules() { - int index = 7; + int index = 8; for (String line : lines[index]) evaluate(rules[index], line); } @@ -68,13 +68,10 @@ private void evaluate(GCParseRule rule, String string) { PRECLEAN_REFERENCE, CORRUPTED_PARNEW_BODY, PARNEW_DETAILS_WITH_CONCURRENT_MODE_FAILURE, - //new GCParseRule("PARNEW_DETAILS_WITH_CONCURRENT_MODE_FAILURE", "^: " + BEFORE_AFTER_CONFIGURED_PAUSE + "\\]" + CMS_TENURED_BLOCK), // + " " + BEFORE_AFTER_CONFIGURED + ", " + PERM_RECORD + ", " + PAUSE_TIME + "\\]"), iCMS_PARNEW_PROMOTION_FAILURE_RECORD, //todo: iCMS test??? iCMS_FULL, iCMS_PARNEW_PROMOTION_FAILURE, - //iCMS_PARNEW_PROMOTION_FAILURE_RECORD //todo: matches the rule above... PARNEW_CONCURRENT_MODE_FAILURE_PERM, - PARNEW_CONCURRENT_MODE_FAILURE_META //todo: no matching data }; private String[][] lines = { @@ -106,9 +103,8 @@ private void evaluate(GCParseRule rule, String string) { }, { // 7 "2015-02-04T17:36:07.103-0500: 199626.435: [GC 199626.435: [ParNew: 812672K->812672K(914240K), 0.0000400 secs]199626.435: [CMS (concurrent mode failure): 1071394K->1081343K(1081344K), 6.8504740 secs] 1884066K->1092775K(1995584K), [CMS Perm : 99417K->99411K(524288K)], 6.8510440 secs] [Times: user=6.63 sys=0.02, real=6.85 secs]", - "199626.435: [GC 199626.435: [ParNew: 812672K->812672K(914240K), 0.0000400 secs]199626.435: [CMS (concurrent mode failure): 1071394K->1081343K(1081344K), 6.8504740 secs] 1884066K->1092775K(1995584K), [CMS Perm : 99417K->99411K(524288K)], 6.8510440 secs] [Times: user=6.63 sys=0.02, real=6.85 secs]" - }, - { // 8 + "199626.435: [GC 199626.435: [ParNew: 812672K->812672K(914240K), 0.0000400 secs]199626.435: [CMS (concurrent mode failure): 1071394K->1081343K(1081344K), 6.8504740 secs] 1884066K->1092775K(1995584K), [CMS Perm : 99417K->99411K(524288K)], 6.8510440 secs] [Times: user=6.63 sys=0.02, real=6.85 secs]", + "199626.435: [GC 199626.435: [ParNew: 812672K->812672K(914240K), 0.0000400 secs]199626.435: [CMS (concurrent mode failure): 1071394K->1081343K(1081344K), 6.8504740 secs] 1884066K->1092775K(1995584K), [Metaspace: 99417K->99411K(524288K)], 6.8510440 secs] [Times: user=6.63 sys=0.02, real=6.85 secs]" } }; } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCParserRulesTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCParserRulesTest.java index 644897a4..7acc2c8f 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCParserRulesTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCParserRulesTest.java @@ -124,7 +124,6 @@ private void evaluate(GCParseRule rule, String string, boolean dump) { { // 9 "100081.540: [Full GC100081.540: [GC concurrent-root-region-scan-start]", "2018-01-29T17:34:24.984+0000: 5115.588: [Full GC (Metadata GC Threshold) 2018-01-29T17:34:24.984+0000: 5115.588: [GC concurrent-root-region-scan-start]", -// "2018-02-08T20:13:02.246+0000: 878632.850: 2018-02-08T20:13:02.246+0000: [Full GC (Metadata GC Threshold) 878632.850: [GC concurrent-root-region-scan-start]" }, { // 10 "2015-12-21T10:26:34.913-0500: 35835.169: [GC remark 35835.169: [Finalize Marking, 0.8985905 secs] 35836.068: [GC ref-proc35836.068: [SoftReference, 74 refs, 0.0002156 secs]35836.068: [WeakReference, 243 refs, 0.0001747 secs]35836.068: [FinalReference, 4154 refs, 0.0012360 secs]35836.070: [PhantomReference, 0 refs, 0 refs, 0.0002882 secs]35836.070: [JNI Weak Reference, 0.0000227 secs], 0.0026411 secs] 35836.071: [Unloading, 0.0048615 secs], 0.9142368 secs]", From 19e655a9f7ea2f263ba3eeea5e36952e9702d71d Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 19 Dec 2024 23:33:10 +0100 Subject: [PATCH 25/34] refactor: adjust unified generational GC log parser to produce SystemGC events --- .../parser/UnifiedGenerationalParser.java | 15 ++++++++-- .../UnifiedGenerationalParserTest.java | 28 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java index 02b0e43a..27d76881 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java @@ -4,6 +4,7 @@ import com.microsoft.gctoolkit.aggregator.EventSource; import com.microsoft.gctoolkit.event.CPUSummary; +import com.microsoft.gctoolkit.event.GCCause; import com.microsoft.gctoolkit.event.GarbageCollectionTypes; import com.microsoft.gctoolkit.event.generational.AbortablePreClean; import com.microsoft.gctoolkit.event.generational.CMSConcurrentEvent; @@ -21,6 +22,7 @@ import com.microsoft.gctoolkit.event.generational.PSFullGC; import com.microsoft.gctoolkit.event.generational.PSYoungGen; import com.microsoft.gctoolkit.event.generational.ParNew; +import com.microsoft.gctoolkit.event.generational.SystemGC; import com.microsoft.gctoolkit.event.generational.YoungGC; import com.microsoft.gctoolkit.event.jvm.JVMTermination; import com.microsoft.gctoolkit.jvm.Diary; @@ -481,12 +483,21 @@ private FullGC fillOutFullGC(FullGC event, GenerationalForwardReference values) } private FullGC buildFullGC(GenerationalForwardReference forwardReference) { + FullGC gc; switch (forwardReference.getGarbageCollectionType()) { case PSFull: - return fillOutFullGC(new PSFullGC(forwardReference.getStartTime(), forwardReference.getGCCause(), forwardReference.getDuration()), forwardReference); + if ( forwardReference.getGCCause().equals(GCCause.JAVA_LANG_SYSTEM)) + gc = new SystemGC(forwardReference.getStartTime(), forwardReference.getGCCause(), forwardReference.getDuration()); + else + gc = new PSFullGC(forwardReference.getStartTime(), forwardReference.getGCCause(), forwardReference.getDuration()); + return fillOutFullGC(gc, forwardReference); case FullGC: case Full: - return fillOutFullGC(new FullGC(forwardReference.getStartTime(), forwardReference.getGCCause(), forwardReference.getDuration()), forwardReference); + if ( forwardReference.getGCCause().equals(GCCause.JAVA_LANG_SYSTEM)) + gc = new SystemGC(forwardReference.getStartTime(), forwardReference.getGCCause(), forwardReference.getDuration()); + else + gc = new FullGC(forwardReference.getStartTime(), forwardReference.getGCCause(), forwardReference.getDuration()); + return fillOutFullGC(gc, forwardReference); default: LOGGER.warning(forwardReference.getGarbageCollectionType() + " is unrecognized"); } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java index 6eb85d48..cf5f7b1a 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java @@ -65,4 +65,32 @@ public void testCMSLogsMissingTimeStamps() { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 { 0, 0, 6167, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2}, }; + + //Serial + @Test + public void testSerialLogs() { + int i = 0; + for (String name : serial) { + try { + Path path = new TestLogFile("serial/" + name).getFile().toPath(); + TestResults testResults = testGenerationalSingleLogFile(path); + analyzeResults(name, testResults, serialNumberOfDifferentCollectors[i], serialCounts[i++]); + } catch (IOException ioe) { + fail(ioe.getMessage()); + } + } + } + + private static final String[] serial = { + "factorization-serialgc-tip.log" + }; + + private static final int[] serialNumberOfDifferentCollectors = { + 2, + }; + + private static final int[][] serialCounts = { + // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 + { 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0}, + }; } From 0493c3030fb900a4c3d4699fa4c24645aff374c8 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Sun, 22 Dec 2024 10:30:12 +0100 Subject: [PATCH 26/34] refactor: update source data to include new serial GC log file --- gclogs/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gclogs/pom.xml b/gclogs/pom.xml index a41c6723..d395dcb9 100644 --- a/gclogs/pom.xml +++ b/gclogs/pom.xml @@ -17,7 +17,7 @@ pom - 1.0.9 + 1.0.10 From 309cc21a8344263408ce9de567cf8992c93c6849 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Wed, 19 Mar 2025 17:58:29 -0700 Subject: [PATCH 27/34] refactor: prep for merge with upstream/main --- .../parser/unittests/UnifiedGenerationalParserTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java index cf5f7b1a..ac81d721 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/unittests/UnifiedGenerationalParserTest.java @@ -82,15 +82,17 @@ public void testSerialLogs() { } private static final String[] serial = { - "factorization-serialgc-tip.log" + "factorization-serialgc-tip.log", + "factorization-serialgc-jdk21.log" }; private static final int[] serialNumberOfDifferentCollectors = { - 2, + 2, 2 }; private static final int[][] serialCounts = { // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 - { 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0}, + { 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0}, // 17 + { 0, 812, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0}, // 835 }; } From b23baa222cf0d8c43f059b19f1ad62e02909e472 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Thu, 10 Apr 2025 17:47:00 -0700 Subject: [PATCH 28/34] refactor: add survivor record to the G1Young events --- .../gctoolkit/event/g1gc/G1Young.java | 10 +++ .../gctoolkit/parser/ForwardReference.java | 15 ++++ .../parser/G1GCForwardReference.java | 2 + .../gctoolkit/parser/UnifiedG1GCParser.java | 28 +++++- .../parser/UnifiedGenerationalParser.java | 28 +++++- .../parser/G1GCUnifiedParserRulesTest.java | 12 ++- .../GenerationalHeapParserRulesTest.java | 12 ++- .../parser/UnifiedG1GCParserFragmentTest.java | 89 +++++++++++++++++++ 8 files changed, 185 insertions(+), 11 deletions(-) diff --git a/api/src/main/java/com/microsoft/gctoolkit/event/g1gc/G1Young.java b/api/src/main/java/com/microsoft/gctoolkit/event/g1gc/G1Young.java index d47f2a46..a44d9e18 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/event/g1gc/G1Young.java +++ b/api/src/main/java/com/microsoft/gctoolkit/event/g1gc/G1Young.java @@ -6,6 +6,7 @@ import com.microsoft.gctoolkit.event.GarbageCollectionTypes; import com.microsoft.gctoolkit.event.StatisticalSummary; import com.microsoft.gctoolkit.event.UnifiedStatisticalSummary; +import com.microsoft.gctoolkit.event.jvm.SurvivorRecord; import com.microsoft.gctoolkit.time.DateTimeStamp; import java.util.Iterator; @@ -36,6 +37,7 @@ public class G1Young extends G1RealPause { private StatisticalSummary workerTotal; private StatisticalSummary processedBuffersSummary; private boolean toSpaceExhausted = false; + private SurvivorRecord survivorRecord; private final Map parallelPhaseSummaries = new ConcurrentHashMap<>(); private final Map phaseDurations = new ConcurrentHashMap<>(); @@ -196,6 +198,14 @@ public void addWorkerActivity(String group, StatisticalSummary statisticalSummar workerTotal = statisticalSummary; } + public void add(SurvivorRecord record) { + this.survivorRecord = record; + } + + public SurvivorRecord getSurvivorRecord() { + return survivorRecord; + } + public StatisticalSummary getWorkerOther() { return this.workerOther; } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/ForwardReference.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/ForwardReference.java index 27077bb9..3969d219 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/ForwardReference.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/ForwardReference.java @@ -4,6 +4,7 @@ import com.microsoft.gctoolkit.event.CPUSummary; import com.microsoft.gctoolkit.event.GCCause; +import com.microsoft.gctoolkit.event.jvm.SurvivorRecord; import com.microsoft.gctoolkit.parser.jvm.Decorators; import com.microsoft.gctoolkit.time.DateTimeStamp; @@ -15,6 +16,7 @@ public class ForwardReference { private double duration = -1.0d; private GCCause gcCause = GCCause.UNKNOWN_GCCAUSE; private CPUSummary cpuSummary = null; + SurvivorRecord survivorRecord = null; ForwardReference(Decorators decorators, int id) { this.gcID = id; @@ -69,4 +71,17 @@ Decorators getDecorators() { return decorators; } + public void survivorRecord(SurvivorRecord survivorRecord) { + this.survivorRecord = survivorRecord; + } + + public void addAgeBreakout(int age, long volume) { + if (this.survivorRecord == null) + return; + this.survivorRecord.add(age, volume); + } + + public SurvivorRecord getSurvivorRecord() { + return this.survivorRecord; + } } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCForwardReference.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCForwardReference.java index a91b08fb..d7d60ffb 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCForwardReference.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCForwardReference.java @@ -782,6 +782,8 @@ private G1Young buildYoung(G1Young collection) { if (toSpaceExhausted) collection.toSpaceExhausted(); if (hasReferenceGCSummary()) collection.add(generateReferenceGCSummary()); + if (survivorRecord != null) + collection.add(survivorRecord); collection.addCPUSummary(getCPUSummary()); return collection; } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java index 9a75822e..1d12ec75 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java @@ -15,6 +15,7 @@ import com.microsoft.gctoolkit.event.g1gc.G1GCPauseEvent; import com.microsoft.gctoolkit.event.jvm.JVMEvent; import com.microsoft.gctoolkit.event.jvm.JVMTermination; +import com.microsoft.gctoolkit.event.jvm.SurvivorRecord; import com.microsoft.gctoolkit.jvm.Diary; import com.microsoft.gctoolkit.message.ChannelName; import com.microsoft.gctoolkit.message.JVMEventChannel; @@ -47,7 +48,7 @@ * - pause time if it is reported or can be calculated * todo: me */ -public class UnifiedG1GCParser extends UnifiedGCLogParser implements UnifiedG1GCPatterns { +public class UnifiedG1GCParser extends UnifiedGCLogParser implements UnifiedG1GCPatterns, TenuredPatterns { private static final Logger LOGGER = Logger.getLogger(UnifiedG1GCParser.class.getName()); @@ -154,6 +155,8 @@ public class UnifiedG1GCParser extends UnifiedGCLogParser implements UnifiedG1GC parseRules.put(REBUILD_FREELIST, this::noop); parseRules.put(NEW_CSET, this::noop); parseRules.put(RESIZE_TLAB, this::noop); + parseRules.put(TENURING_SUMMARY, this::tenuringSummary); + parseRules.put(TENURING_AGE_BREAKDOWN, this::tenuringAgeBreakout); } public UnifiedG1GCParser() { @@ -672,6 +675,25 @@ private void fullStringSymbolTable(GCLogTrace trace, String line) { // forwardReference.scrubStringSymbolTableDuration(trace.getMillisecondDurationInSeconds()); } + /** + * Capture logged tenuring summary data + * @param trace + * @param line + */ + private void tenuringSummary(GCLogTrace trace, String line) { + forwardReference.survivorRecord(new SurvivorRecord(getClock(), trace.getLongGroup(1), trace.getIntegerGroup(2), trace.getIntegerGroup(3))); + } + + /** + * Capture logged age table data + * @param trace + * @param line + */ + private void tenuringAgeBreakout(GCLogTrace trace, String line) { + notYetImplemented(trace,line); + forwardReference.addAgeBreakout(trace.getIntegerGroup(1), trace.getLongGroup(2)); + } + /** * records a concurrent phase of a concurrent cycle. After the event has been recorded, all other events * that occurred during the concurrent event will be recorded. @@ -726,8 +748,6 @@ private void jvmExit(GCLogTrace trace, String s) { } private boolean ignoreFrequentlySeenButUnwantedLines(String line) { - if (line.contains("Desired survivor size")) return true; - if (line.contains("Age table with threshold")) return true; if (line.contains("safepoint")) return true; if (line.contains(") Skipped phase ")) return true; if (line.contains(" Total Min: ")) return true; @@ -738,7 +758,7 @@ private boolean ignoreFrequentlySeenButUnwantedLines(String line) { if (line.contains(" StringTable Weak Min:")) return true; if (line.contains(" ResolvedMethodTable Weak Min:")) return true; if (line.contains(" JNI Weak Min:")) return true; - return line.contains(" - age "); + return false; } /** diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java index 27d76881..019c4493 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java @@ -25,6 +25,7 @@ import com.microsoft.gctoolkit.event.generational.SystemGC; import com.microsoft.gctoolkit.event.generational.YoungGC; import com.microsoft.gctoolkit.event.jvm.JVMTermination; +import com.microsoft.gctoolkit.event.jvm.SurvivorRecord; import com.microsoft.gctoolkit.jvm.Diary; import com.microsoft.gctoolkit.message.ChannelName; import com.microsoft.gctoolkit.message.JVMEventChannel; @@ -63,7 +64,7 @@ * - from, to, configured * - pause time if it is reported or can be calculated */ -public class UnifiedGenerationalParser extends UnifiedGCLogParser implements UnifiedGenerationalPatterns { +public class UnifiedGenerationalParser extends UnifiedGCLogParser implements UnifiedGenerationalPatterns, TenuredPatterns { private static final Logger LOGGER = Logger.getLogger(UnifiedGenerationalParser.class.getName()); @@ -100,6 +101,8 @@ public class UnifiedGenerationalParser extends UnifiedGCLogParser implements Uni parseRules.put(JVM_EXIT, this::jvmExit); parseRules.put(END_OF_FILE, this::jvmExit); parseRules.put(METASPACE_DETAILED, this::metaSpaceDetails); + parseRules.put(TENURING_SUMMARY, this::tenuringSummary); + parseRules.put(TENURING_AGE_BREAKDOWN, this::tenuringAgeBreakout); } @@ -229,6 +232,26 @@ private void youngDetails(GCLogTrace trace, String line) { pauseEvent.setHeap(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(2)); } + /** + * Capture logged tenuring summary data + * @param trace + * @param line + */ + private void tenuringSummary(GCLogTrace trace, String line) { + if ( pauseEvent != null) + pauseEvent.survivorRecord(new SurvivorRecord(getClock(), trace.getLongGroup(1), trace.getIntegerGroup(2), trace.getIntegerGroup(3))); + } + + /** + * Capture logged age table data + * @param trace + * @param line + */ + private void tenuringAgeBreakout(GCLogTrace trace, String line) { + if (pauseEvent != null) + pauseEvent.addAgeBreakout(trace.getIntegerGroup(1), trace.getLongGroup(2)); + } + /** * If the concurrentCyclePauseEvent has not been recorded, something has gone wrong and it's likely * that it doesn't have a consistent state. The default action is to lose it. @@ -565,8 +588,7 @@ private boolean ignoreFrequentlySeenButUnwantedLines(String line) { if (line.contains("exit")) if (line.contains("used")) return true; if (line.contains("workers")) return true; - if (line.contains("Heap address")) return true; - return line.contains("Desired") || line.contains("Age table") || line.contains("- age "); + return line.contains("Heap address"); } @Override diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCUnifiedParserRulesTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCUnifiedParserRulesTest.java index 0e081dde..b3b1be50 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCUnifiedParserRulesTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/G1GCUnifiedParserRulesTest.java @@ -12,7 +12,7 @@ import static com.microsoft.gctoolkit.parser.CommonTestHelper.captureTest; import static org.junit.jupiter.api.Assertions.*; -public class G1GCUnifiedParserRulesTest implements UnifiedG1GCPatterns { +public class G1GCUnifiedParserRulesTest implements UnifiedG1GCPatterns, TenuredPatterns { /** * The rules are; @@ -152,7 +152,9 @@ private void evaluate(GCParseRule rule, String string, boolean dump) { WEAK_PROCESSING, CLEANUP__FINALIZE_CONC_MARK, CONCURRENT_UNDO_CYCLE_START, - CONCURRENT_UNDO_CYCLE_END // 70 + CONCURRENT_UNDO_CYCLE_END , // 70 + TENURING_SUMMARY, + TENURING_AGE_BREAKDOWN }; /* @@ -446,6 +448,12 @@ private void evaluate(GCParseRule rule, String string, boolean dump) { }, { // 70 "[155.836s][info ][gc ] GC(2457) Concurrent Undo Cycle 49.351ms", + }, + { // 71 + "[10.749s][debug][gc,age ] GC(0) Desired survivor size 1572864 bytes, new threshold 15 (max threshold 15)\n" + }, + { // 72 + "[10.754s][trace][gc,age ] GC(0) - age 1: 2579584 bytes, 2579584 total" } // Remaining lines which may not need to be parsed... diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserRulesTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserRulesTest.java index 54edf0eb..ce3146da 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserRulesTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserRulesTest.java @@ -9,7 +9,7 @@ import static com.microsoft.gctoolkit.parser.CommonTestHelper.captureTest; import static org.junit.jupiter.api.Assertions.*; -public class GenerationalHeapParserRulesTest implements SimplePatterns, SerialPatterns, ParallelPatterns, CMSPatterns, ICMSPatterns { +public class GenerationalHeapParserRulesTest implements SimplePatterns, SerialPatterns, ParallelPatterns, CMSPatterns, ICMSPatterns, TenuredPatterns { private static final Logger LOGGER = Logger.getLogger(GenerationalHeapParserRulesTest.class.getName()); @@ -250,7 +250,9 @@ private void evaluate(GCParseRule rule, String string, boolean dump) { // CMF_LARGE_BLOCK, // //this rule must be evaluated before CONCURRENT_PHASE_END_BLOCK // ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE, - PRECLEAN_REFERENCE + PRECLEAN_REFERENCE, + TENURING_SUMMARY, + TENURING_AGE_BREAKDOWN }; private String[][] lines = { @@ -591,6 +593,12 @@ private void evaluate(GCParseRule rule, String string, boolean dump) { // }, { "2016-04-01T15:03:42.171-0700: 11025.637: [Preclean SoftReferences, 0.0000530 secs]2016-04-01T15:03:42.172-0700: 11025.637: [Preclean WeakReferences, 0.0006860 secs]2016-04-01T15:03:42.172-0700: 11025.638: [Preclean FinalReferences, 0.0005450 secs]2016-04-01T15:03:42.173-0700: 11025.639: [Preclean PhantomReferences, 0.0000230 secs]2016-04-01T15:03:42.197-0700: 11025.663: [CMS-concurrent-preclean: 0.025/0.026 secs] [Times: user=0.04 sys=0.01, real=0.03 secs]" + }, + { + "[10.749s][debug][gc,age ] GC(0) Desired survivor size 1572864 bytes, new threshold 15 (max threshold 15)\n" + }, + { + "[10.754s][trace][gc,age ] GC(0) - age 1: 2579584 bytes, 2579584 total" } }; } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java index b63ec483..db0b162f 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParserFragmentTest.java @@ -2,8 +2,12 @@ // Licensed under the MIT License. package com.microsoft.gctoolkit.parser; +import com.microsoft.gctoolkit.event.CPUSummary; +import com.microsoft.gctoolkit.event.GCCause; +import com.microsoft.gctoolkit.event.MemoryPoolSummary; import com.microsoft.gctoolkit.event.g1gc.G1Young; import com.microsoft.gctoolkit.event.jvm.JVMEvent; +import com.microsoft.gctoolkit.event.jvm.SurvivorRecord; import com.microsoft.gctoolkit.jvm.Diarizer; import com.microsoft.gctoolkit.parser.diary.TestLogFile; import com.microsoft.gctoolkit.parser.jvm.UnifiedDiarizer; @@ -12,6 +16,7 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.Iterator; import java.util.List; import static org.junit.jupiter.api.Assertions.fail; @@ -93,4 +98,88 @@ public void testNewDecoratorCombination() { fail(t); } } + + @Test + public void testSurvivorRecord() { + String[] lines = {"[0.016s][info][gc,heap] Heap region size: 1M", + "[0.018s][info][gc ] Using G1", + "[0.018s][info][gc,heap,coops] Heap address: 0x00000007fc000000, size: 64 MB, Compressed Oops mode: Zero based, Oop shift amount: 3\n" + + "[10.749s][info][gc,start ] GC(0) Pause Young (Normal) (G1 Evacuation Pause)", + "[10.749s][info][gc,task ] GC(0) Using 8 workers of 8 for evacuation", + "[10.749s][debug][gc,age ] GC(0) Desired survivor size 1572864 bytes, new threshold 15 (max threshold 15)", + "[10.753s][trace][gc,age ] GC(0) Age table with threshold 15 (max threshold 15)", + "[10.754s][trace][gc,age ] GC(0) - age 1: 2579584 bytes, 2579584 total", + "[10.754s][info ][gc,phases ] GC(0) Pre Evacuate Collection Set: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Prepare TLABs: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Choose Collection Set: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Humongous Register: 0.0ms", + "[10.754s][info ][gc,phases ] GC(0) Evacuate Collection Set: 4.1ms", + "[10.754s][debug][gc,phases ] GC(0) Ext Root Scanning (ms): Min: 0.2, Avg: 0.9, Max: 1.7, Diff: 1.6, Sum: 7.3, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Update RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Processed Buffers: Min: 0, Avg: 0.0, Max: 0, Diff: 0, Sum: 0, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Scanned Cards: Min: 0, Avg: 0.0, Max: 0, Diff: 0, Sum: 0, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Skipped Cards: Min: 0, Avg: 0.0, Max: 0, Diff: 0, Sum: 0, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Scan RS (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Scanned Cards: Min: 0, Avg: 0.0, Max: 0, Diff: 0, Sum: 0, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Claimed Cards: Min: 0, Avg: 0.0, Max: 0, Diff: 0, Sum: 0, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Skipped Cards: Min: 0, Avg: 0.0, Max: 0, Diff: 0, Sum: 0, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Code Root Scanning (ms): Min: 0.0, Avg: 0.2, Max: 0.4, Diff: 0.4, Sum: 1.8, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) AOT Root Scanning (ms): skipped", + "[10.754s][debug][gc,phases ] GC(0) Object Copy (ms): Min: 1.9, Avg: 2.8, Max: 3.5, Diff: 1.6, Sum: 22.7, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Termination (ms): Min: 0.0, Avg: 0.1, Max: 0.1, Diff: 0.1, Sum: 0.5, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) Termination Attempts: Min: 1, Avg: 71.4, Max: 95, Diff: 94, Sum: 571, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) GC Worker Other (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.1, Workers: 8", + "[10.754s][debug][gc,phases ] GC(0) GC Worker Total (ms): Min: 4.0, Avg: 4.1, Max: 4.1, Diff: 0.0, Sum: 32.5, Workers: 8", + "[10.754s][info ][gc,phases ] GC(0) Post Evacuate Collection Set: 0.3ms", + "[10.754s][debug][gc,phases ] GC(0) Code Roots Fixup: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Clear Card Table: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Reference Processing: 0.1ms", + "[10.754s][debug][gc,phases ] GC(0) Weak Processing: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Merge Per-Thread State: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Code Roots Purge: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Redirty Cards: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) DerivedPointerTable Update: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Free Collection Set: 0.1ms", + "[10.754s][debug][gc,phases ] GC(0) Humongous Reclaim: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Start New Collection Set: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Resize TLABs: 0.0ms", + "[10.754s][debug][gc,phases ] GC(0) Expand Heap After Collection: 0.0ms", + "[10.754s][info ][gc,phases ] GC(0) Other: 0.7ms", + "[10.754s][info ][gc,heap ] GC(0) Eden regions: 24->0(32)", + "[10.754s][info ][gc,heap ] GC(0) Survivor regions: 0->3(3)", + "[10.754s][info ][gc,heap ] GC(0) Old regions: 0->3", + "[10.754s][info ][gc,heap ] GC(0) Humongous regions: 0->0", + "[10.754s][info ][gc,metaspace ] GC(0) Metaspace: 15753K->15753K(1062912K)", + "[10.754s][info ][gc ] GC(0) Pause Young (Normal) (G1 Evacuation Pause) 23M->5M(64M) 5.662ms", + "[10.754s][info ][gc,cpu ] GC(0) User=0.03s Sys=0.01s Real=0.00s" + }; + List jvmEvents = feedParser(lines); + + try { + Assertions.assertEquals(1, jvmEvents.size()); + Assertions.assertEquals(G1Young.class, jvmEvents.get(0).getClass()); + G1Young cycle = (G1Young) jvmEvents.get(0); + SurvivorRecord survivorRecord = cycle.getSurvivorRecord(); + Assertions.assertEquals(1572864, survivorRecord.getDesiredOccupancyAfterCollection()); + Assertions.assertEquals(15, survivorRecord.getMaxTenuringThreshold()); + Assertions.assertEquals(15, survivorRecord.getCalculatedTenuringThreshold()); + Assertions.assertEquals(2579584, survivorRecord.getBytesAtAge(1)); + Assertions.assertEquals(0.0, cycle.phaseDurationFor("Pre Evacuate Collection")); + Assertions.assertEquals(0.0041, cycle.phaseDurationFor("Evacuate Collection"), 0.0001); + Assertions.assertEquals(0.0003, cycle.phaseDurationFor("Post Evacuate Collection Set"), 0.00001); + Assertions.assertEquals(0.0007, cycle.phaseDurationFor("Other"), 0.00001); + Assertions.assertEquals(0.005662, cycle.getDuration(), 0.0000001); + Assertions.assertEquals(GCCause.G1_EVACUATION_PAUSE, cycle.getGCCause()); + MemoryPoolSummary memoryPoolSummary = cycle.getHeap(); + Assertions.assertEquals(23*1024, memoryPoolSummary.getOccupancyBeforeCollection()); + Assertions.assertEquals(5*1024, memoryPoolSummary.getOccupancyAfterCollection()); + Assertions.assertEquals(64*1024, memoryPoolSummary.getSizeAfterCollection()); + CPUSummary cpuSummary = cycle.getCpuSummary(); + Assertions.assertEquals(0.03, cpuSummary.getUser()); + Assertions.assertEquals(0.01, cpuSummary.getKernel()); + Assertions.assertEquals(0.00, cpuSummary.getWallClock()); + } catch(Throwable t) { + fail(t); + } + } } From 882c86d5ce6de14fe8658ccfeeca1fa805ff3f6e Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Mon, 14 Apr 2025 15:55:38 -0700 Subject: [PATCH 29/34] refactor: adding in tenuring record --- .../event/generational/GenerationalGCPauseEvent.java | 10 ++++++++++ .../gctoolkit/event/generational/PSYoungGen.java | 3 +++ .../gctoolkit/parser/UnifiedGenerationalParser.java | 7 ++++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/com/microsoft/gctoolkit/event/generational/GenerationalGCPauseEvent.java b/api/src/main/java/com/microsoft/gctoolkit/event/generational/GenerationalGCPauseEvent.java index 46f6de1e..6569fb87 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/event/generational/GenerationalGCPauseEvent.java +++ b/api/src/main/java/com/microsoft/gctoolkit/event/generational/GenerationalGCPauseEvent.java @@ -7,6 +7,7 @@ import com.microsoft.gctoolkit.event.GarbageCollectionTypes; import com.microsoft.gctoolkit.event.MemoryPoolSummary; import com.microsoft.gctoolkit.event.ReferenceGCSummary; +import com.microsoft.gctoolkit.event.jvm.SurvivorRecord; import com.microsoft.gctoolkit.time.DateTimeStamp; public abstract class GenerationalGCPauseEvent extends GenerationalGCEvent { @@ -23,6 +24,7 @@ public abstract class GenerationalGCPauseEvent extends GenerationalGCEvent { private double stringTableProcessingTime; private double symbolAndStringTableProcessingTime; private BinaryTreeDictionary binaryTreeDictionary; + private SurvivorRecord survivorRecord; private CPUSummary cpuSummary; @@ -114,6 +116,14 @@ public void addClassUnloadingAndStringTableProcessingDurations(double classUnloa this.symbolAndStringTableProcessingTime = symbolAndStringTable; } + public void add(SurvivorRecord record) { + this.survivorRecord = record; + } + + public SurvivorRecord getSurvivorRecord() { + return survivorRecord; + } + public double getClassUnloadingProcessingTime() { return this.classUnloadingProcessingTime; } diff --git a/api/src/main/java/com/microsoft/gctoolkit/event/generational/PSYoungGen.java b/api/src/main/java/com/microsoft/gctoolkit/event/generational/PSYoungGen.java index f5e1ffba..2d81af7d 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/event/generational/PSYoungGen.java +++ b/api/src/main/java/com/microsoft/gctoolkit/event/generational/PSYoungGen.java @@ -4,6 +4,7 @@ import com.microsoft.gctoolkit.event.GCCause; import com.microsoft.gctoolkit.event.GarbageCollectionTypes; +import com.microsoft.gctoolkit.event.jvm.SurvivorRecord; import com.microsoft.gctoolkit.time.DateTimeStamp; public class PSYoungGen extends GenerationalGCPauseEvent { @@ -15,4 +16,6 @@ public PSYoungGen(DateTimeStamp timeStamp, GarbageCollectionTypes type, GCCause public PSYoungGen(DateTimeStamp timeStamp, GCCause cause, double duration) { super(timeStamp, GarbageCollectionTypes.PSYoungGen, cause, duration); } + + } diff --git a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java index 019c4493..5ddd788a 100644 --- a/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java +++ b/parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalParser.java @@ -472,13 +472,14 @@ private GenerationalGCPauseEvent buildYoungEvent(GenerationalForwardReference fo youngCollection = new PSYoungGen(forwardReference.getStartTime(), forwardReference.getGCCause(), forwardReference.getDuration()); break; default: - LOGGER.warning(forwardReference.getGarbageCollectionType() + " not recognized"); + throw new IllegalStateException(forwardReference.getGarbageCollectionType() + " not recognized"); } fillOutMemoryPoolData(youngCollection, forwardReference); fillOutMetaspaceData(youngCollection, forwardReference); youngCollection.add(forwardReference.getCPUSummary()); - // add in reference processing + youngCollection.add(forwardReference.getSurvivorRecord()); + // todo: add in reference processing return youngCollection; } @@ -542,7 +543,7 @@ private GenerationalGCPauseEvent buildPauseEvent(GenerationalForwardReference fo return buildInitialMark(forwardReference); case Remark: return buildRemark(forwardReference); - case PSFull: //todo: + case PSFull: case FullGC: case Full: return buildFullGC(forwardReference); From 96e4db1924549f75c651459a187ac67c34118e49 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Mon, 21 Apr 2025 12:25:49 -0700 Subject: [PATCH 30/34] refactor: update testdata version # --- gclogs/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gclogs/pom.xml b/gclogs/pom.xml index 5af3300e..448945fc 100644 --- a/gclogs/pom.xml +++ b/gclogs/pom.xml @@ -17,7 +17,7 @@ pom - 1.0.10 + 1.0.11 From 12e7e2b62c47eff83d7683ecd3e3e682ea246192 Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Mon, 21 Apr 2025 19:02:14 -0700 Subject: [PATCH 31/34] refactor: add in tests for Survivor records found in GCEvent --- .../gctoolkit/event/jvm/SurvivorRecord.java | 18 +-- .../parser/GenerationalHeapParserTest.java | 2 + .../UnifiedGenerationalEventsTest.java | 116 ++++++++++++++++-- 3 files changed, 114 insertions(+), 22 deletions(-) rename parser/src/test/java/com/microsoft/gctoolkit/parser/{patterns => }/UnifiedGenerationalEventsTest.java (75%) diff --git a/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java b/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java index 18a946ae..0be6f105 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java +++ b/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java @@ -17,7 +17,7 @@ public class SurvivorRecord extends JVMEvent { // JDK bug, we have now seen a max tenuring threshold of 32, even 64 // Fold anything older than 15 back into the 15th slot - private ArrayList bytesAtAge = null; + private long[] bytesAtAge; public SurvivorRecord(DateTimeStamp timeStamp, long desiredOccupancy, int calculatedTenuringThreshold, int maxTenuringThreshold) { super(timeStamp, 0.0d); @@ -56,7 +56,7 @@ public int getMaxTenuringThreshold() { public long getBytesAtAge(int age) { if (this.bytesAtAge == null) return 0L; - return this.bytesAtAge.get(age); + return this.bytesAtAge[age]; } /* @@ -67,20 +67,20 @@ public long getBytesAtAge(int age) { public void add(int age, long bytes) { if (bytesAtAge == null) { - bytesAtAge = new ArrayList<>(); - bytesAtAge.add(0L); //throw away the first slow. + bytesAtAge = new long[maxTenuringThreshold+1]; + bytesAtAge[0] = 0L; //throw away the first slot. } if (age <= maxTenuringThreshold) { - bytesAtAge.add(bytes); + bytesAtAge[age] = bytes; } else { - bytesAtAge.set(maxTenuringThreshold, bytesAtAge.get(maxTenuringThreshold) + bytes); + bytesAtAge[age] += bytes; } } - public Long[] getBytesAtEachAge() { + public long[] getBytesAtEachAge() { if (bytesAtAge == null) - return new Long[0]; - return bytesAtAge.toArray(new Long[0]); + return new long[0]; + return bytesAtAge; } } diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java index 4fbd777d..b7eee66b 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/GenerationalHeapParserTest.java @@ -11,8 +11,10 @@ import com.microsoft.gctoolkit.event.generational.ConcurrentSweep; import com.microsoft.gctoolkit.event.generational.FullGC; import com.microsoft.gctoolkit.event.generational.InitialMark; +import com.microsoft.gctoolkit.event.generational.PSYoungGen; import com.microsoft.gctoolkit.event.generational.ParNew; import com.microsoft.gctoolkit.event.jvm.JVMEvent; +import com.microsoft.gctoolkit.event.jvm.SurvivorRecord; import com.microsoft.gctoolkit.jvm.Diarizer; import com.microsoft.gctoolkit.parser.jvm.PreUnifiedDiarizer; import org.junit.jupiter.api.Assertions; diff --git a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/UnifiedGenerationalEventsTest.java b/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalEventsTest.java similarity index 75% rename from parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/UnifiedGenerationalEventsTest.java rename to parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalEventsTest.java index bb11bf06..9e4ff179 100644 --- a/parser/src/test/java/com/microsoft/gctoolkit/parser/patterns/UnifiedGenerationalEventsTest.java +++ b/parser/src/test/java/com/microsoft/gctoolkit/parser/UnifiedGenerationalEventsTest.java @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -package com.microsoft.gctoolkit.parser.patterns; +package com.microsoft.gctoolkit.parser; import com.microsoft.gctoolkit.event.MemoryPoolSummary; @@ -17,10 +17,8 @@ import com.microsoft.gctoolkit.event.generational.PSYoungGen; import com.microsoft.gctoolkit.event.generational.ParNew; import com.microsoft.gctoolkit.event.jvm.JVMEvent; +import com.microsoft.gctoolkit.event.jvm.SurvivorRecord; import com.microsoft.gctoolkit.jvm.Diarizer; -import com.microsoft.gctoolkit.parser.GCLogParser; -import com.microsoft.gctoolkit.parser.ParserTest; -import com.microsoft.gctoolkit.parser.UnifiedGenerationalParser; import com.microsoft.gctoolkit.parser.jvm.UnifiedDiarizer; import org.junit.jupiter.api.Test; @@ -332,15 +330,15 @@ public void testEnhancedMetaspaceRecord() { try { assertEquals(1, jvmEvents.size()); PSYoungGen collection = (PSYoungGen) jvmEvents.get(0); - assertEquals(53228,collection.getYoung().getOccupancyBeforeCollection()); - assertEquals(7148,collection.getYoung().getOccupancyAfterCollection()); - assertEquals(53248,collection.getYoung().getSizeAfterCollection()); - assertEquals(65568,collection.getTenured().getOccupancyBeforeCollection()); - assertEquals(111512,collection.getTenured().getOccupancyAfterCollection()); - assertEquals(121856,collection.getTenured().getSizeAfterCollection()); - assertEquals(116*1024,collection.getHeap().getOccupancyBeforeCollection()); - assertEquals(115*1024,collection.getHeap().getOccupancyAfterCollection()); - assertEquals(171*1024,collection.getHeap().getSizeAfterCollection()); + assertEquals(53228, collection.getYoung().getOccupancyBeforeCollection()); + assertEquals(7148, collection.getYoung().getOccupancyAfterCollection()); + assertEquals(53248, collection.getYoung().getSizeAfterCollection()); + assertEquals(65568, collection.getTenured().getOccupancyBeforeCollection()); + assertEquals(111512, collection.getTenured().getOccupancyAfterCollection()); + assertEquals(121856, collection.getTenured().getSizeAfterCollection()); + assertEquals(116 * 1024, collection.getHeap().getOccupancyBeforeCollection()); + assertEquals(115 * 1024, collection.getHeap().getOccupancyAfterCollection()); + assertEquals(171 * 1024, collection.getHeap().getSizeAfterCollection()); assertEquals(3646, collection.getPermOrMetaspace().getOccupancyBeforeCollection()); assertEquals(4864, collection.getPermOrMetaspace().getSizeBeforeCollection()); assertEquals(3646, collection.getPermOrMetaspace().getOccupancyAfterCollection()); @@ -358,4 +356,96 @@ public void testEnhancedMetaspaceRecord() { fail(cce.getMessage()); } } + + + @Test + public void parallelWithSurvivorRecordsTest() { + String[] lines = { + "[9.371s][info ][gc,start ] GC(18) Pause Young (Allocation Failure)", + "[9.371s][debug][gc,age ] GC(18) Desired survivor size 2097152 bytes, new threshold 1 (max threshold 15)", + "[9.371s][info ][gc,heap ] GC(18) PSYoungGen: 309244K(311296K)->454K(298496K) Eden: 308694K(308736K)->0K(297984K) From: 550K(2560K)->454K(512K)", + "[9.371s][info ][gc,heap ] GC(18) ParOldGen: 4018K(349696K)->4042K(349696K)", + "[9.371s][info ][gc,metaspace ] GC(18) Metaspace: 9573K(9856K)->9573K(9856K) NonClass: 8504K(8640K)->8504K(8640K) Class: 1069K(1216K)->1069K(1216K)", + "[9.371s][info ][gc ] GC(18) Pause Young (Allocation Failure) 305M->4M(633M) 0.242ms", + "[9.371s][info ][gc,cpu ] GC(18) User=0.01s Sys=0.00s Real=0.00s" + }; + + List jvmEvents = feedParser(lines); + assertEquals(1, jvmEvents.size()); + PSYoungGen collection = (PSYoungGen) jvmEvents.get(0); + assertEquals(9.371, collection.getDateTimeStamp().toSeconds()); + SurvivorRecord record = collection.getSurvivorRecord(); + assertNotNull(record); + assertEquals(15, record.getMaxTenuringThreshold()); + assertEquals(1, record.getCalculatedTenuringThreshold()); + assertEquals(2097152, record.getDesiredOccupancyAfterCollection()); + assertEquals(0, record.getBytesAtEachAge().length); + } + + @Test + public void serialWithSurvivorRecordsTest() { + + String[] lines = {"[8.726s][info ][gc,start ] GC(9) Pause Young (Allocation Failure)", + "[8.726s][debug][gc,age ] GC(9) Desired survivor size 8945664 bytes, new threshold 15 (max threshold 15)", + "[8.726s][trace][gc,age ] GC(9) Age table with threshold 15 (max threshold 15)", + "[8.726s][trace][gc,age ] GC(9) - age 1: 400368 bytes, 400368 total", + "[8.726s][trace][gc,age ] GC(9) - age 3: 304 bytes, 400672 total", + "[8.726s][trace][gc,age ] GC(9) - age 8: 32 bytes, 400704 total", + "[8.726s][trace][gc,age ] GC(9) - age 9: 554176 bytes, 954880 total", + "[8.726s][info ][gc,heap ] GC(9) DefNew: 140789K(157376K)->932K(157376K) Eden: 139857K(139904K)->0K(139904K) From: 932K(17472K)->932K(17472K)", + "[8.726s][info ][gc,heap ] GC(9) Tenured: 3357K(349568K)->3357K(349568K)", + "[8.726s][info ][gc,metaspace ] GC(9) Metaspace: 9628K(9856K)->9628K(9856K) NonClass: 8545K(8640K)->8545K(8640K) Class: 1082K(1216K)->1082K(1216K)", + "[8.727s][info ][gc ] GC(9) Pause Young (Allocation Failure) 140M->4M(495M) 0.445ms", + "[8.727s][info ][gc,cpu ] GC(9) User=0.00s Sys=0.00s Real=0.00s" + }; + + List jvmEvents = feedParser(lines); + assertEquals(1, jvmEvents.size()); + DefNew collection = (DefNew) jvmEvents.get(0); + assertEquals(8.726, collection.getDateTimeStamp().toSeconds()); + SurvivorRecord record = collection.getSurvivorRecord(); + assertNotNull(record); + assertEquals(15, record.getMaxTenuringThreshold()); + assertEquals(15, record.getCalculatedTenuringThreshold()); + assertEquals(8945664, record.getDesiredOccupancyAfterCollection()); + assertEquals(16, record.getBytesAtEachAge().length); + int[] bytesAtAge = {0, 400368, 0, 304, 0, 0, 0, 0, 32, 554176, 0, 0, 0, 0, 0, 0}; + for (int i = 1; i < bytesAtAge.length; i++) { + assertEquals(bytesAtAge[i], record.getBytesAtAge(i)); + } + assertThrows(IndexOutOfBoundsException.class, () -> record.getBytesAtAge(16)); + } + + @Test + public void cmsWithSurvivorRecordsTest() { + + String[] lines = {"[11.633s][info ][gc,start ] GC(12) Pause Young (Allocation Failure)", + "[11.633s][info ][gc,task ] GC(12) Using 9 workers of 9 for evacuation", + "[11.634s][debug][gc,age ] GC(12) Desired survivor size 8945664 bytes, new threshold 6 (max threshold 6)", + "[11.634s][trace][gc,age ] GC(12) Age table with threshold 6 (max threshold 6)", + "[11.634s][trace][gc,age ] GC(12) - age 1: 400208 bytes, 400208 total", + "[11.634s][trace][gc,age ] GC(12) - age 6: 304 bytes, 400512 total", + "[11.634s][info ][gc,heap ] GC(12) ParNew: 140026K->398K(157376K)", + "[11.634s][info ][gc,heap ] GC(12) CMS: 3662K->3662K(349568K)", + "[11.634s][info ][gc,metaspace ] GC(12) Metaspace: 15714K(16256K)->15714K(16256K) NonClass: 14027K(14336K)->14027K(14336K) Class: 1686K(1920K)->1686K(1920K)", + "[11.634s][info ][gc ] GC(12) Pause Young (Allocation Failure) 140M->3M(495M) 0.430ms", + "[11.634s][info ][gc,cpu ] GC(12) User=0.00s Sys=0.00s Real=0.00s", + }; + List jvmEvents = feedParser(lines); + assertEquals(1, jvmEvents.size()); + ParNew collection = (ParNew) jvmEvents.get(0); + + assertEquals(11.633, collection.getDateTimeStamp().toSeconds()); + SurvivorRecord record = collection.getSurvivorRecord(); + assertNotNull(record); + assertEquals(6, record.getMaxTenuringThreshold()); + assertEquals(6, record.getCalculatedTenuringThreshold()); + assertEquals(8945664, record.getDesiredOccupancyAfterCollection()); + assertEquals(7, record.getBytesAtEachAge().length); + int[] bytesAtAge = {0, 400208, 0, 0, 0, 0, 304}; + for (int i = 1; i < bytesAtAge.length; i++) { + assertEquals(bytesAtAge[i], record.getBytesAtAge(i)); + } + assertThrows(IndexOutOfBoundsException.class, () -> record.getBytesAtAge(7)); + } } From 1b36a1eaa7e619d92661be996d926a9149de076e Mon Sep 17 00:00:00 2001 From: Kirk Pepperdine Date: Tue, 22 Apr 2025 08:54:15 -0700 Subject: [PATCH 32/34] refactor: update test data --- gclogs/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gclogs/pom.xml b/gclogs/pom.xml index 448945fc..2328db25 100644 --- a/gclogs/pom.xml +++ b/gclogs/pom.xml @@ -17,7 +17,7 @@ pom - 1.0.11 + 1.0.12 From bd2f366412000bd24fa716a0890bf95c1fdc367e Mon Sep 17 00:00:00 2001 From: David Grieve Date: Tue, 22 Apr 2025 20:16:54 +0000 Subject: [PATCH 33/34] fix highly unlikely AIOBE --- .../gctoolkit/event/generational/PSYoungGen.java | 3 --- .../gctoolkit/event/jvm/SurvivorRecord.java | 13 ++++++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/com/microsoft/gctoolkit/event/generational/PSYoungGen.java b/api/src/main/java/com/microsoft/gctoolkit/event/generational/PSYoungGen.java index 2d81af7d..f5e1ffba 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/event/generational/PSYoungGen.java +++ b/api/src/main/java/com/microsoft/gctoolkit/event/generational/PSYoungGen.java @@ -4,7 +4,6 @@ import com.microsoft.gctoolkit.event.GCCause; import com.microsoft.gctoolkit.event.GarbageCollectionTypes; -import com.microsoft.gctoolkit.event.jvm.SurvivorRecord; import com.microsoft.gctoolkit.time.DateTimeStamp; public class PSYoungGen extends GenerationalGCPauseEvent { @@ -16,6 +15,4 @@ public PSYoungGen(DateTimeStamp timeStamp, GarbageCollectionTypes type, GCCause public PSYoungGen(DateTimeStamp timeStamp, GCCause cause, double duration) { super(timeStamp, GarbageCollectionTypes.PSYoungGen, cause, duration); } - - } diff --git a/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java b/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java index 0be6f105..b6f1a791 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java +++ b/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java @@ -16,7 +16,7 @@ public class SurvivorRecord extends JVMEvent { private int maxTenuringThreshold; // JDK bug, we have now seen a max tenuring threshold of 32, even 64 - // Fold anything older than 15 back into the 15th slot + // Fold anything older than maxTenuringThreshold back into the maxTenuringThreshold slot private long[] bytesAtAge; public SurvivorRecord(DateTimeStamp timeStamp, long desiredOccupancy, int calculatedTenuringThreshold, int maxTenuringThreshold) { @@ -56,13 +56,12 @@ public int getMaxTenuringThreshold() { public long getBytesAtAge(int age) { if (this.bytesAtAge == null) return 0L; - return this.bytesAtAge[age]; + return this.bytesAtAge[normalizeAge(age)]; } /* * There is a bug in the JVM that allows tenuring threshold to appear to be greater than 15. - * Fold anything greater than 15 into 15. - * + * Fold anything greater than maxTenuringThreshold into maxTenuringThreshold. */ public void add(int age, long bytes) { @@ -74,7 +73,7 @@ public void add(int age, long bytes) { if (age <= maxTenuringThreshold) { bytesAtAge[age] = bytes; } else { - bytesAtAge[age] += bytes; + bytesAtAge[maxTenuringThreshold] += bytes; } } @@ -83,4 +82,8 @@ public long[] getBytesAtEachAge() { return new long[0]; return bytesAtAge; } + + private int normalizeAge(int age) { + return age <= maxTenuringThreshold ? age : maxTenuringThreshold; + } } From ef88999c5a13e0d4a0f3e0e87bddd6acef940df5 Mon Sep 17 00:00:00 2001 From: David Grieve Date: Tue, 22 Apr 2025 20:34:44 +0000 Subject: [PATCH 34/34] do not range check age arg in SurvivorRecord --- .../com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java b/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java index b6f1a791..cf9bb054 100644 --- a/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java +++ b/api/src/main/java/com/microsoft/gctoolkit/event/jvm/SurvivorRecord.java @@ -56,7 +56,7 @@ public int getMaxTenuringThreshold() { public long getBytesAtAge(int age) { if (this.bytesAtAge == null) return 0L; - return this.bytesAtAge[normalizeAge(age)]; + return this.bytesAtAge[age]; } /* @@ -82,8 +82,4 @@ public long[] getBytesAtEachAge() { return new long[0]; return bytesAtAge; } - - private int normalizeAge(int age) { - return age <= maxTenuringThreshold ? age : maxTenuringThreshold; - } }