Cleanups.

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrFeature.java

@@ -27,7 +27,6 @@
 import java.util.Collections;
 import java.util.List;
 
-import com.oracle.svm.core.sampler.SamplerStatistics;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.hosted.Feature;
@@ -43,6 +42,7 @@
 import com.oracle.svm.core.sampler.SamplerJfrStackTraceSerializer;
 import com.oracle.svm.core.sampler.SamplerStackTraceSerializer;
 import com.oracle.svm.core.sampler.SamplerStackWalkVisitor;
+import com.oracle.svm.core.sampler.SamplerStatistics;
 import com.oracle.svm.core.thread.ThreadListenerSupport;
 import com.oracle.svm.core.thread.ThreadListenerSupportFeature;
 import com.oracle.svm.core.util.UserError;
@@ -175,6 +175,7 @@ public void afterRegistration(AfterRegistrationAccess access) {
         if (VMInspectionOptions.hasNativeMemoryTrackingSupport()) {
             JfrSerializerSupport.get().register(new JfrNmtCategorySerializer());
         }
+
         ThreadListenerSupport.get().register(SubstrateJVM.getThreadLocal());
 
         RuntimeClassInitializationSupport rci = ImageSingletons.lookup(RuntimeClassInitializationSupport.class);

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrNmtCategorySerializer.java

@@ -26,10 +26,11 @@
 
 package com.oracle.svm.core.jfr;
 
-import com.oracle.svm.core.nmt.NmtCategory;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 
+import com.oracle.svm.core.nmt.NmtCategory;
+
 public class JfrNmtCategorySerializer implements JfrSerializer {
     @Platforms(Platform.HOSTED_ONLY.class)
     public JfrNmtCategorySerializer() {
@@ -41,9 +42,9 @@ public void write(JfrChunkWriter writer) {
 
         NmtCategory[] nmtCategories = NmtCategory.values();
         writer.writeCompressedLong(nmtCategories.length);
-        for (int i = 0; i < nmtCategories.length; i++) {
-            writer.writeCompressedInt(i);
-            writer.writeString(nmtCategories[i].getName());
+        for (NmtCategory nmtCategory : nmtCategories) {
+            writer.writeCompressedInt(nmtCategory.ordinal());
+            writer.writeString(nmtCategory.getName());
         }
     }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/EveryChunkNativePeriodicEvents.java

@@ -30,6 +30,7 @@
 import org.graalvm.nativeimage.StackValue;
 
 import com.oracle.svm.core.Uninterruptible;
+import com.oracle.svm.core.VMInspectionOptions;
 import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.heap.PhysicalMemory;
 import com.oracle.svm.core.heap.VMOperationInfos;
@@ -39,11 +40,10 @@
 import com.oracle.svm.core.jfr.JfrNativeEventWriterData;
 import com.oracle.svm.core.jfr.JfrNativeEventWriterDataAccess;
 import com.oracle.svm.core.jfr.JfrTicks;
-import com.oracle.svm.core.nmt.NmtCategory;
 import com.oracle.svm.core.nmt.NativeMemoryTracking;
+import com.oracle.svm.core.nmt.NmtCategory;
 import com.oracle.svm.core.thread.JavaVMOperation;
 import com.oracle.svm.core.thread.VMThreads;
-import com.oracle.svm.core.VMInspectionOptions;
 
 import jdk.jfr.Event;
 import jdk.jfr.Name;
@@ -120,7 +120,7 @@ private static void emitNativeMemoryTrackingEvents() {
         }
     }
 
-    /** These peak events are specific to SubstrateVM. */
+    /** Emit Native Image-specific events that report the peak memory usage. */
     private static void emitNmtPeakEvents() {
         NativeMemoryUsageTotalPeakEvent nmtTotalPeakEvent = new NativeMemoryUsageTotalPeakEvent();
         nmtTotalPeakEvent.countAtPeak = NativeMemoryTracking.singleton().getCountAtTotalPeakUsage();
@@ -144,27 +144,26 @@ private static void emitJdkNmtEvents(NmtCategory[] nmtCategories) {
 
         if (JfrEvent.NativeMemoryUsage.shouldEmit()) {
             for (NmtCategory nmtCategory : nmtCategories) {
+                long usedMemory = NativeMemoryTracking.singleton().getUsedMemory(nmtCategory);
+
                 JfrNativeEventWriter.beginSmallEvent(data, JfrEvent.NativeMemoryUsage);
                 JfrNativeEventWriter.putLong(data, timestamp);
-                /* Category */
                 JfrNativeEventWriter.putLong(data, nmtCategory.ordinal());
-                /* Reserved usage */
-                JfrNativeEventWriter.putLong(data, NativeMemoryTracking.singleton().getUsedMemory(nmtCategory));
-                /* Committed usage */
-                JfrNativeEventWriter.putLong(data, NativeMemoryTracking.singleton().getUsedMemory(nmtCategory));
+                JfrNativeEventWriter.putLong(data, usedMemory); // reserved
+                JfrNativeEventWriter.putLong(data, usedMemory); // committed
                 JfrNativeEventWriter.endSmallEvent(data);
             }
         }
+
         if (JfrEvent.NativeMemoryUsageTotal.shouldEmit()) {
+            long totalUsedMemory = NativeMemoryTracking.singleton().getTotalUsedMemory();
+
             JfrNativeEventWriter.beginSmallEvent(data, JfrEvent.NativeMemoryUsageTotal);
             JfrNativeEventWriter.putLong(data, timestamp);
-            /* Reserved usage */
-            JfrNativeEventWriter.putLong(data, NativeMemoryTracking.singleton().getTotalUsedMemory());
-            /* Committed usage */
-            JfrNativeEventWriter.putLong(data, NativeMemoryTracking.singleton().getTotalUsedMemory());
+            JfrNativeEventWriter.putLong(data, totalUsedMemory); // reserved
+            JfrNativeEventWriter.putLong(data, totalUsedMemory); // committed
             JfrNativeEventWriter.endSmallEvent(data);
         }
-
     }
 
     private static void emitPerThreadEvents() {
@@ -176,7 +175,7 @@ private static void emitPerThreadEvents() {
 
     @Uninterruptible(reason = "Used to avoid the VM operation if it is not absolutely needed.")
     private static boolean needsVMOperation() {
-        /* The returned value is racy. */
+        /* The returned value is racy but this is fine because we recheck in the VM operation. */
         return JfrEvent.ThreadCPULoad.shouldEmit() || JfrEvent.ThreadAllocationStatistics.shouldEmit();
     }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/NativeMemoryUsagePeakEvent.java

@@ -28,14 +28,17 @@
 import jdk.jfr.Category;
 import jdk.jfr.Description;
 import jdk.jfr.Event;
+import jdk.jfr.Experimental;
 import jdk.jfr.Label;
 import jdk.jfr.Name;
 import jdk.jfr.StackTrace;
 
-@Name("svm.NativeMemoryUsagePeak")
+/** Similar to the JFR event jdk.NativeMemoryUsage, except that it tracks the peak usage. */
+@Experimental
+@Name("jdk.NativeMemoryUsagePeak")
 @Label("Native Memory Usage Peak")
-@Description("Information about native memory peak usage of committed virtual memory and malloc.")
-@Category("Native Image")
+@Description("Native memory peak usage for a given memory type in the JVM.")
+@Category({"Java Virtual Machine", "Memory"})
 @StackTrace(false)
 public class NativeMemoryUsagePeakEvent extends Event {
     @Label("Memory Type") public String type;

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/events/NativeMemoryUsageTotalPeakEvent.java

@@ -28,14 +28,17 @@
 import jdk.jfr.Category;
 import jdk.jfr.Description;
 import jdk.jfr.Event;
+import jdk.jfr.Experimental;
 import jdk.jfr.Label;
 import jdk.jfr.Name;
 import jdk.jfr.StackTrace;
 
-@Name("svm.NativeMemoryUsageTotalPeak")
+/** Similar to the JFR event jdk.NativeMemoryUsageTotal, except that it tracks the peak usage. */
+@Experimental
+@Name("jdk.NativeMemoryUsageTotalPeak")
 @Label("Native Memory Usage Total Peak")
-@Description("Information about native memory peak usage of committed virtual memory and malloc.")
-@Category("Native Image")
+@Description("Total native memory peak usage for the JVM. Can be slightly smaller than the exact sum of the NativeMemoryUsagePeak events due to timing.")
+@Category({"Java Virtual Machine", "Memory"})
 @StackTrace(false)
 public class NativeMemoryUsageTotalPeakEvent extends Event {
     @Label("Peak Usage") public long peakUsage;

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NativeMemoryTracking.java

@@ -57,10 +57,17 @@
 public class NativeMemoryTracking {
     private static final UnsignedWord ALIGNMENT = WordFactory.unsigned(16);
     private static final int MAGIC = 0xF0F1F2F3;
-    private final NmtMallocMemorySnapshot mallocMemorySnapshot = new NmtMallocMemorySnapshot();
+
+    private final NmtMallocMemoryInfo[] categories;
+    private final NmtMallocMemoryInfo total;
 
     @Platforms(Platform.HOSTED_ONLY.class)
     public NativeMemoryTracking() {
+        total = new NmtMallocMemoryInfo();
+        categories = new NmtMallocMemoryInfo[NmtCategory.values().length];
+        for (int i = 0; i < categories.length; i++) {
+            categories[i] = new NmtMallocMemoryInfo();
+        }
     }
 
     @Fold
@@ -71,7 +78,7 @@ public static NativeMemoryTracking singleton() {
     @Fold
     public static UnsignedWord sizeOfNmtHeader() {
         /*
-         * Align the header to 16 bytes to preserve platform-specific malloc alignments up to 16
+         * Align the header to 16 bytes to preserve platform-specific malloc alignment up to 16
          * bytes (i.e., the allocation payload is aligned to 16 bytes if the platform-specific
          * malloc implementation returns a pointer that is aligned to at least 16 bytes).
          */
@@ -109,9 +116,9 @@ public void track(PointerBase innerPtr) {
         UnsignedWord allocationSize = header.getAllocationSize();
         UnsignedWord totalSize = allocationSize.add(nmtHeaderSize);
 
-        mallocMemorySnapshot.getInfoByCategory(header.getCategory()).track(allocationSize);
-        mallocMemorySnapshot.getInfoByCategory(NmtCategory.NMT).track(nmtHeaderSize);
-        mallocMemorySnapshot.getTotalInfo().track(totalSize);
+        getInfo(header.getCategory()).track(allocationSize);
+        getInfo(NmtCategory.NMT).track(nmtHeaderSize);
+        total.track(totalSize);
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
@@ -127,9 +134,9 @@ public PointerBase untrack(PointerBase innerPtr) {
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public void untrack(UnsignedWord size, int category) {
-        mallocMemorySnapshot.getInfoByCategory(category).untrack(size);
-        mallocMemorySnapshot.getInfoByCategory(NmtCategory.NMT).untrack(sizeOfNmtHeader());
-        mallocMemorySnapshot.getTotalInfo().untrack(size.add(sizeOfNmtHeader()));
+        getInfo(category).untrack(size);
+        getInfo(NmtCategory.NMT).untrack(sizeOfNmtHeader());
+        total.untrack(size.add(sizeOfNmtHeader()));
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
@@ -146,44 +153,55 @@ private static Pointer getInnerPointer(NmtMallocHeader mallocHeader) {
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public long getUsedMemory(NmtCategory category) {
-        return mallocMemorySnapshot.getInfoByCategory(category).getUsed();
+        return getInfo(category).getUsed();
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public long getPeakUsedMemory(NmtCategory category) {
-        return mallocMemorySnapshot.getInfoByCategory(category).getPeakUsed();
+        return getInfo(category).getPeakUsed();
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public long getCountAtPeakUsage(NmtCategory category) {
-        return mallocMemorySnapshot.getInfoByCategory(category).getCountAtPeakUsage();
+        return getInfo(category).getCountAtPeakUsage();
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public long getTotalCount() {
-        return mallocMemorySnapshot.getTotalInfo().getCount();
+        return total.getCount();
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public long getTotalUsedMemory() {
-        return mallocMemorySnapshot.getTotalInfo().getUsed();
+        return total.getUsed();
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public long getPeakTotalUsedMemory() {
-        return mallocMemorySnapshot.getTotalInfo().getPeakUsed();
+        return total.getPeakUsed();
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public long getCountAtTotalPeakUsage() {
-        return mallocMemorySnapshot.getTotalInfo().getCountAtPeakUsage();
+        return total.getCountAtPeakUsage();
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    private NmtMallocMemoryInfo getInfo(NmtCategory category) {
+        return getInfo(category.ordinal());
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    private NmtMallocMemoryInfo getInfo(int category) {
+        assert category < categories.length;
+        return categories[category];
     }
 
     public static RuntimeSupport.Hook shutdownHook() {
         return isFirstIsolate -> NativeMemoryTracking.singleton().printStatistics();
     }
 
-    public void printStatistics() {
+    private void printStatistics() {
         if (VMInspectionOptions.PrintNMTStatistics.getValue()) {
             System.out.println();
             System.out.println("Native memory tracking");
@@ -194,12 +212,12 @@ public void printStatistics() {
 
             for (int i = 0; i < NmtCategory.values().length; i++) {
                 String name = NmtCategory.values()[i].getName();
-                NmtMallocMemoryInfo info = mallocMemorySnapshot.getInfoByCategory(i);
+                NmtMallocMemoryInfo info = getInfo(i);
 
                 System.out.println("  " + name + " peak used memory: " + info.getPeakUsed() + " bytes");
                 System.out.println("  " + name + " alive allocations at peak: " + info.getCountAtPeakUsage());
-                System.out.println("  " + name + " used memory: " + info.getUsed() + " bytes");
-                System.out.println("  " + name + " alive allocations: " + info.getCount());
+                System.out.println("  " + name + " currently used memory: " + info.getUsed() + " bytes");
+                System.out.println("  " + name + " currently alive allocations: " + info.getCount());
             }
         }
     }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/nmt/NmtMallocMemoryInfo.java

@@ -35,38 +35,33 @@
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong;
 
-import jdk.internal.misc.Unsafe;
-
 class NmtMallocMemoryInfo {
-    private static final Unsafe U = Unsafe.getUnsafe();
-    protected static final long PEAK_USAGE_OFFSET = U.objectFieldOffset(NmtMallocMemoryInfo.class, "peakUsage");
     private final AtomicLong count = new AtomicLong(0);
     private final AtomicLong used = new AtomicLong(0);
-    private volatile long peakUsage;
-    private volatile long peakCount;
+    private final AtomicLong countAtPeakUsage = new AtomicLong(0);
+    private final AtomicLong peakUsed = new AtomicLong(0);
 
     @Platforms(Platform.HOSTED_ONLY.class)
     NmtMallocMemoryInfo() {
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     void track(UnsignedWord allocationSize) {
-        /*
-         * Similar to Hotspot, we only make a best effort to record the count at the time of the
-         * peak. Observing the memory used and count is not together atomic.
-         */
-        updatePeak(used.addAndGet(allocationSize.rawValue()), count.incrementAndGet());
+        long newUsed = used.addAndGet(allocationSize.rawValue());
+        long newCount = count.incrementAndGet();
+        updatePeak(newUsed, newCount);
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     private void updatePeak(long newUsed, long newCount) {
-        long expectedPeakUsage = peakUsage;
-        while (expectedPeakUsage < newUsed) {
-            if (U.compareAndSetLong(this, PEAK_USAGE_OFFSET, expectedPeakUsage, newUsed)) {
-                peakCount = newCount;
+        long oldUsed = peakUsed.get();
+        while (oldUsed < newUsed) {
+            if (peakUsed.compareAndSet(oldUsed, newUsed)) {
+                /* Recording the count at peak usage is racy (similar to Hotspot). */
+                countAtPeakUsage.set(newCount);
                 return;
             }
-            expectedPeakUsage = peakUsage;
+            oldUsed = peakUsed.get();
         }
     }
 
@@ -89,11 +84,11 @@ long getCount() {
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     long getPeakUsed() {
-        return peakUsage;
+        return peakUsed.get();
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     long getCountAtPeakUsage() {
-        return peakCount;
+        return countAtPeakUsage.get();
     }
 }