From 4f4c701014fd69bbe4fca50e9c92649d468d885c Mon Sep 17 00:00:00 2001 From: Jack Berg Date: Mon, 24 Oct 2022 12:25:53 -0500 Subject: [PATCH 1/4] Record memory usage after garbage collection --- .../runtimemetrics/MemoryPools.java | 27 ++++++++---- .../runtimemetrics/MemoryPoolsTest.java | 42 +++++++++++++++++-- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java b/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java index 44b83c66c469..8bdf6dcbc52a 100644 --- a/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java +++ b/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java @@ -34,7 +34,8 @@ * process.runtime.jvm.memory.init{type="heap",pool="G1 Eden Space"} 1000000 * process.runtime.jvm.memory.usage{type="heap",pool="G1 Eden Space"} 2500000 * process.runtime.jvm.memory.committed{type="heap",pool="G1 Eden Space"} 3000000 - * process.runtime.jvm.memory.max{type="heap",pool="G1 Eden Space"} 4000000 + * process.runtime.jvm.memory.limit{type="heap",pool="G1 Eden Space"} 4000000 + * process.runtime.jvm.memory.usage_after_gc{type="heap",pool="G1 Eden Space"} 1500000 * process.runtime.jvm.memory.init{type="non_heap",pool="Metaspace"} 200 * process.runtime.jvm.memory.usage{type="non_heap",pool="Metaspace"} 400 * process.runtime.jvm.memory.committed{type="non_heap",pool="Metaspace"} 500 @@ -61,30 +62,41 @@ static void registerObservers(OpenTelemetry openTelemetry, List callback( - List poolBeans, Function extractor) { + List poolBeans, + Function memoryUsageExtractor, + Function valueExtractor) { List attributeSets = new ArrayList<>(poolBeans.size()); for (MemoryPoolMXBean pool : poolBeans) { attributeSets.add( @@ -97,7 +109,8 @@ static Consumer callback( return measurement -> { for (int i = 0; i < poolBeans.size(); i++) { Attributes attributes = attributeSets.get(i); - long value = extractor.apply(poolBeans.get(i).getUsage()); + MemoryUsage memoryUsage = memoryUsageExtractor.apply(poolBeans.get(i)); + long value = valueExtractor.apply(memoryUsage); if (value != -1) { measurement.record(value, attributes); } diff --git a/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java b/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java index a58bfb6eca56..9c97f65f8b4d 100644 --- a/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java +++ b/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java @@ -31,8 +31,11 @@ import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; @ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) class MemoryPoolsTest { @RegisterExtension @@ -45,6 +48,8 @@ class MemoryPoolsTest { @Mock private MemoryUsage heapPoolUsage; @Mock private MemoryUsage nonHeapUsage; + @Mock private MemoryUsage heapCollectionUsage; + @Mock private MemoryUsage nonHeapCollectionUsage; private List beans; @@ -53,9 +58,11 @@ void setup() { when(heapPoolBean.getName()).thenReturn("heap_pool"); when(heapPoolBean.getType()).thenReturn(MemoryType.HEAP); when(heapPoolBean.getUsage()).thenReturn(heapPoolUsage); + when(heapPoolBean.getCollectionUsage()).thenReturn(heapCollectionUsage); when(nonHeapPoolBean.getName()).thenReturn("non_heap_pool"); when(nonHeapPoolBean.getType()).thenReturn(MemoryType.NON_HEAP); when(nonHeapPoolBean.getUsage()).thenReturn(nonHeapUsage); + when(nonHeapPoolBean.getCollectionUsage()).thenReturn(nonHeapCollectionUsage); beans = Arrays.asList(heapPoolBean, nonHeapPoolBean); } @@ -69,7 +76,8 @@ void registerObservers() { when(nonHeapUsage.getUsed()).thenReturn(15L); when(nonHeapUsage.getCommitted()).thenReturn(16L); when(nonHeapUsage.getMax()).thenReturn(17L); - + when(heapCollectionUsage.getUsed()).thenReturn(18L); + when(nonHeapCollectionUsage.getUsed()).thenReturn(19L); MemoryPools.registerObservers(testing.getOpenTelemetry(), beans); testing.waitAndAssertMetrics( @@ -176,6 +184,33 @@ void registerObservers() { AttributeKey.stringKey("pool"), "non_heap_pool") .hasAttribute( AttributeKey.stringKey("type"), "non_heap"))))); + testing.waitAndAssertMetrics( + "io.opentelemetry.runtime-metrics", + "process.runtime.jvm.memory.usage_after_gc", + metrics -> + metrics.anySatisfy( + metricData -> + assertThat(metricData) + .hasInstrumentationScope(EXPECTED_SCOPE) + .hasDescription( + "Measure of memory used after the most recent garbage collection event on this pool") + .hasUnit("By") + .hasLongSumSatisfying( + sum -> + sum.hasPointsSatisfying( + point -> + point + .hasValue(18) + .hasAttribute( + AttributeKey.stringKey("pool"), "heap_pool") + .hasAttribute(AttributeKey.stringKey("type"), "heap"), + point -> + point + .hasValue(19) + .hasAttribute( + AttributeKey.stringKey("pool"), "non_heap_pool") + .hasAttribute( + AttributeKey.stringKey("type"), "non_heap"))))); } @Test @@ -184,7 +219,7 @@ void callback_Records() { when(nonHeapUsage.getUsed()).thenReturn(2L); Consumer callback = - MemoryPools.callback(beans, MemoryUsage::getUsed); + MemoryPools.callback(beans, MemoryPoolMXBean::getUsage, MemoryUsage::getUsed); callback.accept(measurement); verify(measurement) @@ -199,7 +234,8 @@ void callback_SkipRecord() { when(heapPoolUsage.getMax()).thenReturn(1L); when(nonHeapUsage.getMax()).thenReturn(-1L); - Consumer callback = MemoryPools.callback(beans, MemoryUsage::getMax); + Consumer callback = + MemoryPools.callback(beans, MemoryPoolMXBean::getUsage, MemoryUsage::getMax); callback.accept(measurement); verify(measurement) From 59bf66d60329ba287bc0148e1b56bd0a3244c1ae Mon Sep 17 00:00:00 2001 From: Jack Berg Date: Mon, 24 Oct 2022 12:34:50 -0500 Subject: [PATCH 2/4] Check for null MemoryUsage --- .../instrumentation/runtimemetrics/MemoryPools.java | 5 +++++ .../runtimemetrics/MemoryPoolsTest.java | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java b/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java index 8bdf6dcbc52a..6ed5c63fbcc1 100644 --- a/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java +++ b/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java @@ -110,6 +110,11 @@ static Consumer callback( for (int i = 0; i < poolBeans.size(); i++) { Attributes attributes = attributeSets.get(i); MemoryUsage memoryUsage = memoryUsageExtractor.apply(poolBeans.get(i)); + if (memoryUsage == null) { + // JVM may return null in special cases for MemoryPoolMXBean.getUsage() and + // MemoryPoolMXBean.getCollectionUsage() + return; + } long value = valueExtractor.apply(memoryUsage); if (value != -1) { measurement.record(value, attributes); diff --git a/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java b/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java index 9c97f65f8b4d..6f8b11642674 100644 --- a/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java +++ b/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java @@ -8,6 +8,7 @@ import static io.opentelemetry.instrumentation.runtimemetrics.ScopeUtil.EXPECTED_SCOPE; import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -22,6 +23,7 @@ import java.lang.management.MemoryType; import java.lang.management.MemoryUsage; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.function.Consumer; import org.junit.jupiter.api.BeforeEach; @@ -242,4 +244,15 @@ void callback_SkipRecord() { .record(1, Attributes.builder().put("pool", "heap_pool").put("type", "heap").build()); verify(measurement, never()).record(eq(-1), any()); } + + + @Test + void callback_NullUsage() { + when(heapPoolBean.getCollectionUsage()).thenReturn(null); + + Consumer callback = + MemoryPools.callback(Collections.singletonList(heapPoolBean), MemoryPoolMXBean::getCollectionUsage, MemoryUsage::getUsed); + callback.accept(measurement); + verify(measurement, never()).record(anyLong(), any()); + } } From 244d5518d0091d32bdd798be5274e36cf360c534 Mon Sep 17 00:00:00 2001 From: Jack Berg Date: Mon, 24 Oct 2022 13:59:59 -0500 Subject: [PATCH 3/4] Continue instead of return --- .../instrumentation/runtimemetrics/MemoryPools.java | 2 +- .../instrumentation/runtimemetrics/MemoryPoolsTest.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java b/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java index 6ed5c63fbcc1..ee398fd9e2e0 100644 --- a/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java +++ b/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java @@ -113,7 +113,7 @@ static Consumer callback( if (memoryUsage == null) { // JVM may return null in special cases for MemoryPoolMXBean.getUsage() and // MemoryPoolMXBean.getCollectionUsage() - return; + continue; } long value = valueExtractor.apply(memoryUsage); if (value != -1) { diff --git a/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java b/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java index 6f8b11642674..c1f4a2992181 100644 --- a/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java +++ b/instrumentation/runtime-metrics/library/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java @@ -245,13 +245,15 @@ void callback_SkipRecord() { verify(measurement, never()).record(eq(-1), any()); } - @Test void callback_NullUsage() { when(heapPoolBean.getCollectionUsage()).thenReturn(null); Consumer callback = - MemoryPools.callback(Collections.singletonList(heapPoolBean), MemoryPoolMXBean::getCollectionUsage, MemoryUsage::getUsed); + MemoryPools.callback( + Collections.singletonList(heapPoolBean), + MemoryPoolMXBean::getCollectionUsage, + MemoryUsage::getUsed); callback.accept(measurement); verify(measurement, never()).record(anyLong(), any()); } From c4bde8abcbf422a3957f2f796e9a9374430581ae Mon Sep 17 00:00:00 2001 From: Jack Berg Date: Tue, 8 Nov 2022 16:15:16 -0600 Subject: [PATCH 4/4] Rename to usage_after_last_gc --- .../javaagent/runtimemetrics/RuntimeMetricsTest.groovy | 1 + instrumentation/runtime-metrics/library/README.md | 6 ++++++ .../instrumentation/runtimemetrics/MemoryPools.java | 4 ++-- .../instrumentation/runtimemetrics/MemoryPoolsTest.java | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/instrumentation/runtime-metrics/javaagent/src/test/groovy/io/opentelemetry/instrumentation/javaagent/runtimemetrics/RuntimeMetricsTest.groovy b/instrumentation/runtime-metrics/javaagent/src/test/groovy/io/opentelemetry/instrumentation/javaagent/runtimemetrics/RuntimeMetricsTest.groovy index 8a55a152c81c..64b6a9c65241 100644 --- a/instrumentation/runtime-metrics/javaagent/src/test/groovy/io/opentelemetry/instrumentation/javaagent/runtimemetrics/RuntimeMetricsTest.groovy +++ b/instrumentation/runtime-metrics/javaagent/src/test/groovy/io/opentelemetry/instrumentation/javaagent/runtimemetrics/RuntimeMetricsTest.groovy @@ -28,6 +28,7 @@ class RuntimeMetricsTest extends AgentInstrumentationSpecification { assert getMetrics().any { it.name == "process.runtime.jvm.memory.usage" } assert getMetrics().any { it.name == "process.runtime.jvm.memory.committed" } assert getMetrics().any { it.name == "process.runtime.jvm.memory.limit" } + assert getMetrics().any { it.name == "process.runtime.jvm.memory.usage_after_last_gc" } assert getMetrics().any { it.name == "process.runtime.jvm.threads.count" } assert getMetrics().any { it.name == "process.runtime.jvm.buffer.limit" } assert getMetrics().any { it.name == "process.runtime.jvm.buffer.count" } diff --git a/instrumentation/runtime-metrics/library/README.md b/instrumentation/runtime-metrics/library/README.md index 8b692acaf082..eafeb540bfbc 100644 --- a/instrumentation/runtime-metrics/library/README.md +++ b/instrumentation/runtime-metrics/library/README.md @@ -53,28 +53,34 @@ The following lists attributes reported for a variety of garbage collectors. Not * `process.runtime.jvm.memory.usage`: {pool=Compressed Class Space,type=non_heap}, {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}, {pool=Code Cache,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.committed`: {pool=Compressed Class Space,type=non_heap}, {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}, {pool=Code Cache,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.limit`: {pool=Compressed Class Space,type=non_heap}, {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}, {pool=Code Cache,type=non_heap} + * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap} * G1 Garbage Collector * `process.runtime.jvm.memory.init`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.usage`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.committed`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap} + * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=G1 Old Gen,type=heap} * Parallel Garbage Collector * `process.runtime.jvm.memory.init`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.usage`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.committed`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap} + * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap} * Serial Garbage Collector * `process.runtime.jvm.memory.init`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.usage`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.committed`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap} * `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap} + * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap} * Shenandoah Garbage Collector * `process.runtime.jvm.memory.init`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap} * `process.runtime.jvm.memory.usage`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap} * `process.runtime.jvm.memory.committed`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap} * `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap} + * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=Shenandoah,type=heap} * Z Garbage Collector * `process.runtime.jvm.memory.init`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap} * `process.runtime.jvm.memory.usage`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap} * `process.runtime.jvm.memory.committed`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap} * `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap} + * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=ZHeap,type=heap} diff --git a/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java b/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java index ee398fd9e2e0..033ddf8d4520 100644 --- a/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java +++ b/instrumentation/runtime-metrics/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java @@ -35,7 +35,7 @@ * process.runtime.jvm.memory.usage{type="heap",pool="G1 Eden Space"} 2500000 * process.runtime.jvm.memory.committed{type="heap",pool="G1 Eden Space"} 3000000 * process.runtime.jvm.memory.limit{type="heap",pool="G1 Eden Space"} 4000000 - * process.runtime.jvm.memory.usage_after_gc{type="heap",pool="G1 Eden Space"} 1500000 + * process.runtime.jvm.memory.usage_after_last_gc{type="heap",pool="G1 Eden Space"} 1500000 * process.runtime.jvm.memory.init{type="non_heap",pool="Metaspace"} 200 * process.runtime.jvm.memory.usage{type="non_heap",pool="Metaspace"} 400 * process.runtime.jvm.memory.committed{type="non_heap",pool="Metaspace"} 500 @@ -84,7 +84,7 @@ static void registerObservers(OpenTelemetry openTelemetry, List metrics.anySatisfy( metricData ->