Add Application Signals runtime metrics (#892)

## Feature request
Add runtime metrics collection into Application Signals.

## Description of changes:
This PR is an umbrella PR to track the ongoing changes for runtime
metrics.
1. [Add Application Signals runtime metrics with feature disabled
#900](#900)
[Merged in main]
2. [feat: Add contract tests for runtime metrics
#893](#893)
3. Enable runtime metrics by default

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

*Issue #, if available:*

*Description of changes:*


By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

---------

Co-authored-by: Reno Seo <renoseo@amazon.com>

Author: bjrara

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/base/ContractTestBase.java

@@ -79,6 +79,7 @@ public abstract class ContractTestBase {
           .withEnv("JAVA_TOOL_OPTIONS", "-javaagent:/opentelemetry-javaagent-all.jar")
           .withEnv("OTEL_METRIC_EXPORT_INTERVAL", "100") // 100 ms
           .withEnv("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "true")
+          .withEnv("OTEL_AWS_APPLICATION_SIGNALS_RUNTIME_ENABLED", isRuntimeEnabled())
           .withEnv("OTEL_METRICS_EXPORTER", "none")
           .withEnv("OTEL_BSP_SCHEDULE_DELAY", "0") // Don't wait to export spans to the collector
           .withEnv(
@@ -159,4 +160,8 @@ protected String getApplicationOtelServiceName() {
   protected String getApplicationOtelResourceAttributes() {
     return "service.name=" + getApplicationOtelServiceName();
   }
+
+  protected String isRuntimeEnabled() {
+    return "false";
+  }
 }

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java

@@ -0,0 +1,165 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.opentelemetry.appsignals.test.misc;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import io.opentelemetry.proto.metrics.v1.Metric;
+import java.util.List;
+import java.util.Set;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import software.amazon.opentelemetry.appsignals.test.base.ContractTestBase;
+import software.amazon.opentelemetry.appsignals.test.utils.AppSignalsConstants;
+import software.amazon.opentelemetry.appsignals.test.utils.ResourceScopeMetric;
+
+public class RuntimeMetricsTest {
+  private abstract static class RuntimeMetricsContractTestBase extends ContractTestBase {
+    @Override
+    protected String getApplicationImageName() {
+      return "aws-appsignals-tests-http-server-spring-mvc";
+    }
+
+    @Override
+    protected String isRuntimeEnabled() {
+      return "true";
+    }
+
+    protected String getApplicationWaitPattern() {
+      return ".*Started Application.*";
+    }
+
+    protected void doTestRuntimeMetrics() {
+      var response = appClient.get("/success").aggregate().join();
+
+      assertThat(response.status().isSuccess()).isTrue();
+      assertRuntimeMetrics();
+    }
+
+    protected void assertRuntimeMetrics() {
+      var metrics =
+          mockCollectorClient.getRuntimeMetrics(
+              Set.of(
+                  AppSignalsConstants.JVM_GC_DURATION,
+                  AppSignalsConstants.JVM_GC_COUNT,
+                  AppSignalsConstants.JVM_HEAP_USED,
+                  AppSignalsConstants.JVM_NON_HEAP_USED,
+                  AppSignalsConstants.JVM_AFTER_GC,
+                  AppSignalsConstants.JVM_POOL_USED,
+                  AppSignalsConstants.JVM_THREAD_COUNT,
+                  AppSignalsConstants.JVM_CLASS_LOADED,
+                  AppSignalsConstants.JVM_CPU_TIME,
+                  AppSignalsConstants.JVM_CPU_UTILIZATION,
+                  AppSignalsConstants.LATENCY_METRIC,
+                  AppSignalsConstants.ERROR_METRIC,
+                  AppSignalsConstants.FAULT_METRIC));
+
+      testResourceAttributes(metrics);
+      for (String metricName : List.of(AppSignalsConstants.JVM_POOL_USED)) {
+        testGaugeMetrics(metrics, metricName, "name");
+      }
+      for (String metricName :
+          List.of(
+              AppSignalsConstants.JVM_HEAP_USED,
+              AppSignalsConstants.JVM_NON_HEAP_USED,
+              AppSignalsConstants.JVM_AFTER_GC,
+              AppSignalsConstants.JVM_THREAD_COUNT,
+              AppSignalsConstants.JVM_CLASS_LOADED,
+              AppSignalsConstants.JVM_CPU_UTILIZATION)) {
+        testGaugeMetrics(metrics, metricName, "");
+      }
+      for (String metricName :
+          List.of(AppSignalsConstants.JVM_GC_DURATION, AppSignalsConstants.JVM_GC_COUNT)) {
+        testCounterMetrics(metrics, metricName, "name");
+      }
+      for (String metricName : List.of(AppSignalsConstants.JVM_CPU_TIME)) {
+        testCounterMetrics(metrics, metricName, "");
+      }
+    }
+
+    private void testGaugeMetrics(
+        List<ResourceScopeMetric> resourceScopeMetrics, String metricName, String attributeKey) {
+      for (ResourceScopeMetric rsm : resourceScopeMetrics) {
+        Metric metric = rsm.getMetric();
+        if (metricName.equals(metric.getName())) {
+          assertThat(metric.getGauge().getDataPointsList())
+              .as(metricName + " is not empty")
+              .isNotEmpty();
+          assertThat(metric.getGauge().getDataPointsList())
+              .as(metricName + " is valid")
+              .allMatch(
+                  dp -> {
+                    boolean valid = true;
+                    if (!attributeKey.isEmpty()) {
+                      valid =
+                          dp.getAttributesList().stream()
+                              .anyMatch(attribute -> attribute.getKey().equals(attributeKey));
+                    }
+                    return valid && dp.getAsInt() >= 0;
+                  });
+        }
+      }
+    }
+
+    private void testCounterMetrics(
+        List<ResourceScopeMetric> resourceScopeMetrics, String metricName, String attributeKey) {
+      for (ResourceScopeMetric rsm : resourceScopeMetrics) {
+        Metric metric = rsm.getMetric();
+        if (metricName.equals(metric.getName())) {
+          assertThat(metric.getSum().getDataPointsList())
+              .as(metricName + " is not empty")
+              .isNotEmpty();
+          assertThat(metric.getSum().getDataPointsList())
+              .as(metricName + " is valid")
+              .allMatch(
+                  dp -> {
+                    boolean valid = true;
+                    if (!attributeKey.isEmpty()) {
+                      valid =
+                          dp.getAttributesList().stream()
+                              .anyMatch(attribute -> attribute.getKey().equals(attributeKey));
+                    }
+                    return valid && dp.getAsInt() >= 0;
+                  });
+        }
+      }
+    }
+
+    private void testResourceAttributes(List<ResourceScopeMetric> resourceScopeMetrics) {
+      for (ResourceScopeMetric rsm : resourceScopeMetrics) {
+        assertThat(rsm.getResource().getResource().getAttributesList())
+            .anyMatch(
+                attr ->
+                    attr.getKey().equals(AppSignalsConstants.AWS_LOCAL_SERVICE)
+                        && attr.getValue()
+                            .getStringValue()
+                            .equals(getApplicationOtelServiceName()));
+      }
+    }
+  }
+
+  @Testcontainers(disabledWithoutDocker = true)
+  @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+  @Nested
+  class ValidateRuntimeMetricsTest extends RuntimeMetricsContractTestBase {
+    @Test
+    void testRuntimeMetrics() {
+      doTestRuntimeMetrics();
+    }
+  }
+}

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java

@@ -33,4 +33,16 @@ public class AppSignalsConstants {
   public static final String AWS_REMOTE_RESOURCE_IDENTIFIER = "aws.remote.resource.identifier";
   public static final String AWS_SPAN_KIND = "aws.span.kind";
   public static final String AWS_REMOTE_DB_USER = "aws.remote.db.user";
+
+  // JVM Metrics
+  public static final String JVM_GC_DURATION = "jvm.gc.collections.elapsed";
+  public static final String JVM_GC_COUNT = "jvm.gc.collections.count";
+  public static final String JVM_HEAP_USED = "jvm.memory.heap.used";
+  public static final String JVM_NON_HEAP_USED = "jvm.memory.nonheap.used";
+  public static final String JVM_AFTER_GC = "jvm.memory.pool.used_after_last_gc";
+  public static final String JVM_POOL_USED = "jvm.memory.pool.used";
+  public static final String JVM_THREAD_COUNT = "jvm.threads.count";
+  public static final String JVM_CLASS_LOADED = "jvm.classes.loaded";
+  public static final String JVM_CPU_TIME = "jvm.cpu.time";
+  public static final String JVM_CPU_UTILIZATION = "jvm.cpu.recent_utilization";
 }

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java

@@ -133,13 +133,21 @@ public List<ResourceScopeSpan> getTraces() {
         .collect(toImmutableList());
   }
 
+  public List<ResourceScopeMetric> getRuntimeMetrics(Set<String> presentMetrics) {
+    return fetchMetrics(presentMetrics, false);
+  }
+
+  public List<ResourceScopeMetric> getMetrics(Set<String> presentMetrics) {
+    return fetchMetrics(presentMetrics, true);
+  }
+
   /**
    * Get all metrics that are currently stored in the mock collector.
    *
    * @return List of `ResourceScopeMetric` which is a flat list containing all metrics and their
    *     related scope and resources.
    */
-  public List<ResourceScopeMetric> getMetrics(Set<String> presentMetrics) {
+  private List<ResourceScopeMetric> fetchMetrics(Set<String> presentMetrics, boolean exactMatch) {
     List<ExportMetricsServiceRequest> exportedMetrics =
         waitForContent(
             "/get-metrics",
@@ -152,9 +160,14 @@ public List<ResourceScopeMetric> getMetrics(Set<String> presentMetrics) {
                       .flatMap(x -> x.getMetricsList().stream())
                       .map(x -> x.getName())
                       .collect(Collectors.toSet());
-
-              return (!exported.isEmpty() && current.size() == exported.size())
-                  && receivedMetrics.containsAll(presentMetrics);
+              if (!exported.isEmpty() && receivedMetrics.containsAll(presentMetrics)) {
+                if (exactMatch) {
+                  return current.size() == exported.size();
+                } else {
+                  return true;
+                }
+              }
+              return false;
             });
 
     return exportedMetrics.stream()

awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsApplicationSignalsCustomizerProvider.java

@@ -104,7 +104,8 @@ private boolean isApplicationSignalsEnabled(ConfigProperties configProps) {
   }
 
   private boolean isApplicationSignalsRuntimeEnabled(ConfigProperties configProps) {
-    return false;
+    return isApplicationSignalsEnabled(configProps)
+        && configProps.getBoolean(APPLICATION_SIGNALS_RUNTIME_ENABLED_CONFIG, true);
   }
 
   private Map<String, String> customizeProperties(ConfigProperties configProps) {

instrumentation/jmx-metrics/src/main/resources/jmx/rules/jvm.yaml

@@ -23,7 +23,7 @@ rules:
         unit: ms
         desc: The approximate accumulated collection elapsed time in milliseconds
   - bean: java.lang:type=Memory
-    unit: by
+    unit: By
     prefix: jvm.memory.
     type: gauge
     mapping:
@@ -52,12 +52,15 @@ rules:
         metric: nonheap.max
         desc: The maximum amount of memory can be used for non-heap purposes
   - bean: java.lang:type=MemoryPool,name=*
-    unit: by
+    unit: By
     prefix: jvm.memory.pool.
     type: gauge
     metricAttribute:
       name: param(name)
     mapping:
+      CollectionUsage.used:
+        metric: used_after_last_gc
+        desc: Memory used after the most recent gc event
       Usage.init:
         metric: init
         desc: The initial amount of memory that the JVM requests from the operating system for the memory pool
@@ -81,37 +84,49 @@ rules:
         metric: jvm.daemon_threads.count
         desc: Number of daemon threads
   - bean: java.lang:type=OperatingSystem
-    type: gauge
     mapping:
       TotalSwapSpaceSize:
         metric: jvm.system.swap.space.total
-        desc: The host swap memory size in bytes
-        unit: by
+        type: gauge
+        desc: The host swap memory size in Bytes
+        unit: By
       FreeSwapSpaceSize:
         metric: jvm.system.swap.space.free
-        desc: The amount of available swap memory in bytes
-        unit: by
+        type: gauge
+        desc: The amount of available swap memory in Bytes
+        unit: By
       TotalPhysicalMemorySize:
         metric: jvm.system.physical.memory.total
+        type: gauge
         desc: The total physical memory size in host
-        unit: by
+        unit: By
       FreePhysicalMemorySize:
         metric: jvm.system.physical.memory.free
+        type: gauge
         desc: The amount of free physical memory in host
-        unit: by
+        unit: By
       AvailableProcessors:
         metric: jvm.system.available.processors
+        type: gauge
         desc: The number of available processors
         unit: "1"
       SystemCpuLoad:
         metric: jvm.system.cpu.utilization
+        type: gauge
         desc: The current load of CPU in host
         unit: "1"
+      ProcessCpuTime:
+        metric: jvm.cpu.time
+        type: counter
+        unit: ns
+        desc: CPU time used
       ProcessCpuLoad:
         metric: jvm.cpu.recent_utilization
+        type: gauge
         unit: "1"
         desc: Recent CPU utilization for the process
       OpenFileDescriptorCount:
         metric: jvm.open_file_descriptor.count
+        type: gauge
         desc: The number of opened file descriptors
         unit: "1"