Add agent self-observability.

.github/actions/build/action.yml

@@ -56,7 +56,7 @@ runs:
         ./mvnw -q --batch-mode clean package -Dmaven.test.skip || \
         ./mvnw -q --batch-mode clean package -Dmaven.test.skip
         echo "::endgroup::"
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v4
       name: Upload Agent
       with:
         name: skywalking-agent
@@ -89,7 +89,7 @@ runs:
         docker save -o test-containers/skywalking-agent-test-jvm-1.0.0.tgz skywalking/agent-test-jvm:1.0.0
         docker save -o test-containers/skywalking-agent-test-tomcat-1.0.0.tgz skywalking/agent-test-tomcat:1.0.0
         echo "::endgroup::"
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v4
       name: Upload Test Containers
       with:
         name: test-tools

.github/actions/run/action.yml

@@ -25,11 +25,11 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: actions/download-artifact@v2
+    - uses: actions/download-artifact@v4
       with:
         name: skywalking-agent
         path: skywalking-agent
-    - uses: actions/download-artifact@v2
+    - uses: actions/download-artifact@v4
       with:
         name: test-tools
     - name: Load Test Containers
@@ -47,13 +47,22 @@ runs:
         path: ~/.m2/repository
         key: ${{ runner.os }}-maven-agent-test-run-${{ hashFiles('**/pom.xml') }}
         restore-keys: ${{ runner.os }}-maven-agent-test-run-
+    - name: Install docker-compose
+      shell: bash
+      if: runner.os != 'Windows'
+      run: |
+        if ! command docker-compose 2>&1 > /dev/null; then
+          echo "Installing docker-compose"
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+        fi
     - name: Run Plugin Test ${{ inputs.test_case }}
       shell: bash
       run: |
         echo "::group::Run Plugin Test ${{ inputs.test_case }}"
         bash test/plugin/run.sh ${{ inputs.test_case }}
         echo "::endgroup::"
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v4
       name: Upload Agent
       if: always()
       with:

.github/workflows/plugins-test.0.yaml

@@ -53,6 +53,7 @@ jobs:
       matrix:
         case:
           - activemq-scenario
+          - agent-so11y-scenario
           - apm-toolkit-trace-scenario
           - apm-toolkit-tracer-scenario
           - armeria-0.96minus-scenario

.github/workflows/publish-docker.yaml

@@ -47,7 +47,7 @@ jobs:
           java-version: 17
       - name: Build Agent
         run: make build
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         name: Upload Agent
         with:
           name: skywalking-agent
@@ -71,7 +71,7 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         with:
           name: skywalking-agent
           path: skywalking-agent

CHANGES.md

@@ -6,6 +6,7 @@ Release Notes.
 ------------------
 
 * Upgrade nats plugin to support 2.16.5
+* Add agent self-observability.
 
 
 All issues and pull requests are [here](https://github.com/apache/skywalking/milestone/222?closed=1)

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/ContextManager.java

@@ -26,6 +26,7 @@
 import org.apache.skywalking.apm.agent.core.logging.api.ILog;
 import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
 import org.apache.skywalking.apm.agent.core.sampling.SamplingService;
+import org.apache.skywalking.apm.agent.core.so11y.AgentSO11Y;
 import org.apache.skywalking.apm.util.StringUtil;
 
 import static org.apache.skywalking.apm.agent.core.conf.Config.Agent.OPERATION_NAME_THRESHOLD;
@@ -52,6 +53,7 @@ private static AbstractTracerContext getOrCreate(String operationName, boolean f
                 if (LOGGER.isDebugEnable()) {
                     LOGGER.debug("No operation name, ignore this trace.");
                 }
+                AgentSO11Y.recordTracingContextCreate(forceSampling, true);
                 context = new IgnoredTracerContext();
             } else {
                 if (EXTEND_SERVICE == null) {

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/ContextManagerExtendService.java

@@ -33,6 +33,7 @@
 import org.apache.skywalking.apm.agent.core.remote.GRPCChannelManager;
 import org.apache.skywalking.apm.agent.core.remote.GRPCChannelStatus;
 import org.apache.skywalking.apm.agent.core.sampling.SamplingService;
+import org.apache.skywalking.apm.agent.core.so11y.AgentSO11Y;
 import org.apache.skywalking.apm.util.StringUtil;
 
 @DefaultImplementor
@@ -81,17 +82,22 @@ public AbstractTracerContext createTraceContext(String operationName, boolean fo
          * Don't trace anything if the backend is not available.
          */
         if (!Config.Agent.KEEP_TRACING && GRPCChannelStatus.DISCONNECT.equals(status)) {
+            AgentSO11Y.recordTracingContextCreate(forceSampling, true);
             return new IgnoredTracerContext();
         }
 
         int suffixIdx = operationName.lastIndexOf(".");
         if (suffixIdx > -1 && ignoreSuffixSet.contains(operationName.substring(suffixIdx))) {
+            AgentSO11Y.recordTracingContextCreate(forceSampling, true);
             context = new IgnoredTracerContext();
         } else {
             SamplingService samplingService = ServiceManager.INSTANCE.findService(SamplingService.class);
             if (forceSampling || samplingService.trySampling(operationName)) {
+                AgentSO11Y.recordTracingContextCreate(forceSampling, false);
                 context = new TracingContext(operationName, spanLimitWatcher);
             } else {
+                AgentSO11Y.recordTracingContextCreate(false, true);
+                AgentSO11Y.recordLeakedTracingContext(true);
                 context = new IgnoredTracerContext();
             }
         }

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/IgnoredTracerContext.java

@@ -23,6 +23,7 @@
 import org.apache.skywalking.apm.agent.core.context.trace.AbstractSpan;
 import org.apache.skywalking.apm.agent.core.context.trace.NoopSpan;
 import org.apache.skywalking.apm.agent.core.profile.ProfileStatusContext;
+import org.apache.skywalking.apm.agent.core.so11y.AgentSO11Y;
 
 /**
  * The <code>IgnoredTracerContext</code> represent a context should be ignored. So it just maintains the stack with an
@@ -116,6 +117,7 @@ public AbstractSpan activeSpan() {
     public boolean stopSpan(AbstractSpan span) {
         stackDepth--;
         if (stackDepth == 0) {
+            AgentSO11Y.recordTracingContextFinish(true);
             ListenerManager.notifyFinish(this);
         }
         return stackDepth == 0;

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/TracingContext.java

@@ -43,6 +43,7 @@
 import org.apache.skywalking.apm.agent.core.logging.api.LogManager;
 import org.apache.skywalking.apm.agent.core.profile.ProfileStatusContext;
 import org.apache.skywalking.apm.agent.core.profile.ProfileTaskExecutionService;
+import org.apache.skywalking.apm.agent.core.so11y.AgentSO11Y;
 import org.apache.skywalking.apm.util.StringUtil;
 
 import static org.apache.skywalking.apm.agent.core.conf.Config.Agent.CLUSTER;
@@ -460,7 +461,12 @@ private void finish() {
             }
 
             if (isFinishedInMainThread && (!isRunningInAsyncMode || asyncSpanCounter == 0)) {
-                TraceSegment finishedSegment = segment.finish(isLimitMechanismWorking());
+                boolean limitMechanismWorking = isLimitMechanismWorking();
+                if (limitMechanismWorking) {
+                    AgentSO11Y.recordLeakedTracingContext(false);
+                }
+                AgentSO11Y.recordTracingContextFinish(false);
+                TraceSegment finishedSegment = segment.finish(limitMechanismWorking);
                 TracingContext.ListenerManager.notifyFinish(finishedSegment);
                 running = false;
             }

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/plugin/AbstractClassEnhancePluginDefine.java

@@ -43,6 +43,10 @@
 public abstract class AbstractClassEnhancePluginDefine {
     private static final ILog LOGGER = LogManager.getLogger(AbstractClassEnhancePluginDefine.class);
 
+    /**
+     * plugin name defined in skywalking-plugin.def
+     */
+    private String pluginName;
     /**
      * New field name.
      */
@@ -199,4 +203,17 @@ public boolean isBootstrapInstrumentation() {
      * @return collections of {@link InstanceMethodsInterceptV2Point}
      */
     public abstract StaticMethodsInterceptV2Point[] getStaticMethodsInterceptV2Points();
+
+    /**
+     * plugin name should be set after create PluginDefine instance
+     *
+     * @param pluginName key defined in skywalking-plugin.def
+     */
+    protected void setPluginName(final String pluginName) {
+        this.pluginName = pluginName;
+    }
+
+    public String getPluginName() {
+        return pluginName;
+    }
 }

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/plugin/PluginBootstrap.java

@@ -65,6 +65,7 @@ public List<AbstractClassEnhancePluginDefine> loadPlugins() throws AgentPackageN
                 LOGGER.debug("loading plugin class {}.", pluginDefine.getDefineClass());
                 AbstractClassEnhancePluginDefine plugin = (AbstractClassEnhancePluginDefine) Class.forName(pluginDefine.getDefineClass(), true, AgentClassLoader
                     .getDefault()).newInstance();
+                plugin.setPluginName(pluginDefine.getName());
                 plugins.add(plugin);
             } catch (Throwable t) {
                 LOGGER.error(t, "load plugin [{}] failure.", pluginDefine.getDefineClass());

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/plugin/bootstrap/BootstrapInstrumentBoost.java

@@ -71,6 +71,9 @@ public class BootstrapInstrumentBoost {
         "org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.v2.InstanceMethodsAroundInterceptorV2",
         "org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.v2.StaticMethodsAroundInterceptorV2",
         "org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.v2.MethodInvocationContext",
+
+        //SO11Y
+        "org.apache.skywalking.apm.agent.core.so11y.bootstrap.BootstrapPluginSO11Y"
     };
 
     private static String INSTANCE_METHOD_DELEGATE_TEMPLATE = "org.apache.skywalking.apm.agent.core.plugin.bootstrap.template.InstanceMethodInterTemplate";
@@ -162,31 +165,39 @@ private static boolean prepareJREInstrumentation(PluginFinder pluginFinder,
                 for (InstanceMethodsInterceptPoint point : define.getInstanceMethodsInterceptPoints()) {
                     if (point.isOverrideArgs()) {
                         generateDelegator(
-                            classesTypeMap, typePool, INSTANCE_METHOD_WITH_OVERRIDE_ARGS_DELEGATE_TEMPLATE, point
-                                .getMethodsInterceptor());
+                            classesTypeMap, typePool, define.getPluginName(),
+                            INSTANCE_METHOD_WITH_OVERRIDE_ARGS_DELEGATE_TEMPLATE, point.getMethodsInterceptor()
+                        );
                     } else {
                         generateDelegator(
-                            classesTypeMap, typePool, INSTANCE_METHOD_DELEGATE_TEMPLATE, point.getMethodsInterceptor());
+                            classesTypeMap, typePool, define.getPluginName(),
+                            INSTANCE_METHOD_DELEGATE_TEMPLATE, point.getMethodsInterceptor()
+                        );
                     }
                 }
             }
 
             if (Objects.nonNull(define.getConstructorsInterceptPoints())) {
                 for (ConstructorInterceptPoint point : define.getConstructorsInterceptPoints()) {
                     generateDelegator(
-                        classesTypeMap, typePool, CONSTRUCTOR_DELEGATE_TEMPLATE, point.getConstructorInterceptor());
+                        classesTypeMap, typePool, define.getPluginName(),
+                        CONSTRUCTOR_DELEGATE_TEMPLATE, point.getConstructorInterceptor()
+                    );
                 }
             }
 
             if (Objects.nonNull(define.getStaticMethodsInterceptPoints())) {
                 for (StaticMethodsInterceptPoint point : define.getStaticMethodsInterceptPoints()) {
                     if (point.isOverrideArgs()) {
                         generateDelegator(
-                            classesTypeMap, typePool, STATIC_METHOD_WITH_OVERRIDE_ARGS_DELEGATE_TEMPLATE, point
-                                .getMethodsInterceptor());
+                            classesTypeMap, typePool, define.getPluginName(),
+                            STATIC_METHOD_WITH_OVERRIDE_ARGS_DELEGATE_TEMPLATE, point.getMethodsInterceptor()
+                        );
                     } else {
                         generateDelegator(
-                            classesTypeMap, typePool, STATIC_METHOD_DELEGATE_TEMPLATE, point.getMethodsInterceptor());
+                            classesTypeMap, typePool, define.getPluginName(),
+                            STATIC_METHOD_DELEGATE_TEMPLATE, point.getMethodsInterceptor()
+                        );
                     }
                 }
             }
@@ -202,14 +213,14 @@ private static boolean prepareJREInstrumentationV2(PluginFinder pluginFinder,
             if (Objects.nonNull(define.getInstanceMethodsInterceptV2Points())) {
                 for (InstanceMethodsInterceptV2Point point : define.getInstanceMethodsInterceptV2Points()) {
                     if (point.isOverrideArgs()) {
-                        generateDelegator(classesTypeMap, typePool,
-                                          INSTANCE_METHOD_V2_WITH_OVERRIDE_ARGS_DELEGATE_TEMPLATE,
-                                          point.getMethodsInterceptorV2()
+                        generateDelegator(
+                            classesTypeMap, typePool, define.getPluginName(),
+                            INSTANCE_METHOD_V2_WITH_OVERRIDE_ARGS_DELEGATE_TEMPLATE, point.getMethodsInterceptorV2()
                         );
                     } else {
                         generateDelegator(
-                            classesTypeMap, typePool, INSTANCE_METHOD_V2_DELEGATE_TEMPLATE,
-                            point.getMethodsInterceptorV2()
+                            classesTypeMap, typePool, define.getPluginName(),
+                            INSTANCE_METHOD_V2_DELEGATE_TEMPLATE, point.getMethodsInterceptorV2()
                         );
                     }
                 }
@@ -218,14 +229,14 @@ private static boolean prepareJREInstrumentationV2(PluginFinder pluginFinder,
             if (Objects.nonNull(define.getStaticMethodsInterceptV2Points())) {
                 for (StaticMethodsInterceptV2Point point : define.getStaticMethodsInterceptV2Points()) {
                     if (point.isOverrideArgs()) {
-                        generateDelegator(classesTypeMap, typePool,
-                                          STATIC_METHOD_V2_WITH_OVERRIDE_ARGS_DELEGATE_TEMPLATE,
-                                          point.getMethodsInterceptorV2()
+                        generateDelegator(
+                            classesTypeMap, typePool, define.getPluginName(),
+                            STATIC_METHOD_V2_WITH_OVERRIDE_ARGS_DELEGATE_TEMPLATE, point.getMethodsInterceptorV2()
                         );
                     } else {
                         generateDelegator(
-                            classesTypeMap, typePool, STATIC_METHOD_V2_DELEGATE_TEMPLATE,
-                            point.getMethodsInterceptorV2()
+                            classesTypeMap, typePool, define.getPluginName(),
+                            STATIC_METHOD_V2_DELEGATE_TEMPLATE, point.getMethodsInterceptorV2()
                         );
                     }
                 }
@@ -245,14 +256,16 @@ private static boolean prepareJREInstrumentationV2(PluginFinder pluginFinder,
      *                          pre-defined in SkyWalking agent core.
      */
     private static void generateDelegator(Map<String, byte[]> classesTypeMap, TypePool typePool,
-        String templateClassName, String methodsInterceptor) {
+        String pluginName, String templateClassName, String methodsInterceptor) {
         String internalInterceptorName = internalDelegate(methodsInterceptor);
         try {
             TypeDescription templateTypeDescription = typePool.describe(templateClassName).resolve();
 
             DynamicType.Unloaded interceptorType = new ByteBuddy().redefine(templateTypeDescription, ClassFileLocator.ForClassLoader
                 .of(BootstrapInstrumentBoost.class.getClassLoader()))
                                                                   .name(internalInterceptorName)
+                                                                  .field(named("PLUGIN_NAME"))
+                                                                  .value(pluginName)
                                                                   .field(named("TARGET_INTERCEPTOR"))
                                                                   .value(methodsInterceptor)
                                                                   .make();

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/plugin/bootstrap/template/ConstructorInterTemplate.java

@@ -25,6 +25,7 @@
 import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.BootstrapInterRuntimeAssist;
 import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.EnhancedInstance;
 import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstanceConstructorInterceptor;
+import org.apache.skywalking.apm.agent.core.so11y.bootstrap.BootstrapPluginSO11Y;
 
 /**
  * --------CLASS TEMPLATE---------
@@ -37,13 +38,21 @@
  * This class wouldn't be loaded in real env. This is a class template for dynamic class generation.
  */
 public class ConstructorInterTemplate {
+
+    /**
+     * This field is never set in the template, but has value in the runtime.
+     */
+    private static String PLUGIN_NAME;
     /**
      * This field is never set in the template, but has value in the runtime.
      */
     private static String TARGET_INTERCEPTOR;
 
     private static InstanceConstructorInterceptor INTERCEPTOR;
     private static IBootstrapLog LOGGER;
+    private static BootstrapPluginSO11Y PLUGIN_SO11Y;
+
+    private static final String INTERCEPTOR_TYPE = "constructor";
 
     /**
      * Intercept the target constructor.
@@ -53,6 +62,8 @@ public class ConstructorInterTemplate {
      */
     @RuntimeType
     public static void intercept(@This Object obj, @AllArguments Object[] allArguments) {
+        long interceptorTimeCost = 0L;
+        long startTime = System.nanoTime();
         try {
             prepare();
 
@@ -64,7 +75,10 @@ public static void intercept(@This Object obj, @AllArguments Object[] allArgumen
             INTERCEPTOR.onConstruct(targetObject, allArguments);
         } catch (Throwable t) {
             LOGGER.error("ConstructorInter failure.", t);
+            PLUGIN_SO11Y.recordInterceptorError(PLUGIN_NAME, INTERCEPTOR_TYPE);
         }
+        interceptorTimeCost += System.nanoTime() - startTime;
+        PLUGIN_SO11Y.recordInterceptorTimeCost(interceptorTimeCost);
     }
 
     /**
@@ -79,6 +93,7 @@ private static void prepare() {
                 if (logger != null) {
                     LOGGER = logger;
 
+                    PLUGIN_SO11Y = BootstrapInterRuntimeAssist.getSO11Y(loader);
                     INTERCEPTOR = BootstrapInterRuntimeAssist.createInterceptor(loader, TARGET_INTERCEPTOR, LOGGER);
                 }
             } else {