MongoDB monitoring showcase (#144)

Makefile.in

@@ -29,10 +29,10 @@ ES_IMAGE ?= docker.elastic.co/elasticsearch/elasticsearch-oss
 ES_IMAGE_TAG ?= 7.10.2
 
 SW_OAP_IMAGE ?= ghcr.io/apache/skywalking/oap
-SW_OAP_IMAGE_TAG ?= 8b2fe884152d7bb03e0a8ac0753fdc84df541ce9
+SW_OAP_IMAGE_TAG ?= e0068154c1253500720651dca5c80c3ae10ca87b
 
 SW_UI_IMAGE ?= ghcr.io/apache/skywalking/ui
-SW_UI_IMAGE_TAG ?= 8b2fe884152d7bb03e0a8ac0753fdc84df541ce9
+SW_UI_IMAGE_TAG ?= e0068154c1253500720651dca5c80c3ae10ca87b
 
 SW_CLI_IMAGE ?= ghcr.io/apache/skywalking-cli/skywalking-cli:0883266bfaa36612927b69e35781b64ea181758d
 SW_EVENT_EXPORTER_IMAGE ?= ghcr.io/apache/skywalking-kubernetes-event-exporter/skywalking-kubernetes-event-exporter:8a012a3f968cb139f817189afb9b3748841bba22

deploy/platform/docker/config/mongodb/otel-collector-config.yaml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+receivers:
+  prometheus:
+    config:
+     scrape_configs:
+       - job_name: 'mongodb-monitoring'
+         metrics_path: "/metrics"
+         scrape_interval: 10s
+         static_configs:
+           - targets: ['mongodb-exporter-1:9216','mongodb-exporter-2:9216']
+             labels:
+               cluster: replset
+
+processors:
+  batch:
+
+exporters:
+  ####################################################################################
+  # If you want to use otlp exporter please ensure that your OAP version is >= 9.2.0 #
+  ####################################################################################
+  otlp:
+    endpoint: oap:11800
+    tls:
+      insecure: true
+service:
+  pipelines:
+    metrics:
+      receivers:
+      - prometheus
+      processors:
+      - batch
+      exporters:
+      - otlp

deploy/platform/docker/config/mongodb/setup.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "Waiting for startup.."
+until mongosh --host mongodb-1:27017 --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 2)' &>/dev/null; do
+  printf '.'
+  sleep 1
+done
+until mongosh --host mongodb-2:27017 --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 2)' &>/dev/null; do
+  printf '.'
+  sleep 1
+done
+
+echo "Started.."
+
+mongosh --host mongodb-1:27017 <<EOF
+     var cfg = {
+      "_id": "rs1",
+      "protocolVersion": 1,
+      "members": [
+         {
+           "_id": 0,
+           "host": "mongodb-1:27017"
+         },
+         {
+           "_id": 1,
+           "host": "mongodb-2:27017"
+         }
+      ]
+     };
+     rs.initiate(cfg, { force: true });
+     rs.reconfig(cfg, { force: true });
+EOF

deploy/platform/docker/docker-compose.cluster.yaml

@@ -53,7 +53,7 @@ services:
       SW_CLUSTER_ZK_HOST_PORT: zookeeper:2181 # @feature: cluster; set up the cluster coordinator address
       SW_HEALTH_CHECKER: default # @feature: health-check;
       SW_OTEL_RECEIVER: default # @feature: vm; enable the OC receiver that receives the VM metrics
-      SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/* # @feature: vm; enable the OC rules that analyse the VM metrics
+      SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/*,mongodb/* # @feature: vm; enable the OC rules that analyse the VM metrics
       SW_STORAGE: elasticsearch
       SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200
       SW_TELEMETRY: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus

deploy/platform/docker/docker-compose.mongodb-monitor.yaml

@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# @feature: mongodb; set up mongodb_exporter and OpenTelemetry collector
+
+version: '2.1'
+
+services:
+  mongodb-1: &mongodb
+    image: mongo:6.0.8
+    networks:
+      - sw
+    command: mongod --replSet rs1 --shardsvr --port 27017
+    healthcheck:
+      test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/27017" ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+  mongodb-2: *mongodb
+  mongodb-repl-setup:
+    image: mongo:6.0.8
+    networks:
+      - sw
+    depends_on:
+      mongodb-1:
+        condition: service_healthy
+      mongodb-2:
+        condition: service_healthy
+    volumes:
+      - ./config/mongodb/setup.sh:/setup.sh
+    entrypoint: [ "bash", "/setup.sh" ]
+  mongodb-exporter-1:
+    image: percona/mongodb_exporter:0.39
+    expose:
+      - 9216
+    networks:
+      - sw
+    depends_on:
+      mongodb-1:
+        condition: service_healthy
+    command:
+      - '--mongodb.uri=mongodb://mongodb-1:27017'
+      - '--discovering-mode'
+      - '--compatible-mode'
+      - '--collector.dbstats'
+      - '--collector.diagnosticdata'
+      - '--collector.replicasetstatus'
+  mongodb-exporter-2:
+    image: percona/mongodb_exporter:0.39
+    expose:
+      - 9216
+    networks:
+      - sw
+    depends_on:
+      mongodb-2:
+        condition: service_healthy
+    command:
+      - '--mongodb.uri=mongodb://mongodb-2:27017'
+      - '--discovering-mode'
+      - '--compatible-mode'
+      - '--collector.dbstats'
+      - '--collector.diagnosticdata'
+      - '--collector.replicasetstatus'
+  otel-collector:
+    image: otel/opentelemetry-collector:0.72.0
+    networks:
+      - sw
+    command: [ "--config=/etc/otel-collector-config.yaml" ]
+    volumes:
+      - ./config/mongodb/otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    expose:
+      - 55678
+    depends_on:
+      mongodb-exporter-1:
+        condition: service_started
+      mongodb-exporter-2:
+        condition: service_started
+
+networks:
+  sw:

deploy/platform/docker/docker-compose.single-node.yaml

@@ -43,7 +43,7 @@ services:
     environment:
       SW_HEALTH_CHECKER: default # @feature: health-check;
       SW_OTEL_RECEIVER: default # @feature: vm; enable the OC receiver that receives the VM metrics
-      SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,mysql/*,postgresql/*,elasticsearch/*,rabbitmq/* # @feature: vm,mysql,postgresql,elasticsearch; enable the OC rules that analyse the metrics
+      SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,mysql/*,postgresql/*,elasticsearch/*,rabbitmq/*,mongodb/* # @feature: vm,mysql,postgresql,elasticsearch; enable the OC rules that analyse the metrics
       SW_STORAGE: elasticsearch
       SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200
       SW_TELEMETRY: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus

deploy/platform/kubernetes/Chart.yaml

@@ -55,6 +55,11 @@ dependencies:
     repository: https://charts.bitnami.com/bitnami
     condition: rabbitmq.enabled
 
+  - name: mongodb
+    version: 13.16.0
+    repository: https://charts.bitnami.com/bitnami
+    condition: mongodb.enabled
+
   - name: base
     version: 1.14.4
     repository: https://istio-release.storage.googleapis.com/charts

deploy/platform/kubernetes/Makefile

@@ -150,6 +150,12 @@ feature-rabbitmq-monitor:
 	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set rabbitmq.enabled=true)
 	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set opentelemetry.enabled=true)
 
+.PHONY: feature-mongodb-monitor
+feature-mongodb-monitor:
+	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.mongodbMonitor.enabled=true)
+	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set mongodb.enabled=true)
+	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set opentelemetry.enabled=true)
+
 .PHONY: feature-function
 feature-function:
 	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.function.enabled=true)

deploy/platform/kubernetes/Makefile.in

@@ -23,4 +23,4 @@ SAMPLE_SERVICES_NAMESPACE ?= sample-services
 RELEASE ?= demo
 AGENTLESS ?= false
 
-FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,apisix-monitor,promql,elasticsearch-monitor,rabbitmq-monitor
+FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,apisix-monitor,promql,elasticsearch-monitor,rabbitmq-monitor,mongodb-monitor

deploy/platform/kubernetes/templates/feature-mongodb/opentelemetry-config.yaml

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- define "opentelemetry-config-mongodb-monitor" }}
+{{- if .Values.features.mongodbMonitor.enabled }}
+
+- job_name: mongodb-monitoring
+  scrape_interval: 10s
+  kubernetes_sd_configs:
+    - role: pod
+  relabel_configs:
+    - source_labels: [ __meta_kubernetes_pod_controller_name, __meta_kubernetes_pod_container_name, __meta_kubernetes_pod_container_port_name ]
+      action: keep
+      regex: .+mongodb;metrics;metrics
+    - source_labels: [ ]
+      target_label: cluster
+      replacement: replset
+    - source_labels: [ __meta_kubernetes_pod_name ]
+      target_label: service_instance_id
+      regex: (.+)
+      replacement: $$1
+
+{{- end }}
+{{- end }}

deploy/platform/kubernetes/templates/otel-collector-config.yaml

@@ -37,6 +37,7 @@ data:
             {{- include "opentelemetry-config-vm" . | indent 12 }}
             {{- include "opentelemetry-config-elasticsearch-monitor" . | indent 12 }}
             {{- include "opentelemetry-config-rabbitmq-monitor" . | indent 12 }}
+            {{- include "opentelemetry-config-mongodb-monitor" . | indent 12 }}
 
     exporters:
       otlp:

deploy/platform/kubernetes/values.yaml

@@ -39,7 +39,7 @@ skywalking:
       # @feature: istiod-monitor; enable rules to analyze Istio control plane metrics
       # @feature: mysql; enable mysql rules to analyze MySQL metrics
       # @feature: apisix-monitor; enable APISIX rules to analyze APISIX metrics
-      SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,oap,k8s/*,istio-controlplane,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/*
+      SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,oap,k8s/*,istio-controlplane,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/*,mongodb/*
       SW_TELEMETRY: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus
       SW_ENVOY_METRIC_ALS_HTTP_ANALYSIS: "mx-mesh,persistence" # @feature: als; enable mesh analyzer (mx-mesh) to analyze ALS logs
       K8S_SERVICE_NAME_RULE: "mesh-svr::${service.metadata.name}"
@@ -225,6 +225,22 @@ rabbitmq:
     password: guest
     erlangCookie: skywalking-showcase
 
+mongodb:
+  enabled: false
+  architecture: replicaset
+  replicaCount: 2
+  auth:
+    enabled: false
+  persistence:
+    enabled: false
+  metrics:
+    enabled: true
+    compatibleMode: true
+    collector:
+      dbstats: true
+      diagnosticdata: true
+      replicasetstatus: true
+
 prometheus-elasticsearch-exporter:
   enabled: false
   fullnameOverride: prometheus-elasticsearch-exporter
@@ -287,6 +303,9 @@ features:
   rabbitmqMonitor:
     enabled: false
 
+  mongodbMonitor:
+    enabled: false
+
   mysqlMonitor:
     enabled: false
 

docs/readme.md

@@ -105,6 +105,7 @@ Currently, the features supported are:
 | `mysql-monitor`         | Start a MySQL server and load generator to execute the sample SQLs periodically, set up fluent bit to fetch slow logs and export to OAP, and export their metrics to SkyWalking.       |                                                                                                                                       |
 | `postgresql-monitor`    | Start a PostgreSQL server, and load generator to execute the sample SQLs periodically, set up fluent bit to fetch slow logs and export to OAP, and export their metrics to SkyWalking. |                                                                                                                                       |
 | `elasticsearch-monitor` | Deploy OpenTelemetry and export Elasticsearch monitoring metrics to SkyWalking for analysis and display on UI.                                                                         |                                                                                                                                       |
+| `mongodb-monitor`       | Deploy OpenTelemetry and export MongoDB monitoring metrics to SkyWalking for analysis and display on UI.                                                                               |                                                                                                                                       |
 | `apisix-monitor`        | Deploy OpenTelemetry and export APISIX metrics to SkyWalking for analysis and display on UI                                                                                            |                                                                                                                                       |
 | `mesh-with-agent`       | Deploy services with java agent in the service mesh environment.                                                                                                                       | Only support deployment in the Kubernetes environment, docker is not supported.                                                       |
 | `promql`                | Deploy a Grafana to use promql service and show SkyWalking UI on the Grafana.                                                                                                          | Feel free to modify the Grafana config when deploy your own environment.                                                              |