From 74eac659d85e03a4a6a52df17109d3259bef0f30 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Tue, 13 Jul 2021 12:15:25 -0700 Subject: [PATCH 01/21] libupdate (#61) * libupdate --- Gemfile.lock | 34 +++++++++++++++--------------- Makefile | 3 +-- VERSION | 2 +- docker/Dockerfile | 5 +---- docker/Gemfile | 2 +- docker/Gemfile.lock | 50 ++++++++++++++++++++++----------------------- docker/build.sh | 3 +-- 7 files changed, 47 insertions(+), 52 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index fe86ba3..9245ffd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-objects (1.1.5) + fluent-plugin-kubernetes-objects (1.1.6) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) @@ -9,29 +9,30 @@ PATH GEM remote: https://rubygems.org/ specs: - addressable (2.7.0) + addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) - concurrent-ruby (1.1.8) + concurrent-ruby (1.1.9) cool.io (1.7.1) crack (0.4.5) rexml - docile (1.3.5) + docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - ffi (1.14.2) + ffi (1.15.3) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - fluentd (1.12.1) + fluentd (1.13.2) bundler cool.io (>= 1.4.5, < 2.0.0) - http_parser.rb (>= 0.5.1, < 0.7.0) + http_parser.rb (>= 0.5.1, < 0.8.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) strptime (>= 0.2.2, < 1.0.0) tzinfo (>= 1.0, < 3.0) tzinfo-data (~> 1.0) + webrick (>= 1.4.2, < 1.8.0) yajl-ruby (~> 1.0) hashdiff (1.0.1) http (4.4.1) @@ -40,7 +41,7 @@ GEM http-form_data (~> 2.2) http-parser (~> 1.2.0) http-accept (1.7.0) - http-cookie (1.0.3) + http-cookie (1.0.4) domain_name (~> 0.5) http-form_data (2.3.0) http-parser (1.2.3) @@ -52,21 +53,21 @@ GEM rest-client (~> 2.0) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0225) + mime-types-data (3.2021.0704) minitest (5.14.4) msgpack (1.4.2) netrc (0.11.0) power_assert (2.0.0) public_suffix (4.0.6) - rake (13.0.3) + rake (13.0.6) recursive-open-struct (1.1.3) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.2.4) - serverengine (2.2.3) + rexml (3.2.5) + serverengine (2.2.4) sigdump (~> 0.2.2) sigdump (0.2.4) simplecov (0.21.2) @@ -74,9 +75,9 @@ GEM simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) - simplecov_json_formatter (0.1.2) + simplecov_json_formatter (0.1.3) strptime (0.2.5) - test-unit (3.4.0) + test-unit (3.4.4) power_assert tzinfo (2.0.4) concurrent-ruby (~> 1.0) @@ -85,10 +86,11 @@ GEM unf (0.1.4) unf_ext unf_ext (0.0.7.7) - webmock (3.12.0) + webmock (3.13.0) addressable (>= 2.3.6) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) + webrick (1.7.0) yajl-ruby (1.4.1) PLATFORMS @@ -104,4 +106,4 @@ DEPENDENCIES webmock (~> 3.5) BUNDLED WITH - 2.2.13 + 2.2.23 diff --git a/Makefile b/Makefile index 5550e44..ed73903 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ VERSION := $(shell sh -c 'cat VERSION') -NODEJS_VERSION := 14.15.1 clean_pkg: @rm -rf pkg/* docker/*.gem @@ -17,7 +16,7 @@ docker: build install-deps @cp pkg/fluent-plugin-*.gem docker @mkdir -p docker/licenses @cp -rp LICENSE docker/licenses/ - @docker build --no-cache --pull --build-arg VERSION=$(VERSION) --build-arg NODEJS_VERSION=$(NODEJS_VERSION) -t splunk/kube-objects:$(VERSION) ./docker + @docker build --no-cache --pull --build-arg VERSION=$(VERSION) -t splunk/kube-objects:$(VERSION) ./docker unit-test: @bundle exec rake test diff --git a/VERSION b/VERSION index e25d8d9..0664a8f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.5 +1.1.6 diff --git a/docker/Dockerfile b/docker/Dockerfile index ca14b29..1a51eb6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,6 @@ FROM registry.access.redhat.com/ubi8/ruby-27 ARG VERSION -ARG NODEJS_VERSION LABEL name="Splunk Connect for Kubernetes Objects container" \ maintainer="DataEdge@splunk.com" \ @@ -23,9 +22,7 @@ RUN dnf install -y jq COPY Gemfile* ./ RUN yum update -y \ - && npm install -g n \ - && yum remove -y nodejs \ - && n ${NODEJS_VERSION} \ + && yum remove -y nodejs npm \ && gem install bundler \ && gem unpack /tmp/*.gem --target gem \ && bundle install diff --git a/docker/Gemfile b/docker/Gemfile index df52003..e111507 100644 --- a/docker/Gemfile +++ b/docker/Gemfile @@ -4,7 +4,7 @@ source 'https://rubygems.org' # for kubernetes log collection agent # List all required gems here and install via bundler to resolve dependencies gem "fluentd", "= 1.11.5" -gem "fluent-plugin-prometheus", "= 1.8.5" +gem "fluent-plugin-prometheus", ">= 2.0" gem "fluent-plugin-record-modifier", "= 2.1.0" gem "fluent-plugin-kubernetes_metadata_filter", "=2.5.3" gem "fluent-plugin-jq", "= 0.5.1" diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index 7b85ac3..c32ab0e 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-objects (1.1.5) + fluent-plugin-kubernetes-objects (1.1.6) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) @@ -9,26 +9,26 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (6.1.3) - activesupport (= 6.1.3) - activesupport (6.1.3) + activemodel (6.1.4) + activesupport (= 6.1.4) + activesupport (6.1.4) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) - addressable (2.7.0) + addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) aes_key_wrap (1.1.0) attr_required (1.0.1) - bigdecimal (3.0.0) - bindata (2.4.8) - concurrent-ruby (1.1.8) - connection_pool (2.2.3) + bigdecimal (3.0.2) + bindata (2.4.10) + concurrent-ruby (1.1.9) + connection_pool (2.2.5) cool.io (1.7.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - ffi (1.14.2) + ffi (1.15.3) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake @@ -39,17 +39,17 @@ GEM fluentd (>= 0.14.0, < 1.12) kubeclient (< 5) lru_redux - fluent-plugin-prometheus (1.8.5) + fluent-plugin-prometheus (2.0.1) fluentd (>= 1.9.1, < 2) - prometheus-client (< 0.10) + prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) - fluent-plugin-splunk-hec (1.2.5) + fluent-plugin-splunk-hec (1.2.7) fluentd (>= 1.4) multi_json (~> 1.13) net-http-persistent (~> 3.1) openid_connect (~> 1.1.8) - prometheus-client (< 0.10.0) + prometheus-client (>= 2.1.0) fluentd (1.11.5) cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.7.0) @@ -66,14 +66,14 @@ GEM http-form_data (~> 2.2) http-parser (~> 1.2.0) http-accept (1.7.0) - http-cookie (1.0.3) + http-cookie (1.0.4) domain_name (~> 0.5) http-form_data (2.3.0) http-parser (1.2.3) ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.5.3) httpclient (2.8.3) - i18n (1.8.9) + i18n (1.8.10) concurrent-ruby (~> 1.0) json-jwt (1.13.0) activesupport (>= 4.2) @@ -88,8 +88,8 @@ GEM mini_mime (>= 0.1.1) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0225) - mini_mime (1.0.2) + mime-types-data (3.2021.0704) + mini_mime (1.1.0) minitest (5.14.4) msgpack (1.4.2) multi_json (1.15.0) @@ -107,25 +107,23 @@ GEM validate_email validate_url webfinger (>= 1.0.1) - prometheus-client (0.9.0) - quantile (~> 0.2.1) + prometheus-client (2.1.0) public_suffix (4.0.6) - quantile (0.2.1) rack (2.2.3) - rack-oauth2 (1.16.0) + rack-oauth2 (1.17.0) activesupport attr_required httpclient json-jwt (>= 1.11.0) rack (>= 2.1.0) - rake (13.0.3) + rake (13.0.6) recursive-open-struct (1.1.3) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - serverengine (2.2.3) + serverengine (2.2.4) sigdump (~> 0.2.2) sigdump (0.2.4) strptime (0.2.5) @@ -160,9 +158,9 @@ DEPENDENCIES fluent-plugin-jq (= 0.5.1) fluent-plugin-kubernetes-objects! fluent-plugin-kubernetes_metadata_filter (= 2.5.3) - fluent-plugin-prometheus (= 1.8.5) + fluent-plugin-prometheus (>= 2.0) fluent-plugin-record-modifier (= 2.1.0) - fluent-plugin-splunk-hec (>= 1.2.3) + fluent-plugin-splunk-hec (>= 1.2.5) fluentd (= 1.11.5) http_parser.rb (>= 0.5.3) kubeclient (>= 4.6.0) diff --git a/docker/build.sh b/docker/build.sh index d43c9b1..f7bbe84 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -e TAG=$1 -NODEJS_VERSION=14.15.1 # Install dependecies gem install bundler @@ -17,4 +16,4 @@ VERSION=`cat VERSION` echo "Copying licenses to be included in the docker image..." mkdir -p docker/licenses cp -rp LICENSE docker/licenses/ -docker build --no-cache --pull --build-arg VERSION=$VERSION --build-arg NODEJS_VERSION=$NODEJS_VERSION -t splunk/kube-objects:$TAG ./docker +docker build --no-cache --pull --build-arg VERSION=$VERSION -t splunk/kube-objects:$TAG ./docker From 0a9215d67f4f9e422d44fd551bbdd1cf50d02008 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Thu, 15 Jul 2021 11:49:09 -0700 Subject: [PATCH 02/21] add description for useful objects (#62) --- objects/daemon_sets.md | 22 +++++++++++++++++++ objects/deployments.md | 27 +++++++++++++++++++++++ objects/namespaces.md | 10 +++++++++ objects/nodes.md | 21 ++++++++++++++++++ objects/persistent_volume_claims.md | 13 +++++++++++ objects/persistent_volumes.md | 13 +++++++++++ objects/pods.md | 34 +++++++++++++++++++++++++++++ objects/replica_sets.md | 26 ++++++++++++++++++++++ objects/resource_quotas.md | 9 ++++++++ objects/service_accounts.md | 9 ++++++++ objects/services.md | 17 +++++++++++++++ objects/stateful_sets.md | 26 ++++++++++++++++++++++ 12 files changed, 227 insertions(+) create mode 100644 objects/daemon_sets.md create mode 100644 objects/deployments.md create mode 100644 objects/namespaces.md create mode 100644 objects/nodes.md create mode 100644 objects/persistent_volume_claims.md create mode 100644 objects/persistent_volumes.md create mode 100644 objects/pods.md create mode 100644 objects/replica_sets.md create mode 100644 objects/resource_quotas.md create mode 100644 objects/service_accounts.md create mode 100644 objects/services.md create mode 100644 objects/stateful_sets.md diff --git a/objects/daemon_sets.md b/objects/daemon_sets.md new file mode 100644 index 0000000..7bfd358 --- /dev/null +++ b/objects/daemon_sets.md @@ -0,0 +1,22 @@ +# Kubernetes Daemon_Sets Object +### SourceType: kube:object:daemon_sets + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#daemonset-v1-apps + +Field | Description | Example +----- | ----------- | ------- +metadata.name | Daemon Set name | sck-splunk-kubernetes-metrics +spec.template.spec.containers{}.image | Docker image name | docker.io/splunk/k8s-metrics:1.1.4 +spec.template.spec.containers{}.name | Name of the container | splunk-fluentd-k8s-metrics +spec.template.spec.containers{}.resources.limits.cpu | maximum amount of compute resources allowed | 200m +spec.template.spec.containers{}.resources.limits.memory | maximum amount of compute resources allowed | 300Mi +spec.template.spec.containers{}.resources.requests.cpu | minimum amount of compute resources required | 200m +spec.template.spec.containers{}.resources.requests.memory | minimum amount of compute resources required |300Mi +spec.template.spec.containers{}.volumeMounts{}.mountPath | Path within the container at which the volume should be mounted | /fluentd/etc +spec.template.spec.containers{}.volumeMounts{}.name | This must match the Name of a Volume | conf-configmap +spec.template.spec.containers{}.volumeMounts{}.readOnly | Mounted read-only if true, read-write otherwise | TRUE +spec.template.spec.serviceAccountName | name of the ServiceAccount to use to run this pod | splunk-kubernetes-metrics +status.currentNumberScheduled | number of nodes that are running at least 1 daemon pod and are supposed to run the daemon pod | 1 +status.desiredNumberScheduled | number of nodes that should be running the daemon pod | 1 +status.numberAvailable | number of nodes that should be running the daemon pod and have one or more of the daemon pod running and available | 1 +status.numberReady | number of nodes that should be running the daemon pod and have one or more of the daemon pod running and ready | 1 \ No newline at end of file diff --git a/objects/deployments.md b/objects/deployments.md new file mode 100644 index 0000000..33a9f32 --- /dev/null +++ b/objects/deployments.md @@ -0,0 +1,27 @@ +# Kubernetes Deployments Object +### SourceType: kube:object:deployments + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#deployment-v1-apps + +Field | Description | Example +----- | ----------- | ------- +metadata.name | Deployment name | cartservice +spec.replicas | Number of desired pods | 1 +spec.strategy.type | Type of deployment | RollingUpdate +spec.template.spec.containers{}.image | Docker image name | gcr.io/google-samples/microservices-demo/cartservice:v0.2.2 +spec.template.spec.containers{}.name | Name of the container | server +spec.template.spec.containers{}.livenessProbe.* | Periodic probe of container liveness +spec.template.spec.containers{}.ports{}.containerPort | Number of port to expose on the pod's IP address | 7070 +spec.template.spec.containers{}.ports{}.protocol | TCP Protocol for port | TCP +spec.template.spec.containers{}.readinessProbe.* | Periodic probe of container service readiness +spec.template.spec.containers{}.resources.limits.cpu | maximum amount of compute resources allowed | 200m +spec.template.spec.containers{}.resources.limits.memory | maximum amount of compute resources allowed | 300Mi +spec.template.spec.containers{}.resources.requests.cpu | minimum amount of compute resources required | 200m +spec.template.spec.containers{}.resources.requests.memory | minimum amount of compute resources required |300Mi +spec.template.spec.containers{}.volumeMounts{}.mountPath | Path within the container at which the volume should be mounted | /fluentd/etc +spec.template.spec.containers{}.volumeMounts{}.name | This must match the Name of a Volume | conf-configmap +spec.template.spec.containers{}.volumeMounts{}.readOnly | Mounted read-only if true, read-write otherwise | TRUE +spec.template.spec.serviceAccountName | name of the ServiceAccount to use to run this pod | default +status.availableReplicas | number of available pods | 1 +status.conditions{}.message ReplicaSet | A human readable message indicating details about the transition | "cartservice-67b89ffc69" has successfully progressed +status.readyReplicas | number of ready pods targeted by this deployment | 1 \ No newline at end of file diff --git a/objects/namespaces.md b/objects/namespaces.md new file mode 100644 index 0000000..cc081d5 --- /dev/null +++ b/objects/namespaces.md @@ -0,0 +1,10 @@ +# Kubernetes Namespaces Object +### SourceType: kube:object:namespaces + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#namespace-v1-core + +Field | Description | Example +----- | ----------- | ------- +metadata.name | Namespace name | splunk +metadata.creationTimestamp | Time the namespace was created | 2020-06-04T14:36:29Z +status.phase | current lifecycle phase of the namespace | Active diff --git a/objects/nodes.md b/objects/nodes.md new file mode 100644 index 0000000..f861f38 --- /dev/null +++ b/objects/nodes.md @@ -0,0 +1,21 @@ +# Kubernetes Nodes Object +### SourceType: kube:object:nodes + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#node-v1-core + +Field | Description | Example +----- | ----------- | ------- +metadata.name | Node Name | ip-192-168-16-232.ec2.internal +status.addresses{}.address | List of addresses for the node | 192.168.16.232 +status.addresses{}.type | Address types for the node that align to the addresses | InternalIP +status.allocatable.* | Resources of a node that are available for scheduling | 1930m, 2886616Ki +status.capacity.* | Total resources of a node | 2, 20959212Ki +status.conditions{}.message | Human readable message indicating details about last transition | kubelet has sufficient memory available +status.nodeInfo.architecture | Architecture reported by the node | amd64 +status.nodeInfo.containerRuntimeVersion | ContainerRuntime Version reported by the node | docker://18.9.9 +status.nodeInfo.kernelVersion | Kernel Version reported by the node | 4.14.177-139.253.amzn2.x86_64 +status.nodeInfo.kubeProxyVersion | KubeProxy Version reported by the node | v1.15.11-eks-af3caf +status.nodeInfo.kubeletVersion | Kubelet Version reported by the node | v1.15.11-eks-af3caf +status.nodeInfo.operatingSystem | The Operating System reported by the node | linux +status.nodeInfo.osImage | OS Image reported by the node | Amazon Linux 2 + diff --git a/objects/persistent_volume_claims.md b/objects/persistent_volume_claims.md new file mode 100644 index 0000000..c87854b --- /dev/null +++ b/objects/persistent_volume_claims.md @@ -0,0 +1,13 @@ +# Kubernetes Persistent_Volume_Claims Object +### SourceType: kube:object:persistent_volume_claims + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#persistentvolumeclaim-v1-core + +Field | Description | Example +----- | ----------- | ------- +metadata.name | Persistent Volume Claim Name | pvc-var-splunk-dev-standalone-0 +spec.resources.requests.storage | Minimum amount of compute resources required | 100Gi +spec.storageClassName | Name of the StorageClass required by the claim | microk8s-hostpath +spec.volumeName | Binding reference to the PersistentVolume backing this claim | pvc-4073c97f-0b85-418c-9b64-ed5154875ec8 +status.capacity.storage | Actual resources of the underlying volume | 100Gi +status.phase | Indicates if a volume is available, bound to a claim, or released by a claim | Bound \ No newline at end of file diff --git a/objects/persistent_volumes.md b/objects/persistent_volumes.md new file mode 100644 index 0000000..d04e349 --- /dev/null +++ b/objects/persistent_volumes.md @@ -0,0 +1,13 @@ +# Kubernetes Persistent_Volumes Object +### SourceType: kube:object:persistent_volumes + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#persistentvolume-v1-core + +Field | Description | Example +----- | ----------- | ------- +metadata.name | Persistent Volume Name | pvc-ff21aff0-b715-4d6a-acec-f855ce2c7535 +spec.capacity.storage | persistent volume's resources and capacity | 50Gi +spec.*type* | different types of persistent volumes that are defined +spec.persistentVolumeReclaimPolicy | What happens to a persistent volume when released from its claim | Delete +spec.storageClassName | Name of StorageClass to which this persistent volume belongs | microk8s-hostpath +status.phase | Indicates if a volume is available, bound to a claim, or released by a claim | Bound \ No newline at end of file diff --git a/objects/pods.md b/objects/pods.md new file mode 100644 index 0000000..237f59a --- /dev/null +++ b/objects/pods.md @@ -0,0 +1,34 @@ +# Kubernetes Pods Object +### SourceType: kube:object:pods + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#pod-v1-core + +Field | Description | Example +----- | ----------- | ------- +metadata.name | Pod Name | adminfrontend-c8b48c574-dsl4q +spec.containers{}.image | Docker image name | docker.io/splunk/k8s-metrics:1.1.3 +spec.containers{}.name | Container Name | splunk-fluentd-k8s-metrics +spec.containers{}.ports{}.containerPort | Number of port to expose on the pod's IP address | 443 +spec.containers{}.volumeMounts{}.mountPath | Path within the container at which the volume should be mounted | /var/run/secrets/kubernetes.io/serviceaccount +spec.containers{}.volumeMounts{}.name | Volume's name that has been mounted | default-token-njbcn +spec.volumes{}.hostPath.path | Path of the directory on the host | /var/lib/docker/overlay2 +spec.volumes{}.name | Volume's name | default-token-njbcn +spec.containers{}.livenessProbe.* | Information on periodic probe of container liveness +spec.containers{}.readinessProbe.* | Information on periodic probe of container readiness +spec.containers{}.securityContext.privileged | Run container in privileged mode | TRUE +spec.containers{}.securityContext.allowPrivilegeEscalation | Whether a process can gain more privileges than its parent process | TRUE +spec.securityContext.runAsGroup | The GID to run the entrypoint of the container process | 4188 +spec.securityContext.runAsUser | The UID to run the entrypoint of the container process | 4188 +spec.securityContext.runAsNonRoot | Indicates that the container must run as a non-root user | 0 +spec.serviceAccountName | ServiceAccount to use to run this pod | default +status.containerStatuses{}.image | The image the container is running | k8tan/admin_frontend:latest +status.containerStatuses{}.name | Container name that is running | adminfrontend +status.containerStatuses{}.ready | Specifies whether the container has passed its readiness probe | TRUE +status.containerStatuses{}.restartCount | The number of times the container has been restarted | 1 +status.containerStatuses{}.state.running.startedAt | Time at which the container was last (re-)started | 2020-07-28T16:41:29Z +status.hostIP | IP address of the host to which the pod is assigned | 192.168.16.232 +status.message | Human readable message indicating details about why the pod is in this condition | Pod The node had condition: [MemoryPressure]. +status.phase | High-level summary of where the Pod is in its lifecycle | Running +status.podIP | IP address allocated to the pod | 192.168.25.49 +status.reason | Brief CamelCase message indicating details about why the pod is in this state | Evicted + diff --git a/objects/replica_sets.md b/objects/replica_sets.md new file mode 100644 index 0000000..2bfeb73 --- /dev/null +++ b/objects/replica_sets.md @@ -0,0 +1,26 @@ +# Kubernetes Replica_Sets Object +### SourceType: kube:object:replica_sets + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#replicaset-v1-apps + +Field | Description | Example +----- | ----------- | ------- +metadata.name | Replica Set name | cartservice +spec.replicas | Number of desired pods | 3 +spec.template.spec.containers{}.image | Docker image name | 602321143452.dkr.ecr.us-east-1.amazonaws.com/eks/coredns:v1.8.0-eksbuild.1 +spec.template.spec.containers{}.name | Name of the container | server +spec.template.spec.containers{}.livenessProbe.* | Periodic probe of container liveness +spec.template.spec.containers{}.ports{}.containerPort | Number of port to expose on the pod's IP address | 7070 +spec.template.spec.containers{}.ports{}.protocol | TCP Protocol for port | TCP +spec.template.spec.containers{}.readinessProbe.* | Periodic probe of container service readiness +spec.template.spec.containers{}.resources.limits.cpu | maximum amount of compute resources allowed | 200m +spec.template.spec.containers{}.resources.limits.memory | maximum amount of compute resources allowed | 300Mi +spec.template.spec.containers{}.resources.requests.cpu | minimum amount of compute resources required | 200m +spec.template.spec.containers{}.resources.requests.memory | minimum amount of compute resources required |300Mi +spec.template.spec.containers{}.volumeMounts{}.mountPath | Path within the container at which the volume should be mounted | /fluentd/etc +spec.template.spec.containers{}.volumeMounts{}.name | This must match the Name of a Volume | conf-configmap +spec.template.spec.containers{}.volumeMounts{}.readOnly | Mounted read-only if true, read-write otherwise | TRUE +spec.template.spec.serviceAccountName | name of the ServiceAccount to use to run this pod | default +status.availableReplicas | number of available pods | 2 +status.readyReplicas | number of ready pods targeted by this deployment | 2 +status.replicas | number of non-terminated pods targeted by this deployment | 2 \ No newline at end of file diff --git a/objects/resource_quotas.md b/objects/resource_quotas.md new file mode 100644 index 0000000..f8702a0 --- /dev/null +++ b/objects/resource_quotas.md @@ -0,0 +1,9 @@ +# Kubernetes Resource_Quotas Object +### SourceType: kube:object:resource_quotas + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcequota-v1-core + +Field | Description | Example +----- | ----------- | ------- +status.used.pods | Total usage of the resource in the namespace | 2 +status.used.services | Total usage of the resource in the namespace | 0 \ No newline at end of file diff --git a/objects/service_accounts.md b/objects/service_accounts.md new file mode 100644 index 0000000..717a62c --- /dev/null +++ b/objects/service_accounts.md @@ -0,0 +1,9 @@ +# Kubernetes Service_Accounts Object +### SourceType: kube:object:service_accounts + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#serviceaccount-v1-core + +Field | Description | Example +----- | ----------- | ------- +metadata.name | Service account name | splunk-operator +metadata.creationTimestamp | Time the account was created | 2020-06-04T14:36:29Z diff --git a/objects/services.md b/objects/services.md new file mode 100644 index 0000000..18eccf4 --- /dev/null +++ b/objects/services.md @@ -0,0 +1,17 @@ +# Kubernetes Services Object +### SourceType: kube:object:services + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#service-v1-core + +Field | Description | Example +----- | ----------- | ------- +metadata.name | Service name | kubernetes +spec.clusterIP | clusterIP is the IP address of the service | 10.100.0.1 +spec.externalName | external reference that discovery mechanisms will return as an alias for this service | frontend.company.com +spec.externalTrafficPolicy | externalTrafficPolicy denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints | Cluster +spec.ports{}.nodePort | The port on each node on which this service is exposed | 30245 +spec.ports{}.name | The name of this port within the service | https +spec.ports{}.port | The port that will be exposed by this service | 443 +spec.ports{}.protocol | The IP protocol for this port | TCP +spec.selector.name | Route service traffic to pods with label keys and values matching this selector | Webfrontend +spec.type| How the Service is exposed | ClusterIP diff --git a/objects/stateful_sets.md b/objects/stateful_sets.md new file mode 100644 index 0000000..c5725ef --- /dev/null +++ b/objects/stateful_sets.md @@ -0,0 +1,26 @@ +# Kubernetes Stateful_Sets Object +### SourceType: kube:object:stateful_sets + +All definitions can be found at https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#statefulset-v1-apps + +Field | Description | Example +----- | ----------- | ------- +metadata.name | StatefulSet name | cartservice +spec.replicas | Number of desired pods | 3 +spec.serviceName | The name of the service that governs this StatefulSet | splunk-splunk-dev-monitoring-console-headless +spec.template.spec.containers{}.image | Docker image name | 602321143452.dkr.ecr.us-east-1.amazonaws.com/eks/coredns:v1.8.0-eksbuild.1 +spec.template.spec.containers{}.name | Name of the container | server +spec.template.spec.containers{}.livenessProbe.* | Periodic probe of container liveness +spec.template.spec.containers{}.ports{}.containerPort | Number of port to expose on the pod's IP address | 7070 +spec.template.spec.containers{}.ports{}.protocol | TCP Protocol for port | TCP +spec.template.spec.containers{}.readinessProbe.* | Periodic probe of container service readiness +spec.template.spec.containers{}.resources.limits.cpu | maximum amount of compute resources allowed | 200m +spec.template.spec.containers{}.resources.limits.memory | maximum amount of compute resources allowed | 300Mi +spec.template.spec.containers{}.resources.requests.cpu | minimum amount of compute resources required | 200m +spec.template.spec.containers{}.resources.requests.memory | minimum amount of compute resources required |300Mi +spec.template.spec.containers{}.volumeMounts{}.mountPath | Path within the container at which the volume should be mounted | /fluentd/etc +spec.template.spec.containers{}.volumeMounts{}.name | This must match the Name of a Volume | conf-configmap +spec.template.spec.containers{}.volumeMounts{}.readOnly | Mounted read-only if true, read-write otherwise | TRUE +status.currentReplicas | Number of Pods created by the StatefulSet controller from the StatefulSet version indicated by currentRevision | 2 +status.readyReplicas | number of ready pods targeted by this deployment | 2 +status.replicas | number of non-terminated pods targeted by this deployment | 2 \ No newline at end of file From 0e5fa66cbe3f22db24de22caf0bf47f058074c17 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Mon, 9 Aug 2021 15:38:32 -0700 Subject: [PATCH 03/21] handle connection error (#63) --- Gemfile.lock | 4 ++-- VERSION | 2 +- lib/fluent/plugin/in_kubernetes_objects.rb | 20 ++++++++++++-------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9245ffd..de4a52f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-objects (1.1.6) + fluent-plugin-kubernetes-objects (1.1.7) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) @@ -106,4 +106,4 @@ DEPENDENCIES webmock (~> 3.5) BUNDLED WITH - 2.2.23 + 2.2.24 diff --git a/VERSION b/VERSION index 0664a8f..2bf1ca5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.6 +1.1.7 diff --git a/lib/fluent/plugin/in_kubernetes_objects.rb b/lib/fluent/plugin/in_kubernetes_objects.rb index 850b389..f57a2b0 100644 --- a/lib/fluent/plugin/in_kubernetes_objects.rb +++ b/lib/fluent/plugin/in_kubernetes_objects.rb @@ -245,15 +245,19 @@ def create_watcher_thread(conf) while thread_current_running? @client.public_send("watch_#{resource_name}", options).tap do |watcher| tag = generate_tag "#{resource_name}" - watcher.each do |entity| - begin - entity = JSON.parse(entity) - router.emit tag, Fluent::Engine.now, entity - options[:resource_version] = entity['object']['metadata']['resourceVersion'] - @storage.put resource_name, entity['object']['metadata']['resourceVersion'] - rescue => e - log.info "Got exception #{e} parsing entity #{entity}. Resetting watcher." + begin + watcher.each do |entity| + begin + entity = JSON.parse(entity) + router.emit tag, Fluent::Engine.now, entity + options[:resource_version] = entity['object']['metadata']['resourceVersion'] + @storage.put resource_name, entity['object']['metadata']['resourceVersion'] + rescue => e + log.info "Got exception #{e} parsing entity #{entity}. Resetting watcher." + end end + rescue => e + log.info "Got exception #{e}. Resetting watcher." end end end From 36314f5d2b3e5038df402c74949957f07ead310a Mon Sep 17 00:00:00 2001 From: Mei Wang Date: Fri, 27 Aug 2021 18:50:56 -0700 Subject: [PATCH 04/21] circleci migration with github action (#65) * circleci migration with github action --- .github/workflows/ci_build_test.yaml | 165 +++++++++++++++++++++++++++ .gitignore | 3 +- ci_scripts/deploy_connector.sh | 29 +++++ ci_scripts/install_dep.sh | 5 + ci_scripts/k8s-splunk.yml | 31 +++++ 5 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci_build_test.yaml create mode 100755 ci_scripts/deploy_connector.sh create mode 100755 ci_scripts/install_dep.sh create mode 100644 ci_scripts/k8s-splunk.yml diff --git a/.github/workflows/ci_build_test.yaml b/.github/workflows/ci_build_test.yaml new file mode 100644 index 0000000..46b004c --- /dev/null +++ b/.github/workflows/ci_build_test.yaml @@ -0,0 +1,165 @@ +name: CI Build Test + +on: + pull_request: + branches-ignore: + - /^release\/.*/ + - main + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup Ruby and install gems + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + ruby-version: 2.6 + + - name: Install dependencies + run: | + sudo ci_scripts/install_dep.sh + + - name: Builder + run: | + rake build -t -v + cp -R pkg /tmp + + - name: Cache pkg + uses: actions/cache@v1 + with: + path: /tmp + key: ${{ runner.os }}-build + + unit-test: + runs-on: ubuntu-latest + needs: + - build + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install dependencies + run: | + sudo ci_scripts/install_dep.sh + + - uses: actions/cache@v2 + with: + path: /tmp + key: ${{ runner.os }}-build + + - name: Run unit tests + run: | + bundle exec rake test -t -v + + func-test: + needs: + - unit-test + runs-on: ubuntu-20.04 + env: + CI_SPLUNK_PORT: 8089 + CI_SPLUNK_USERNAME: admin + CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959 + CI_SPLUNK_PASSWORD: helloworld + CI_INDEX_EVENTS: ci_events + CI_INDEX_OBJECTS: ci_objects + CI_INDEX_METRICS: ci_metrics + KUBERNETES_VERSION: v1.15.2 + MINIKUBE_VERSION: v1.21.0 + GITHUB_ACTIONS: true + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Build kube-objects image + run: | + sed -i 's/VERSION=$VERSION -t splunk\/kube-objects:$TAG .\/docker/VERSION=$VERSION -t kube-objects:$TAG .\/docker/' docker/build.sh + sudo docker/build.sh $GITHUB_RUN_ID + docker image ls + + - name: Setup Minikube + run: | + # Install Kubectl + curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl + chmod +x kubectl + sudo mv kubectl /usr/local/bin/ + mkdir -p ${HOME}/.kube + touch ${HOME}/.kube/config + # Install Minikube + curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64 + chmod +x minikube + sudo mv minikube /usr/local/bin/ + # Start Minikube and Wait + minikube start --driver=docker --container-runtime=docker --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --no-vtx-check + export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' + until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do + sleep 1; + done + + - name: Install Splunk + run: | + # Wait until minikube is ready + kubectl apply -f https://docs.projectcalico.org/v3.14/manifests/calico.yaml + export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' + until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do + echo "wait for minikube ready ..." + sleep 1; + done + kubectl get nodes + # Install Splunk on minikube + kubectl apply -f ci_scripts/k8s-splunk.yml + # Wait until splunk is ready + until kubectl logs splunk --tail=2 | grep -q 'Ansible playbook complete'; do + sleep 1; + done + export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}}) + # Setup Indexes + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_EVENTS -d datatype=event + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_OBJECTS -d datatype=event + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_METRICS -d datatype=metric + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=default-events -d datatype=event + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=ns-anno -d datatype=event + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=pod-anno -d datatype=event + # Enable HEC services + curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable + # Create new HEC token + curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=a6b5e77f-d5f6-415a-bd43-930cecb12959&disabled=0&index=default-events&indexes=default-events,$CI_INDEX_METRICS,$CI_INDEX_OBJECTS,$CI_INDEX_EVENTS,ns-anno,pod-anno" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http + # Restart Splunk + curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/server/control/restart -X POST + + - name: Deploy k8s connector + run: | + export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}}) + ci_scripts/deploy_connector.sh + + - name: Deploy log generator + run: | + cd /opt/splunk-connect-for-kubernetes + kubectl apply -f test/test_setup.yaml + sleep 60 + + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Run functional tests + run: | + echo "check the pods" + kubectl get pods -A + cd /opt/splunk-connect-for-kubernetes + kubectl get nodes + export PYTHONWARNINGS="ignore:Unverified HTTPS request" + export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}}) + cd test + pip install --upgrade pip + pip install -r requirements.txt + echo "Running functional tests....." + python -m pytest \ + --splunkd-url https://$CI_SPLUNK_HOST:8089 \ + --splunk-user admin \ + --splunk-password $CI_SPLUNK_PASSWORD \ + -p no:warnings -s diff --git a/.gitignore b/.gitignore index 1fe9aaf..a9afffc 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ docker/licenses/ docker/gem/ docker/gems/ -docker/.bundle/ \ No newline at end of file +docker/.bundle/ +.idea/ diff --git a/ci_scripts/deploy_connector.sh b/ci_scripts/deploy_connector.sh new file mode 100755 index 0000000..7b40512 --- /dev/null +++ b/ci_scripts/deploy_connector.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -e + +#Make sure to check and clean previously failed deployment +echo "Checking if previous deployment exist..." +if [ "`helm ls --short`" == "" ]; then + echo "Nothing to clean, ready for deployment" +else + helm delete $(helm ls --short) +fi + +# Clone splunk-connect-for-kubernetes repo +cd /opt +git clone https://github.com/splunk/splunk-connect-for-kubernetes.git +cd splunk-connect-for-kubernetes + +sed -i 's/name: splunk\/kube-objects/repository: kube-objects/' ci_scripts/sck_values.yml +cat ci_scripts/sck_values.yml + +echo "Deploying k8s-connect with latest changes" +helm install ci-sck --set global.splunk.hec.token=$CI_SPLUNK_HEC_TOKEN \ +--set global.splunk.hec.host=$CI_SPLUNK_HOST \ +--set kubelet.serviceMonitor.https=true \ +--set splunk-kubernetes-objects.image.pullPolicy=IfNotPresent \ +-f ci_scripts/sck_values.yml helm-chart/splunk-connect-for-kubernetes +#wait for deployment to finish +until kubectl get pod | grep Running | [[ $(wc -l) == 4 ]]; do + sleep 1; +done \ No newline at end of file diff --git a/ci_scripts/install_dep.sh b/ci_scripts/install_dep.sh new file mode 100755 index 0000000..7c4d131 --- /dev/null +++ b/ci_scripts/install_dep.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -e +gem install bundler +bundle update --bundler +bundle install \ No newline at end of file diff --git a/ci_scripts/k8s-splunk.yml b/ci_scripts/k8s-splunk.yml new file mode 100644 index 0000000..543c6ca --- /dev/null +++ b/ci_scripts/k8s-splunk.yml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Pod +metadata: + name: splunk +spec: + hostNetwork: true + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: splunk + image: splunk/splunk:latest + ports: + - containerPort: 8000 + hostPort: 8000 + protocol: TCP + - containerPort: 8088 + hostPort: 8088 + protocol: TCP + - containerPort: 8089 + hostPort: 8089 + protocol: TCP + env: + - name: SPLUNK_START_ARGS + value: --accept-license + - name: SPLUNK_USER + value: root + - name: SPLUNK_PASSWORD + value: helloworld + - name: SPLUNK_LAUNCH_CONF + value: OPTIMISTIC_ABOUT_FILE_LOCKING=1 From 52ca48442f99824ed44ad0d2ae35ac4c9a7bd352 Mon Sep 17 00:00:00 2001 From: Mei Wang Date: Tue, 7 Sep 2021 16:21:34 -0700 Subject: [PATCH 05/21] remove circleci pipeline (#67) --- .circleci/config.yml | 87 ---------------------------------- .circleci/install_dep.sh | 5 -- .circleci/trigger_func_test.sh | 42 ---------------- README.md | 2 +- 4 files changed, 1 insertion(+), 135 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100755 .circleci/install_dep.sh delete mode 100755 .circleci/trigger_func_test.sh diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index cd1c23a..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,87 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: circleci/ruby:2.6.1-node-browsers - working_directory: ~/repo - steps: - - checkout - - setup_remote_docker: - reusable: true - - run: - name: Install dependencies - command: | - .circleci/install_dep.sh - - run: - name: Builder - command: | - rake build -t -v - cp -R pkg /tmp - - persist_to_workspace: - root: /tmp - paths: - - pkg - unit_test: - docker: - - image: circleci/ruby:2.6.1-node-browsers - working_directory: ~/repo - steps: - - attach_workspace: - at: /tmp - - checkout - - setup_remote_docker: - reusable: true - - run: - name: Install dependencies - command: | - .circleci/install_dep.sh - - run: - name: Run unit tests - command: | - bundle exec rake test -t -v - - func_test_main: - docker: - - image: circleci/ruby:2.6.1-node-browsers - working_directory: ~/repo - steps: - - checkout - - setup_remote_docker: - reusable: true - - run: - name: trigger - command: | - .circleci/trigger_func_test.sh main - - func_test_develop: - docker: - - image: circleci/ruby:2.6.1-node-browsers - working_directory: ~/repo - steps: - - checkout - - setup_remote_docker: - reusable: true - - run: - name: trigger - command: | - .circleci/trigger_func_test.sh develop - -workflows: - version: 2 - build_test: - jobs: - - build: - filters: - branches: - ignore: - - /^release\/.*/ - - main - - unit_test: - requires: - - build - - func_test_main: - requires: - - unit_test - - func_test_develop: - requires: - - unit_test diff --git a/.circleci/install_dep.sh b/.circleci/install_dep.sh deleted file mode 100755 index 7c4d131..0000000 --- a/.circleci/install_dep.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -e -gem install bundler -bundle update --bundler -bundle install \ No newline at end of file diff --git a/.circleci/trigger_func_test.sh b/.circleci/trigger_func_test.sh deleted file mode 100755 index b3c49b8..0000000 --- a/.circleci/trigger_func_test.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# trigger SCK 'main' branch to introduce this new image from this commit -# to working version of every other component. -ORGANIZATION=splunk -PROJECT=splunk-connect-for-kubernetes -BRANCH=$1 - -# Trigger functional test -curl -X POST --header "Content-Type: application/json" \ - -d '{"build_parameters": {"CIRCLE_JOB":"build_test", "TRIG_BRANCH":"'"$CIRCLE_BRANCH"'", "TRIG_PROJECT":"'"$CIRCLE_PROJECT_REPONAME"'", "TRIG_REPO":"'"$CIRCLE_REPOSITORY_URL"'"}}' "https://circleci.com/api/v1/project/$ORGANIZATION/$PROJECT/tree/$BRANCH?circle-token=$CIRCLE_TOKEN" > build.json -cat build.json -BUILD_NUM=$(jq -r .build_num build.json) -sleep 10 - -# Wait until finish or maximum 20 minutes -TIMEOUT=20 -DONE="FALSE" -until [ "$TIMEOUT" -lt 0 ] || [ "$DONE" == "TRUE" ]; do - curl -s https://circleci.com/api/v1/project/$ORGANIZATION/$PROJECT/$BUILD_NUM?circle-token=$CIRCLE_TOKEN > build_progress.json - STATUS=$(jq -r .status build_progress.json) - echo "STATUS = $STATUS" - if [ "$STATUS" != "running" ] && [ "$STATUS" != "queued" ]; then - DONE="TRUE" - else - let TIMEOUT-- - sleep 60 - fi -done - -BUILD_URL=$(jq -r .build_url build_progress.json) -if [ "$DONE" == "FALSE" ]; then - # Cancel hanging job and fail - curl -X POST https://circleci.com/api/v1/project/$ORGANIZATION/$PROJECT/$BUILD_NUM/cancel?circle-token=$CIRCLE_TOKEN -else - if [ "$STATUS" != "success" ] && [ "$STATUS" != "fixed" ]; then - echo "Functional test have failed please see:" - echo $BUILD_URL - exit 1 - fi - exit 0 -fi diff --git a/README.md b/README.md index 18f4a4d..f5c6943 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![CircleCI](https://circleci.com/gh/git-lfs/git-lfs.svg?style=shield&circle-token=856152c2b02bfd236f54d21e1f581f3e4ebf47ad)](https://circleci.com/gh/splunk/fluent-plugin-kubernetes-objects) + # fluent-plugin-kubernetes-objects [Fluentd](https://fluentd.org/) input plugin to collect [objects](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/) in a kubernetes cluster. From fd70f23dd20a6cb7ab33fe75f5994a5ca36e5231 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Thu, 28 Oct 2021 09:38:04 -0700 Subject: [PATCH 06/21] remove unnecessary components (#68) --- Gemfile.lock | 2 +- docker/Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index de4a52f..956959e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,4 +106,4 @@ DEPENDENCIES webmock (~> 3.5) BUNDLED WITH - 2.2.24 + 2.2.28 diff --git a/docker/Dockerfile b/docker/Dockerfile index 1a51eb6..8e02c43 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,7 +25,8 @@ RUN yum update -y \ && yum remove -y nodejs npm \ && gem install bundler \ && gem unpack /tmp/*.gem --target gem \ - && bundle install + && bundle install \ + && rpm -e --nodeps python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* RUN groupadd -r $FLUENT_USER && \ useradd -r -g $FLUENT_USER $FLUENT_USER && \ From eb9ef448a4870d19735adc857a98f26b8e65c177 Mon Sep 17 00:00:00 2001 From: foram-splunk <89519924+foram-splunk@users.noreply.github.com> Date: Fri, 29 Oct 2021 01:59:54 +0530 Subject: [PATCH 07/21] Update python3 for Red Hat Enterprise Linux 8 (#69) * Update python3 for Red Hat Enterprise Linux 8 to remediate the CVE-2020-14422 Co-authored-by: foram-splunk Co-authored-by: Rock Baek --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8e02c43..7d2f9c6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,7 +26,7 @@ RUN yum update -y \ && gem install bundler \ && gem unpack /tmp/*.gem --target gem \ && bundle install \ - && rpm -e --nodeps python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* + && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* RUN groupadd -r $FLUENT_USER && \ useradd -r -g $FLUENT_USER $FLUENT_USER && \ From 50902b4e2480912f8d7ffa85a7fbec6341beede2 Mon Sep 17 00:00:00 2001 From: Helena Brewster <86707578+hbrewster-splunk@users.noreply.github.com> Date: Wed, 3 Nov 2021 11:10:04 +1100 Subject: [PATCH 08/21] Multi-arch images (#70) * Multi-arch images --- .github/workflows/ci_build_test.yaml | 33 +++++++++++++++++++++++++--- ci_scripts/deploy_connector.sh | 6 +++-- docker/Dockerfile | 18 ++++++++++++--- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_build_test.yaml b/.github/workflows/ci_build_test.yaml index 46b004c..40438e0 100644 --- a/.github/workflows/ci_build_test.yaml +++ b/.github/workflows/ci_build_test.yaml @@ -75,10 +75,37 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - name: Build kube-objects image + - name: Prepare container build + id: prep + run: | + VERSION=`cat VERSION` + TAGS=splunk/kube-objects:recent + echo ::set-output name=tags::${TAGS} + echo ::set-output name=version::${VERSION} + + - name: Set up QEMU + uses: docker/setup-qemu-action@master + with: + platforms: all + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@master + + - name: Build multi-arch kubernetes-metrics image + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: ./docker/Dockerfile + platforms: linux/amd64 + push: false + load: true + tags: ${{ steps.prep.outputs.tags }} + build-args: VERSION=${{ steps.prep.outputs.version }} + + - name: Check kube-objects image run: | - sed -i 's/VERSION=$VERSION -t splunk\/kube-objects:$TAG .\/docker/VERSION=$VERSION -t kube-objects:$TAG .\/docker/' docker/build.sh - sudo docker/build.sh $GITHUB_RUN_ID docker image ls - name: Setup Minikube diff --git a/ci_scripts/deploy_connector.sh b/ci_scripts/deploy_connector.sh index 7b40512..708c55d 100755 --- a/ci_scripts/deploy_connector.sh +++ b/ci_scripts/deploy_connector.sh @@ -14,16 +14,18 @@ cd /opt git clone https://github.com/splunk/splunk-connect-for-kubernetes.git cd splunk-connect-for-kubernetes -sed -i 's/name: splunk\/kube-objects/repository: kube-objects/' ci_scripts/sck_values.yml cat ci_scripts/sck_values.yml +minikube image load splunk/kube-objects:recent + echo "Deploying k8s-connect with latest changes" helm install ci-sck --set global.splunk.hec.token=$CI_SPLUNK_HEC_TOKEN \ --set global.splunk.hec.host=$CI_SPLUNK_HOST \ --set kubelet.serviceMonitor.https=true \ --set splunk-kubernetes-objects.image.pullPolicy=IfNotPresent \ +--set splunk-kubernetes-objects.image.tag=recent \ -f ci_scripts/sck_values.yml helm-chart/splunk-connect-for-kubernetes #wait for deployment to finish until kubectl get pod | grep Running | [[ $(wc -l) == 4 ]]; do sleep 1; -done \ No newline at end of file +done diff --git a/docker/Dockerfile b/docker/Dockerfile index 7d2f9c6..d241f48 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,13 @@ +FROM ruby:2.7.4-buster as builder + +ADD ./ /app/ +WORKDIR /app +RUN gem install bundler +RUN bundle update --bundler +RUN bundle install + +RUN bundle exec rake build -t -v + FROM registry.access.redhat.com/ubi8/ruby-27 ARG VERSION @@ -15,12 +25,14 @@ ENV FLUENT_USER fluent USER root -COPY *.gem /tmp/ -COPY licenses /licenses +COPY --from=builder /app/pkg/fluent-plugin-*.gem /tmp/ + +RUN mkdir /licenses +COPY --from=builder /app/LICENSE /licenses/LICENSE RUN dnf install -y jq -COPY Gemfile* ./ +COPY --from=builder /app/docker/Gemfile* ./ RUN yum update -y \ && yum remove -y nodejs npm \ && gem install bundler \ From 65e65646263266d862eabbebf487b62409aa0bbc Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Wed, 3 Nov 2021 15:46:25 -0700 Subject: [PATCH 09/21] 1.1.8 (#71) --- Gemfile.lock | 24 +++++++++++----------- Makefile | 8 +++----- VERSION | 2 +- docker/Gemfile | 4 ++-- docker/Gemfile.lock | 49 ++++++++++++++++++++++++--------------------- 5 files changed, 44 insertions(+), 43 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 956959e..7138c8b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-objects (1.1.7) + fluent-plugin-kubernetes-objects (1.1.8) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) @@ -18,18 +18,18 @@ GEM docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - ffi (1.15.3) + ffi (1.15.4) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - fluentd (1.13.2) + fluentd (1.14.2) bundler cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.8.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) - strptime (>= 0.2.2, < 1.0.0) + strptime (>= 0.2.4, < 1.0.0) tzinfo (>= 1.0, < 3.0) tzinfo-data (~> 1.0) webrick (>= 1.4.2, < 1.8.0) @@ -53,11 +53,11 @@ GEM rest-client (~> 2.0) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0704) + mime-types-data (3.2021.0901) minitest (5.14.4) msgpack (1.4.2) netrc (0.11.0) - power_assert (2.0.0) + power_assert (2.0.1) public_suffix (4.0.6) rake (13.0.6) recursive-open-struct (1.1.3) @@ -77,17 +77,17 @@ GEM simplecov-html (0.12.3) simplecov_json_formatter (0.1.3) strptime (0.2.5) - test-unit (3.4.4) + test-unit (3.5.0) power_assert tzinfo (2.0.4) concurrent-ruby (~> 1.0) - tzinfo-data (1.2021.1) + tzinfo-data (1.2021.5) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext - unf_ext (0.0.7.7) - webmock (3.13.0) - addressable (>= 2.3.6) + unf_ext (0.0.8) + webmock (3.14.0) + addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) webrick (1.7.0) @@ -106,4 +106,4 @@ DEPENDENCIES webmock (~> 3.5) BUNDLED WITH - 2.2.28 + 2.2.30 diff --git a/Makefile b/Makefile index ed73903..2ad4486 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,9 @@ clean: clean_pkg clean_gems build: clean_pkg @bundle exec rake build -docker: build install-deps - @cp pkg/fluent-plugin-*.gem docker - @mkdir -p docker/licenses - @cp -rp LICENSE docker/licenses/ - @docker build --no-cache --pull --build-arg VERSION=$(VERSION) -t splunk/kube-objects:$(VERSION) ./docker +.PHONY: docker +docker: + @docker build --no-cache --pull --build-arg VERSION=$(VERSION) -t splunk/kube-objects:$(VERSION) . -f docker/Dockerfile unit-test: @bundle exec rake test diff --git a/VERSION b/VERSION index 2bf1ca5..18efdb9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.7 +1.1.8 diff --git a/docker/Gemfile b/docker/Gemfile index e111507..359acd9 100644 --- a/docker/Gemfile +++ b/docker/Gemfile @@ -3,10 +3,10 @@ source 'https://rubygems.org' # This is separate gemfile for building docker image that has all plugins # for kubernetes log collection agent # List all required gems here and install via bundler to resolve dependencies -gem "fluentd", "= 1.11.5" +gem "fluentd", ">= 1.14.2" gem "fluent-plugin-prometheus", ">= 2.0" gem "fluent-plugin-record-modifier", "= 2.1.0" -gem "fluent-plugin-kubernetes_metadata_filter", "=2.5.3" +gem "fluent-plugin-kubernetes_metadata_filter", ">= 2.9" gem "fluent-plugin-jq", "= 0.5.1" gem "oj", "= 3.10.18" gem 'multi_json', '>= 1.14.1' diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index c32ab0e..c964ef5 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-objects (1.1.6) + fluent-plugin-kubernetes-objects (1.1.8) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) @@ -9,9 +9,9 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (6.1.4) - activesupport (= 6.1.4) - activesupport (6.1.4) + activemodel (6.1.4.1) + activesupport (= 6.1.4.1) + activesupport (6.1.4.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -28,18 +28,18 @@ GEM cool.io (1.7.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - ffi (1.15.3) + ffi (1.15.4) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake fluent-plugin-jq (0.5.1) fluentd (>= 0.14.10, < 2) multi_json (~> 1.13) - fluent-plugin-kubernetes_metadata_filter (2.5.3) - fluentd (>= 0.14.0, < 1.12) - kubeclient (< 5) + fluent-plugin-kubernetes_metadata_filter (2.9.2) + fluentd (>= 0.14.0, < 1.15) + kubeclient (>= 4.0.0, < 5.0.0) lru_redux - fluent-plugin-prometheus (2.0.1) + fluent-plugin-prometheus (2.0.2) fluentd (>= 1.9.1, < 2) prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) @@ -50,15 +50,17 @@ GEM net-http-persistent (~> 3.1) openid_connect (~> 1.1.8) prometheus-client (>= 2.1.0) - fluentd (1.11.5) + fluentd (1.14.2) + bundler cool.io (>= 1.4.5, < 2.0.0) - http_parser.rb (>= 0.5.1, < 0.7.0) + http_parser.rb (>= 0.5.1, < 0.8.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) - strptime (>= 0.2.2, < 1.0.0) + strptime (>= 0.2.4, < 1.0.0) tzinfo (>= 1.0, < 3.0) tzinfo-data (~> 1.0) + webrick (>= 1.4.2, < 1.8.0) yajl-ruby (~> 1.0) http (4.4.1) addressable (~> 2.3) @@ -73,7 +75,7 @@ GEM ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.5.3) httpclient (2.8.3) - i18n (1.8.10) + i18n (1.8.11) concurrent-ruby (~> 1.0) json-jwt (1.13.0) activesupport (>= 4.2) @@ -88,8 +90,8 @@ GEM mini_mime (>= 0.1.1) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0704) - mini_mime (1.1.0) + mime-types-data (3.2021.0901) + mini_mime (1.1.2) minitest (5.14.4) msgpack (1.4.2) multi_json (1.15.0) @@ -110,7 +112,7 @@ GEM prometheus-client (2.1.0) public_suffix (4.0.6) rack (2.2.3) - rack-oauth2 (1.17.0) + rack-oauth2 (1.19.0) activesupport attr_required httpclient @@ -127,28 +129,29 @@ GEM sigdump (~> 0.2.2) sigdump (0.2.4) strptime (0.2.5) - swd (1.2.0) + swd (1.3.0) activesupport (>= 3) attr_required (>= 0.0.5) httpclient (>= 2.4) tzinfo (2.0.4) concurrent-ruby (~> 1.0) - tzinfo-data (1.2021.1) + tzinfo-data (1.2021.5) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext - unf_ext (0.0.7.7) + unf_ext (0.0.8) validate_email (0.1.6) activemodel (>= 3.0) mail (>= 2.2.5) validate_url (1.0.13) activemodel (>= 3.0.0) public_suffix - webfinger (1.1.0) + webfinger (1.2.0) activesupport httpclient (>= 2.4) + webrick (1.7.0) yajl-ruby (1.4.1) - zeitwerk (2.4.2) + zeitwerk (2.5.1) PLATFORMS ruby @@ -157,11 +160,11 @@ DEPENDENCIES bigdecimal (>= 2.0.0) fluent-plugin-jq (= 0.5.1) fluent-plugin-kubernetes-objects! - fluent-plugin-kubernetes_metadata_filter (= 2.5.3) + fluent-plugin-kubernetes_metadata_filter (>= 2.9) fluent-plugin-prometheus (>= 2.0) fluent-plugin-record-modifier (= 2.1.0) fluent-plugin-splunk-hec (>= 1.2.5) - fluentd (= 1.11.5) + fluentd (>= 1.14.2) http_parser.rb (>= 0.5.3) kubeclient (>= 4.6.0) multi_json (>= 1.14.1) From 664266fa422785fd91bb0b07f35ef0d303ec0cf4 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Wed, 3 Nov 2021 17:15:47 -0700 Subject: [PATCH 10/21] wrap rpm (#72) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d241f48..ab8d9a1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,7 +38,7 @@ RUN yum update -y \ && gem install bundler \ && gem unpack /tmp/*.gem --target gem \ && bundle install \ - && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* + && REMOVE=$(rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-*) RUN groupadd -r $FLUENT_USER && \ useradd -r -g $FLUENT_USER $FLUENT_USER && \ From 5c46993fa3aeb22db7d05101ea77777c614aa0a2 Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Wed, 3 Nov 2021 18:23:44 -0700 Subject: [PATCH 11/21] err handle (#73) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ab8d9a1..efdc0a4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,7 +38,7 @@ RUN yum update -y \ && gem install bundler \ && gem unpack /tmp/*.gem --target gem \ && bundle install \ - && REMOVE=$(rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-*) + && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* || true RUN groupadd -r $FLUENT_USER && \ useradd -r -g $FLUENT_USER $FLUENT_USER && \ From 722bed78aa9cbdd09a80653eb55b4ade46e82fda Mon Sep 17 00:00:00 2001 From: Rock Baek Date: Thu, 16 Dec 2021 00:44:59 -0800 Subject: [PATCH 12/21] 1.1.9 (#76) --- Gemfile.lock | 2 +- VERSION | 2 +- docker/Gemfile.lock | 24 +++++++++++------------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 7138c8b..acd0532 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-objects (1.1.8) + fluent-plugin-kubernetes-objects (1.1.9) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) diff --git a/VERSION b/VERSION index db15278..a5e4282 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.8 \ No newline at end of file +1.1.9 \ No newline at end of file diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index c964ef5..28cac1a 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-objects (1.1.8) + fluent-plugin-kubernetes-objects (1.1.9) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) @@ -9,14 +9,13 @@ PATH GEM remote: https://rubygems.org/ specs: - activemodel (6.1.4.1) - activesupport (= 6.1.4.1) - activesupport (6.1.4.1) + activemodel (7.0.0) + activesupport (= 7.0.0) + activesupport (7.0.0) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) - zeitwerk (~> 2.3) addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) aes_key_wrap (1.1.0) @@ -35,7 +34,7 @@ GEM fluent-plugin-jq (0.5.1) fluentd (>= 0.14.10, < 2) multi_json (~> 1.13) - fluent-plugin-kubernetes_metadata_filter (2.9.2) + fluent-plugin-kubernetes_metadata_filter (2.9.3) fluentd (>= 0.14.0, < 1.15) kubeclient (>= 4.0.0, < 5.0.0) lru_redux @@ -44,16 +43,16 @@ GEM prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) - fluent-plugin-splunk-hec (1.2.7) + fluent-plugin-splunk-hec (1.2.9) fluentd (>= 1.4) multi_json (~> 1.13) net-http-persistent (~> 3.1) openid_connect (~> 1.1.8) prometheus-client (>= 2.1.0) - fluentd (1.14.2) + fluentd (1.14.3) bundler cool.io (>= 1.4.5, < 2.0.0) - http_parser.rb (>= 0.5.1, < 0.8.0) + http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) @@ -88,11 +87,11 @@ GEM lru_redux (1.1.0) mail (2.7.1) mini_mime (>= 0.1.1) - mime-types (3.3.1) + mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0901) + mime-types-data (3.2021.1115) mini_mime (1.1.2) - minitest (5.14.4) + minitest (5.15.0) msgpack (1.4.2) multi_json (1.15.0) net-http-persistent (3.1.0) @@ -151,7 +150,6 @@ GEM httpclient (>= 2.4) webrick (1.7.0) yajl-ruby (1.4.1) - zeitwerk (2.5.1) PLATFORMS ruby From 17323d06cd51c97aaa134240dd0dd703888f0cc7 Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Wed, 12 Jan 2022 08:14:29 +0530 Subject: [PATCH 13/21] stop pull/watch thread when resource not found (#78) --- lib/fluent/plugin/in_kubernetes_objects.rb | 38 ++++++++++++------- .../plugin/in_kubernetes_objects_test.rb | 23 +++++++++++ 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/lib/fluent/plugin/in_kubernetes_objects.rb b/lib/fluent/plugin/in_kubernetes_objects.rb index f57a2b0..6fa46fd 100644 --- a/lib/fluent/plugin/in_kubernetes_objects.rb +++ b/lib/fluent/plugin/in_kubernetes_objects.rb @@ -200,7 +200,12 @@ def create_pull_thread(conf) tag = generate_tag resource_name while thread_current_running? log.debug "Going to pull #{resource_name}" - response = @client.public_send "get_#{resource_name}", options + begin + response = @client.public_send "get_#{resource_name}", options + rescue Kubeclient::ResourceNotFoundError, NoMethodError + log.error "resource '#{resource_name}' not found. Stopped pulling it" + break + end now = Fluent::Engine.now es = Fluent::MultiEventStream.new @@ -243,22 +248,27 @@ def create_watcher_thread(conf) thread_create :"watch_#{resource_name}" do while thread_current_running? - @client.public_send("watch_#{resource_name}", options).tap do |watcher| - tag = generate_tag "#{resource_name}" - begin - watcher.each do |entity| - begin - entity = JSON.parse(entity) - router.emit tag, Fluent::Engine.now, entity - options[:resource_version] = entity['object']['metadata']['resourceVersion'] - @storage.put resource_name, entity['object']['metadata']['resourceVersion'] - rescue => e - log.info "Got exception #{e} parsing entity #{entity}. Resetting watcher." + begin + @client.public_send("watch_#{resource_name}", options).tap do |watcher| + tag = generate_tag "#{resource_name}" + begin + watcher.each do |entity| + begin + entity = JSON.parse(entity) + router.emit tag, Fluent::Engine.now, entity + options[:resource_version] = entity['object']['metadata']['resourceVersion'] + @storage.put resource_name, entity['object']['metadata']['resourceVersion'] + rescue => e + log.info "Got exception #{e} parsing entity #{entity}. Resetting watcher." + end end + rescue => e + log.info "Got exception #{e}. Resetting watcher." end - rescue => e - log.info "Got exception #{e}. Resetting watcher." end + rescue Kubeclient::ResourceNotFoundError, NoMethodError + log.error "resource '#{resource_name}' not found. Stopped watching it" + break end end end diff --git a/test/fluent/plugin/in_kubernetes_objects_test.rb b/test/fluent/plugin/in_kubernetes_objects_test.rb index 8063d50..687e3a5 100644 --- a/test/fluent/plugin/in_kubernetes_objects_test.rb +++ b/test/fluent/plugin/in_kubernetes_objects_test.rb @@ -144,5 +144,28 @@ f.unlink end end + it "checks for invalid pull request" do + d = create_input_driver(<<~CONF) + kubernetes_url #{k8s_url} + + resource_name fakeResource + + CONF + + d.run expect_emits: 0, timeout: 3 + + expect(d.logs.any? { |log| log.include? "resource 'fakeResource' not found." }).must_equal(true) + end + it "checks for invalid watch request" do + d = create_input_driver(<<~CONF) + kubernetes_url #{k8s_url} + + resource_name fakeResource + + CONF + + d.run expect_emits: 0, timeout: 3 + expect(d.logs.any? { |log| log.include? "resource 'fakeResource' not found." }).must_equal(true) + end end end From 029c3e98600f3fb0d34d5ae92a53c999d616683e Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Thu, 24 Feb 2022 23:40:51 +0530 Subject: [PATCH 14/21] updated cgi,bundler,date,i18n gems (#82) * updated cgi,bundler,date,i18n gems * Update Dockerfile --- docker/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index efdc0a4..0bc423a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,11 +33,16 @@ COPY --from=builder /app/LICENSE /licenses/LICENSE RUN dnf install -y jq COPY --from=builder /app/docker/Gemfile* ./ +RUN gem update date cgi +RUN rm -f /usr/share/gems/specifications/default/cgi-0.1.0.gemspec /usr/share/gems/specifications/default/date-3.0.0.gemspec RUN yum update -y \ && yum remove -y nodejs npm \ && gem install bundler \ + && gem uninstall -i /usr/share/gems bundler \ && gem unpack /tmp/*.gem --target gem \ && bundle install \ + && bundle update i18n \ + && gem uninstall -i /usr/share/gems i18n --version 1.8.11 \ && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* || true RUN groupadd -r $FLUENT_USER && \ From 2f927dd596345ca921b91bcc8be262e194dfd25f Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Tue, 1 Mar 2022 06:57:10 +0530 Subject: [PATCH 15/21] updated Github actions workflow (#81) * updated ci * removed calico --- .github/workflows/ci_build_test.yaml | 18 +++++++++++------- ci_scripts/deploy_connector.sh | 8 ++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_build_test.yaml b/.github/workflows/ci_build_test.yaml index 40438e0..d4625d3 100644 --- a/.github/workflows/ci_build_test.yaml +++ b/.github/workflows/ci_build_test.yaml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout code uses: actions/checkout@v2 @@ -35,7 +35,7 @@ jobs: key: ${{ runner.os }}-build unit-test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 needs: - build steps: @@ -67,8 +67,9 @@ jobs: CI_INDEX_EVENTS: ci_events CI_INDEX_OBJECTS: ci_objects CI_INDEX_METRICS: ci_metrics - KUBERNETES_VERSION: v1.15.2 - MINIKUBE_VERSION: v1.21.0 + KUBERNETES_VERSION: v1.23.2 + MINIKUBE_VERSION: v1.24.0 + MINIKUBE_NODE_COUNTS: 2 GITHUB_ACTIONS: true steps: @@ -121,7 +122,7 @@ jobs: chmod +x minikube sudo mv minikube /usr/local/bin/ # Start Minikube and Wait - minikube start --driver=docker --container-runtime=docker --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --no-vtx-check + minikube start --driver=docker --container-runtime=docker --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --no-vtx-check -n=${MINIKUBE_NODE_COUNTS} export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; @@ -130,13 +131,15 @@ jobs: - name: Install Splunk run: | # Wait until minikube is ready - kubectl apply -f https://docs.projectcalico.org/v3.14/manifests/calico.yaml export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do echo "wait for minikube ready ..." sleep 1; done kubectl get nodes + until kubectl get sa | grep -q 'default'; do + sleep 1; + done # Install Splunk on minikube kubectl apply -f ci_scripts/k8s-splunk.yml # Wait until splunk is ready @@ -189,4 +192,5 @@ jobs: --splunkd-url https://$CI_SPLUNK_HOST:8089 \ --splunk-user admin \ --splunk-password $CI_SPLUNK_PASSWORD \ - -p no:warnings -s + --nodes-count $MINIKUBE_NODE_COUNTS\ + -p no:warnings -s -n auto diff --git a/ci_scripts/deploy_connector.sh b/ci_scripts/deploy_connector.sh index 708c55d..40ed6cd 100755 --- a/ci_scripts/deploy_connector.sh +++ b/ci_scripts/deploy_connector.sh @@ -25,7 +25,11 @@ helm install ci-sck --set global.splunk.hec.token=$CI_SPLUNK_HEC_TOKEN \ --set splunk-kubernetes-objects.image.pullPolicy=IfNotPresent \ --set splunk-kubernetes-objects.image.tag=recent \ -f ci_scripts/sck_values.yml helm-chart/splunk-connect-for-kubernetes -#wait for deployment to finish -until kubectl get pod | grep Running | [[ $(wc -l) == 4 ]]; do + +kubectl get pod +# wait for deployment to finish +# metric and logging deamon set for each node + aggr + object + splunk +PODS=$((MINIKUBE_NODE_COUNTS*2+2+1)) +until kubectl get pod | grep Running | [[ $(wc -l) == $PODS ]]; do sleep 1; done From 36d44aace404491304d1764d73bcb250f41db915 Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Tue, 15 Mar 2022 22:28:14 +0530 Subject: [PATCH 16/21] prep for 1.1.10 (#83) --- Gemfile.lock | 16 ++++++++-------- VERSION | 2 +- docker/Gemfile.lock | 5 ++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index acd0532..2d82301 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-objects (1.1.9) + fluent-plugin-kubernetes-objects (1.1.10) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) @@ -18,14 +18,14 @@ GEM docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - ffi (1.15.4) + ffi (1.15.5) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - fluentd (1.14.2) + fluentd (1.14.4) bundler cool.io (>= 1.4.5, < 2.0.0) - http_parser.rb (>= 0.5.1, < 0.8.0) + http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) serverengine (>= 2.2.2, < 3.0.0) sigdump (~> 0.2.2) @@ -51,11 +51,11 @@ GEM http (>= 3.0, < 5.0) recursive-open-struct (~> 1.0, >= 1.0.4) rest-client (~> 2.0) - mime-types (3.3.1) + mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0901) + mime-types-data (3.2022.0105) minitest (5.14.4) - msgpack (1.4.2) + msgpack (1.4.4) netrc (0.11.0) power_assert (2.0.1) public_suffix (4.0.6) @@ -67,7 +67,7 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) rexml (3.2.5) - serverengine (2.2.4) + serverengine (2.2.5) sigdump (~> 0.2.2) sigdump (0.2.4) simplecov (0.21.2) diff --git a/VERSION b/VERSION index a5e4282..9695327 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.9 \ No newline at end of file +1.1.10 \ No newline at end of file diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index 5a56c61..4f9de62 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-objects (1.1.9) + fluent-plugin-kubernetes-objects (1.1.10) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.6.0) @@ -43,7 +43,7 @@ GEM prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) - fluent-plugin-splunk-hec (1.2.9) + fluent-plugin-splunk-hec (1.2.11) fluentd (>= 1.4) multi_json (~> 1.13) net-http-persistent (~> 3.1) @@ -150,7 +150,6 @@ GEM httpclient (>= 2.4) webrick (1.7.0) yajl-ruby (1.4.1) - zeitwerk (2.5.1) PLATFORMS ruby From 00230b9c3590735d6d6da0f17a57099457ebc27b Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Tue, 29 Mar 2022 11:11:08 +0530 Subject: [PATCH 17/21] updated kubeclient to v4.9.3 (#85) * updated kubeclient to v4.9.3 * added api_endpoint config --- Gemfile.lock | 12 +- docker/Gemfile | 2 +- docker/Gemfile.lock | 17 +- fluent-plugin-kubernetes-objects.gemspec | 2 +- lib/fluent/plugin/in_kubernetes_objects.rb | 5 +- test/apps-v1-namespace.json | 806 ++++++++++++++++++ test/apps-v1.json | 199 +++++ test/apps.json | 15 + .../plugin/in_kubernetes_objects_test.rb | 5 +- test/test_helper.rb | 21 + 10 files changed, 1068 insertions(+), 16 deletions(-) create mode 100644 test/apps-v1-namespace.json create mode 100644 test/apps-v1.json create mode 100644 test/apps.json diff --git a/Gemfile.lock b/Gemfile.lock index 2d82301..a5850dd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -4,7 +4,7 @@ PATH fluent-plugin-kubernetes-objects (1.1.10) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) - kubeclient (~> 4.6.0) + kubeclient (~> 4.9.3) GEM remote: https://rubygems.org/ @@ -47,15 +47,19 @@ GEM http-parser (1.2.3) ffi-compiler (>= 1.0, < 2.0) http_parser.rb (0.5.3) - kubeclient (4.6.0) + jsonpath (1.1.0) + multi_json + kubeclient (4.9.3) http (>= 3.0, < 5.0) - recursive-open-struct (~> 1.0, >= 1.0.4) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) rest-client (~> 2.0) mime-types (3.4.1) mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) minitest (5.14.4) msgpack (1.4.4) + multi_json (1.15.0) netrc (0.11.0) power_assert (2.0.1) public_suffix (4.0.6) @@ -85,7 +89,7 @@ GEM tzinfo (>= 1.0.0) unf (0.1.4) unf_ext - unf_ext (0.0.8) + unf_ext (0.0.8.1) webmock (3.14.0) addressable (>= 2.8.0) crack (>= 0.3.2) diff --git a/docker/Gemfile b/docker/Gemfile index 359acd9..6ac6508 100644 --- a/docker/Gemfile +++ b/docker/Gemfile @@ -11,7 +11,7 @@ gem "fluent-plugin-jq", "= 0.5.1" gem "oj", "= 3.10.18" gem 'multi_json', '>= 1.14.1' gem 'bigdecimal', '>= 2.0.0' -gem 'kubeclient', '>= 4.6.0' +gem 'kubeclient', '>= 4.9.3' gem 'http_parser.rb', '>= 0.5.3' gem "fluent-plugin-splunk-hec", ">= 1.2.5" diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index 4f9de62..bb87688 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -4,7 +4,7 @@ PATH fluent-plugin-kubernetes-objects (1.1.10) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) - kubeclient (~> 4.6.0) + kubeclient (~> 4.9.3) GEM remote: https://rubygems.org/ @@ -27,7 +27,7 @@ GEM cool.io (1.7.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - ffi (1.15.4) + ffi (1.15.5) ffi-compiler (1.0.1) ffi (>= 1.0.0) rake @@ -80,16 +80,19 @@ GEM activesupport (>= 4.2) aes_key_wrap bindata - kubeclient (4.6.0) + jsonpath (1.1.0) + multi_json + kubeclient (4.9.3) http (>= 3.0, < 5.0) - recursive-open-struct (~> 1.0, >= 1.0.4) + jsonpath (~> 1.0) + recursive-open-struct (~> 1.1, >= 1.1.1) rest-client (~> 2.0) lru_redux (1.1.0) mail (2.7.1) mini_mime (>= 0.1.1) mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.1115) + mime-types-data (3.2022.0105) mini_mime (1.1.2) minitest (5.15.0) msgpack (1.4.2) @@ -138,7 +141,7 @@ GEM tzinfo (>= 1.0.0) unf (0.1.4) unf_ext - unf_ext (0.0.8) + unf_ext (0.0.8.1) validate_email (0.1.6) activemodel (>= 3.0) mail (>= 2.2.5) @@ -164,7 +167,7 @@ DEPENDENCIES fluent-plugin-splunk-hec (>= 1.2.5) fluentd (>= 1.14.2) http_parser.rb (>= 0.5.3) - kubeclient (>= 4.6.0) + kubeclient (>= 4.9.3) multi_json (>= 1.14.1) oj (= 3.10.18) diff --git a/fluent-plugin-kubernetes-objects.gemspec b/fluent-plugin-kubernetes-objects.gemspec index a0538c5..e115f71 100644 --- a/fluent-plugin-kubernetes-objects.gemspec +++ b/fluent-plugin-kubernetes-objects.gemspec @@ -22,7 +22,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.required_ruby_version = '>= 2.3.0' spec.add_runtime_dependency 'fluentd', '>= 1.9.1' - spec.add_runtime_dependency 'kubeclient', '~> 4.6.0' + spec.add_runtime_dependency 'kubeclient', '~> 4.9.3' spec.add_runtime_dependency 'http_parser.rb', '= 0.5.3' spec.add_development_dependency 'bundler', '~> 2.0' diff --git a/lib/fluent/plugin/in_kubernetes_objects.rb b/lib/fluent/plugin/in_kubernetes_objects.rb index 6fa46fd..2e1ae6f 100644 --- a/lib/fluent/plugin/in_kubernetes_objects.rb +++ b/lib/fluent/plugin/in_kubernetes_objects.rb @@ -20,6 +20,9 @@ class KubernetesObjectsInput < Fluent::Plugin::Input desc 'Kubernetes API version.' config_param :api_version, :string, default: 'v1' + desc 'Kubernetes API endpoint.' + config_param :api_endpoint, :string, default: 'api' + desc 'Path to the certificate file for this client.' config_param :client_cert, :string, default: nil @@ -139,7 +142,7 @@ def initialize_client env_host = ENV['KUBERNETES_SERVICE_HOST'] env_port = ENV['KUBERNETES_SERVICE_PORT'] if env_host && env_port - @kubernetes_url = "https://#{env_host}:#{env_port}/#{@api_version == 'v1' ? 'api' : 'apis'}" + @kubernetes_url = "https://#{env_host}:#{env_port}/#{@api_endpoint.delete_prefix('/')}" end end diff --git a/test/apps-v1-namespace.json b/test/apps-v1-namespace.json new file mode 100644 index 0000000..954d3da --- /dev/null +++ b/test/apps-v1-namespace.json @@ -0,0 +1,806 @@ +{ + "kind": "DeploymentList", + "apiVersion": "apps/v1", + "metadata": { + "resourceVersion": "20030" + }, + "items": [ + { + "metadata": { + "name": "sck-splunk-kubernetes-objects", + "namespace": "default", + "uid": "09819f91-bc02-4896-b3e9-fd237da9322a", + "resourceVersion": "18812", + "generation": 1, + "creationTimestamp": "2022-03-28T12:12:10Z", + "labels": { + "app": "splunk-kubernetes-objects", + "app.kubernetes.io/managed-by": "Helm", + "chart": "splunk-kubernetes-objects-1.4.13", + "heritage": "Helm", + "release": "sck" + }, + "annotations": { + "deployment.kubernetes.io/revision": "1", + "meta.helm.sh/release-name": "sck", + "meta.helm.sh/release-namespace": "default" + }, + "managedFields": [ + { + "manager": "helm", + "operation": "Update", + "apiVersion": "apps/v1", + "time": "2022-03-28T12:12:10Z", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:meta.helm.sh/release-name": {}, + "f:meta.helm.sh/release-namespace": {} + }, + "f:labels": { + ".": {}, + "f:app": {}, + "f:app.kubernetes.io/managed-by": {}, + "f:chart": {}, + "f:heritage": {}, + "f:release": {} + } + }, + "f:spec": { + "f:progressDeadlineSeconds": {}, + "f:replicas": {}, + "f:revisionHistoryLimit": {}, + "f:selector": {}, + "f:strategy": { + "f:rollingUpdate": { + ".": {}, + "f:maxSurge": {}, + "f:maxUnavailable": {} + }, + "f:type": {} + }, + "f:template": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:checksum/config": {} + }, + "f:labels": { + ".": {}, + "f:app": {}, + "f:engine": {}, + "f:release": {} + } + }, + "f:spec": { + "f:containers": { + "k:{\"name\":\"splunk-fluentd-k8s-objects\"}": { + ".": {}, + "f:env": { + ".": {}, + "k:{\"name\":\"MY_NAMESPACE\"}": { + ".": {}, + "f:name": {}, + "f:valueFrom": { + ".": {}, + "f:fieldRef": {} + } + }, + "k:{\"name\":\"MY_POD_NAME\"}": { + ".": {}, + "f:name": {}, + "f:valueFrom": { + ".": {}, + "f:fieldRef": {} + } + }, + "k:{\"name\":\"NODE_NAME\"}": { + ".": {}, + "f:name": {}, + "f:valueFrom": { + ".": {}, + "f:fieldRef": {} + } + }, + "k:{\"name\":\"SPLUNK_HEC_TOKEN\"}": { + ".": {}, + "f:name": {}, + "f:valueFrom": { + ".": {}, + "f:secretKeyRef": {} + } + } + }, + "f:image": {}, + "f:imagePullPolicy": {}, + "f:livenessProbe": { + ".": {}, + "f:failureThreshold": {}, + "f:httpGet": { + ".": {}, + "f:path": {}, + "f:port": {}, + "f:scheme": {} + }, + "f:initialDelaySeconds": {}, + "f:periodSeconds": {}, + "f:successThreshold": {}, + "f:timeoutSeconds": {} + }, + "f:name": {}, + "f:resources": { + ".": {}, + "f:requests": { + ".": {}, + "f:cpu": {}, + "f:memory": {} + } + }, + "f:securityContext": { + ".": {}, + "f:runAsNonRoot": {} + }, + "f:terminationMessagePath": {}, + "f:terminationMessagePolicy": {}, + "f:volumeMounts": { + ".": {}, + "k:{\"mountPath\":\"/fluentd/etc\"}": { + ".": {}, + "f:mountPath": {}, + "f:name": {} + }, + "k:{\"mountPath\":\"/fluentd/etc/splunk\"}": { + ".": {}, + "f:mountPath": {}, + "f:name": {}, + "f:readOnly": {} + } + } + } + }, + "f:dnsPolicy": {}, + "f:nodeSelector": {}, + "f:restartPolicy": {}, + "f:schedulerName": {}, + "f:securityContext": {}, + "f:serviceAccount": {}, + "f:serviceAccountName": {}, + "f:terminationGracePeriodSeconds": {}, + "f:volumes": { + ".": {}, + "k:{\"name\":\"conf-configmap\"}": { + ".": {}, + "f:configMap": { + ".": {}, + "f:defaultMode": {}, + "f:name": {} + }, + "f:name": {} + }, + "k:{\"name\":\"secrets\"}": { + ".": {}, + "f:name": {}, + "f:secret": { + ".": {}, + "f:defaultMode": {}, + "f:secretName": {} + } + } + } + } + } + } + } + }, + { + "manager": "kube-controller-manager", + "operation": "Update", + "apiVersion": "apps/v1", + "time": "2022-03-28T12:12:11Z", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + "f:deployment.kubernetes.io/revision": {} + } + }, + "f:status": { + "f:availableReplicas": {}, + "f:conditions": { + ".": {}, + "k:{\"type\":\"Available\"}": { + ".": {}, + "f:lastTransitionTime": {}, + "f:lastUpdateTime": {}, + "f:message": {}, + "f:reason": {}, + "f:status": {}, + "f:type": {} + }, + "k:{\"type\":\"Progressing\"}": { + ".": {}, + "f:lastTransitionTime": {}, + "f:lastUpdateTime": {}, + "f:message": {}, + "f:reason": {}, + "f:status": {}, + "f:type": {} + } + }, + "f:observedGeneration": {}, + "f:readyReplicas": {}, + "f:replicas": {}, + "f:updatedReplicas": {} + } + }, + "subresource": "status" + } + ] + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "app": "splunk-kubernetes-objects", + "engine": "fluentd", + "release": "sck" + } + }, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "app": "splunk-kubernetes-objects", + "engine": "fluentd", + "release": "sck" + }, + "annotations": { + "checksum/config": "0448623a3086ce1dee0966dd304e5e7859039a38d06be17f716bfea1777263b5" + } + }, + "spec": { + "volumes": [ + { + "name": "conf-configmap", + "configMap": { + "name": "sck-splunk-kubernetes-objects", + "defaultMode": 420 + } + }, + { + "name": "secrets", + "secret": { + "secretName": "splunk-kubernetes-objects", + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "splunk-fluentd-k8s-objects", + "image": "docker.io/splunk/kube-objects:1.1.10", + "env": [ + { + "name": "MY_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "MY_POD_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.name" + } + } + }, + { + "name": "NODE_NAME", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "spec.nodeName" + } + } + }, + { + "name": "SPLUNK_HEC_TOKEN", + "valueFrom": { + "secretKeyRef": { + "name": "splunk-kubernetes-objects", + "key": "splunk_hec_token" + } + } + } + ], + "resources": { + "requests": { + "cpu": "100m", + "memory": "200Mi" + } + }, + "volumeMounts": [ + { + "name": "conf-configmap", + "mountPath": "/fluentd/etc" + }, + { + "name": "secrets", + "readOnly": true, + "mountPath": "/fluentd/etc/splunk" + } + ], + "livenessProbe": { + "httpGet": { + "path": "/api/plugins.json", + "port": 24220, + "scheme": "HTTP" + }, + "initialDelaySeconds": 60, + "timeoutSeconds": 1, + "periodSeconds": 60, + "successThreshold": 1, + "failureThreshold": 3 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "runAsNonRoot": false + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "nodeSelector": { + "beta.kubernetes.io/os": "linux" + }, + "serviceAccountName": "sck-splunk-kubernetes-objects", + "serviceAccount": "sck-splunk-kubernetes-objects", + "securityContext": {}, + "schedulerName": "default-scheduler" + } + }, + "strategy": { + "type": "RollingUpdate", + "rollingUpdate": { + "maxUnavailable": "25%", + "maxSurge": "25%" + } + }, + "revisionHistoryLimit": 10, + "progressDeadlineSeconds": 600 + }, + "status": { + "observedGeneration": 1, + "replicas": 1, + "updatedReplicas": 1, + "readyReplicas": 1, + "availableReplicas": 1, + "conditions": [ + { + "type": "Available", + "status": "True", + "lastUpdateTime": "2022-03-28T12:12:11Z", + "lastTransitionTime": "2022-03-28T12:12:11Z", + "reason": "MinimumReplicasAvailable", + "message": "Deployment has minimum availability." + }, + { + "type": "Progressing", + "status": "True", + "lastUpdateTime": "2022-03-28T12:12:11Z", + "lastTransitionTime": "2022-03-28T12:12:10Z", + "reason": "NewReplicaSetAvailable", + "message": "ReplicaSet \"sck-splunk-kubernetes-objects-84f69bcff9\" has successfully progressed." + } + ] + } + }, + { + "metadata": { + "name": "coredns", + "namespace": "kube-system", + "uid": "3e20e775-3e2e-45b6-a2d1-d0d3d78f9e4c", + "resourceVersion": "19867", + "generation": 3, + "creationTimestamp": "2022-03-28T05:06:59Z", + "labels": { + "k8s-app": "kube-dns" + }, + "annotations": { + "deployment.kubernetes.io/revision": "1" + }, + "managedFields": [ + { + "manager": "Go-http-client", + "operation": "Update", + "apiVersion": "apps/v1", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:spec": { + "f:replicas": {} + } + }, + "subresource": "scale" + }, + { + "manager": "Go-http-client", + "operation": "Update", + "apiVersion": "apps/v1", + "time": "2022-03-28T05:06:59Z", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:labels": { + ".": {}, + "f:k8s-app": {} + } + }, + "f:spec": { + "f:progressDeadlineSeconds": {}, + "f:revisionHistoryLimit": {}, + "f:selector": {}, + "f:strategy": { + "f:rollingUpdate": { + ".": {}, + "f:maxSurge": {}, + "f:maxUnavailable": {} + }, + "f:type": {} + }, + "f:template": { + "f:metadata": { + "f:labels": { + ".": {}, + "f:k8s-app": {} + } + }, + "f:spec": { + "f:containers": { + "k:{\"name\":\"coredns\"}": { + ".": {}, + "f:args": {}, + "f:image": {}, + "f:imagePullPolicy": {}, + "f:livenessProbe": { + ".": {}, + "f:failureThreshold": {}, + "f:httpGet": { + ".": {}, + "f:path": {}, + "f:port": {}, + "f:scheme": {} + }, + "f:initialDelaySeconds": {}, + "f:periodSeconds": {}, + "f:successThreshold": {}, + "f:timeoutSeconds": {} + }, + "f:name": {}, + "f:ports": { + ".": {}, + "k:{\"containerPort\":53,\"protocol\":\"TCP\"}": { + ".": {}, + "f:containerPort": {}, + "f:name": {}, + "f:protocol": {} + }, + "k:{\"containerPort\":53,\"protocol\":\"UDP\"}": { + ".": {}, + "f:containerPort": {}, + "f:name": {}, + "f:protocol": {} + }, + "k:{\"containerPort\":9153,\"protocol\":\"TCP\"}": { + ".": {}, + "f:containerPort": {}, + "f:name": {}, + "f:protocol": {} + } + }, + "f:readinessProbe": { + ".": {}, + "f:failureThreshold": {}, + "f:httpGet": { + ".": {}, + "f:path": {}, + "f:port": {}, + "f:scheme": {} + }, + "f:periodSeconds": {}, + "f:successThreshold": {}, + "f:timeoutSeconds": {} + }, + "f:resources": { + ".": {}, + "f:limits": { + ".": {}, + "f:memory": {} + }, + "f:requests": { + ".": {}, + "f:cpu": {}, + "f:memory": {} + } + }, + "f:securityContext": { + ".": {}, + "f:allowPrivilegeEscalation": {}, + "f:capabilities": { + ".": {}, + "f:add": {}, + "f:drop": {} + }, + "f:readOnlyRootFilesystem": {} + }, + "f:terminationMessagePath": {}, + "f:terminationMessagePolicy": {}, + "f:volumeMounts": { + ".": {}, + "k:{\"mountPath\":\"/etc/coredns\"}": { + ".": {}, + "f:mountPath": {}, + "f:name": {}, + "f:readOnly": {} + } + } + } + }, + "f:dnsPolicy": {}, + "f:nodeSelector": {}, + "f:priorityClassName": {}, + "f:restartPolicy": {}, + "f:schedulerName": {}, + "f:securityContext": {}, + "f:serviceAccount": {}, + "f:serviceAccountName": {}, + "f:terminationGracePeriodSeconds": {}, + "f:tolerations": {}, + "f:volumes": { + ".": {}, + "k:{\"name\":\"config-volume\"}": { + ".": {}, + "f:configMap": { + ".": {}, + "f:defaultMode": {}, + "f:items": {}, + "f:name": {} + }, + "f:name": {} + } + } + } + } + } + } + }, + { + "manager": "kube-controller-manager", + "operation": "Update", + "apiVersion": "apps/v1", + "time": "2022-03-28T12:46:52Z", + "fieldsType": "FieldsV1", + "fieldsV1": { + "f:metadata": { + "f:annotations": { + ".": {}, + "f:deployment.kubernetes.io/revision": {} + } + }, + "f:status": { + "f:availableReplicas": {}, + "f:conditions": { + ".": {}, + "k:{\"type\":\"Available\"}": { + ".": {}, + "f:lastTransitionTime": {}, + "f:lastUpdateTime": {}, + "f:message": {}, + "f:reason": {}, + "f:status": {}, + "f:type": {} + }, + "k:{\"type\":\"Progressing\"}": { + ".": {}, + "f:lastTransitionTime": {}, + "f:lastUpdateTime": {}, + "f:message": {}, + "f:reason": {}, + "f:status": {}, + "f:type": {} + } + }, + "f:observedGeneration": {}, + "f:readyReplicas": {}, + "f:replicas": {}, + "f:updatedReplicas": {} + } + }, + "subresource": "status" + } + ] + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "k8s-app": "kube-dns" + } + }, + "template": { + "metadata": { + "creationTimestamp": null, + "labels": { + "k8s-app": "kube-dns" + } + }, + "spec": { + "volumes": [ + { + "name": "config-volume", + "configMap": { + "name": "coredns", + "items": [ + { + "key": "Corefile", + "path": "Corefile" + } + ], + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "coredns", + "image": "k8s.gcr.io/coredns/coredns:v1.8.6", + "args": [ + "-conf", + "/etc/coredns/Corefile" + ], + "ports": [ + { + "name": "dns", + "containerPort": 53, + "protocol": "UDP" + }, + { + "name": "dns-tcp", + "containerPort": 53, + "protocol": "TCP" + }, + { + "name": "metrics", + "containerPort": 9153, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "memory": "170Mi" + }, + "requests": { + "cpu": "100m", + "memory": "70Mi" + } + }, + "volumeMounts": [ + { + "name": "config-volume", + "readOnly": true, + "mountPath": "/etc/coredns" + } + ], + "livenessProbe": { + "httpGet": { + "path": "/health", + "port": 8080, + "scheme": "HTTP" + }, + "initialDelaySeconds": 60, + "timeoutSeconds": 5, + "periodSeconds": 10, + "successThreshold": 1, + "failureThreshold": 5 + }, + "readinessProbe": { + "httpGet": { + "path": "/ready", + "port": 8181, + "scheme": "HTTP" + }, + "timeoutSeconds": 1, + "periodSeconds": 10, + "successThreshold": 1, + "failureThreshold": 3 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent", + "securityContext": { + "capabilities": { + "add": [ + "NET_BIND_SERVICE" + ], + "drop": [ + "all" + ] + }, + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false + } + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "Default", + "nodeSelector": { + "kubernetes.io/os": "linux" + }, + "serviceAccountName": "coredns", + "serviceAccount": "coredns", + "securityContext": {}, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "key": "CriticalAddonsOnly", + "operator": "Exists" + }, + { + "key": "node-role.kubernetes.io/master", + "effect": "NoSchedule" + }, + { + "key": "node-role.kubernetes.io/control-plane", + "effect": "NoSchedule" + } + ], + "priorityClassName": "system-cluster-critical" + } + }, + "strategy": { + "type": "RollingUpdate", + "rollingUpdate": { + "maxUnavailable": 1, + "maxSurge": "25%" + } + }, + "revisionHistoryLimit": 10, + "progressDeadlineSeconds": 600 + }, + "status": { + "observedGeneration": 3, + "replicas": 1, + "updatedReplicas": 1, + "readyReplicas": 1, + "availableReplicas": 1, + "conditions": [ + { + "type": "Available", + "status": "True", + "lastUpdateTime": "2022-03-28T05:07:14Z", + "lastTransitionTime": "2022-03-28T05:07:14Z", + "reason": "MinimumReplicasAvailable", + "message": "Deployment has minimum availability." + }, + { + "type": "Progressing", + "status": "True", + "lastUpdateTime": "2022-03-28T05:07:16Z", + "lastTransitionTime": "2022-03-28T05:07:14Z", + "reason": "NewReplicaSetAvailable", + "message": "ReplicaSet \"coredns-64897985d\" has successfully progressed." + } + ] + } + } + ] +} \ No newline at end of file diff --git a/test/apps-v1.json b/test/apps-v1.json new file mode 100644 index 0000000..cf2283b --- /dev/null +++ b/test/apps-v1.json @@ -0,0 +1,199 @@ +{ + "kind": "APIResourceList", + "apiVersion": "v1", + "groupVersion": "apps/v1", + "resources": [ + { + "name": "controllerrevisions", + "singularName": "", + "namespaced": true, + "kind": "ControllerRevision", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "storageVersionHash": "85nkx63pcBU=" + }, + { + "name": "daemonsets", + "singularName": "", + "namespaced": true, + "kind": "DaemonSet", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "ds" + ], + "categories": [ + "all" + ], + "storageVersionHash": "dd7pWHUlMKQ=" + }, + { + "name": "daemonsets/status", + "singularName": "", + "namespaced": true, + "kind": "DaemonSet", + "verbs": [ + "get", + "patch", + "update" + ] + }, + { + "name": "deployments", + "singularName": "", + "namespaced": true, + "kind": "Deployment", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "deploy" + ], + "categories": [ + "all" + ], + "storageVersionHash": "8aSe+NMegvE=" + }, + { + "name": "deployments/scale", + "singularName": "", + "namespaced": true, + "group": "autoscaling", + "version": "v1", + "kind": "Scale", + "verbs": [ + "get", + "patch", + "update" + ] + }, + { + "name": "deployments/status", + "singularName": "", + "namespaced": true, + "kind": "Deployment", + "verbs": [ + "get", + "patch", + "update" + ] + }, + { + "name": "replicasets", + "singularName": "", + "namespaced": true, + "kind": "ReplicaSet", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "rs" + ], + "categories": [ + "all" + ], + "storageVersionHash": "P1RzHs8/mWQ=" + }, + { + "name": "replicasets/scale", + "singularName": "", + "namespaced": true, + "group": "autoscaling", + "version": "v1", + "kind": "Scale", + "verbs": [ + "get", + "patch", + "update" + ] + }, + { + "name": "replicasets/status", + "singularName": "", + "namespaced": true, + "kind": "ReplicaSet", + "verbs": [ + "get", + "patch", + "update" + ] + }, + { + "name": "statefulsets", + "singularName": "", + "namespaced": true, + "kind": "StatefulSet", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "sts" + ], + "categories": [ + "all" + ], + "storageVersionHash": "H+vl74LkKdo=" + }, + { + "name": "statefulsets/scale", + "singularName": "", + "namespaced": true, + "group": "autoscaling", + "version": "v1", + "kind": "Scale", + "verbs": [ + "get", + "patch", + "update" + ] + }, + { + "name": "statefulsets/status", + "singularName": "", + "namespaced": true, + "kind": "StatefulSet", + "verbs": [ + "get", + "patch", + "update" + ] + } + ] +} \ No newline at end of file diff --git a/test/apps.json b/test/apps.json new file mode 100644 index 0000000..cfb3695 --- /dev/null +++ b/test/apps.json @@ -0,0 +1,15 @@ +{ + "kind": "APIGroup", + "apiVersion": "v1", + "name": "apps", + "versions": [ + { + "groupVersion": "apps/v1", + "version": "v1" + } + ], + "preferredVersion": { + "groupVersion": "apps/v1", + "version": "v1" + } +} \ No newline at end of file diff --git a/test/fluent/plugin/in_kubernetes_objects_test.rb b/test/fluent/plugin/in_kubernetes_objects_test.rb index 687e3a5..5047e27 100644 --- a/test/fluent/plugin/in_kubernetes_objects_test.rb +++ b/test/fluent/plugin/in_kubernetes_objects_test.rb @@ -57,8 +57,9 @@ puts 'Test: should use pick the right path' ENV['KUBERNETES_SERVICE_HOST'] = k8s_host ENV['KUBERNETES_SERVICE_PORT'] = k8s_port - expect(create_input_driver(<<~CONF).instance.kubernetes_url).must_equal k8s_url('apis') - api_version apps/v1 + expect(create_input_driver(<<~CONF).instance.kubernetes_url).must_equal k8s_url('apis/apps') + api_endpoint apis/apps + api_version v1 resource_name deployments diff --git a/test/test_helper.rb b/test/test_helper.rb index 2d30474..75b8511 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -54,6 +54,9 @@ def stub_k8s_requests stub_k8s_nodes stub_k8s_pods stub_k8s_events + stub_k8s_apps + stub_k8s_apps_v1 + stub_k8s_apps_v1_namespace end def stub_k8s_api @@ -68,6 +71,24 @@ def stub_k8s_apis end.close end + def stub_k8s_apps + File.open(File.expand_path('apps.json', __dir__)).tap do |f| + stub_request(:get, k8s_url('apis/apps')).to_return(body: f.read) + end.close + end + + def stub_k8s_apps_v1 + File.open(File.expand_path('apps-v1.json', __dir__)).tap do |f| + stub_request(:get, k8s_url('apis/apps/v1')).to_return(body: f.read) + end.close + end + + def stub_k8s_apps_v1_namespace + File.open(File.expand_path('apps-v1-namespace.json', __dir__)).tap do |f| + stub_request(:get, k8s_url('apis/apps')).to_return(body: f.read) + end.close + end + def stub_k8s_v1 File.open(File.expand_path('v1.json', __dir__)).tap do |f| stub_request(:get, "#{k8s_url}/v1").to_return(body: f.read) From d07ddfe943231f0aa046bde4e74d52fcf8128e5e Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Tue, 29 Mar 2022 11:52:58 +0530 Subject: [PATCH 18/21] fixed broken yum (#86) fixed broken yum command on docker image --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0bc423a..d0ba644 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -43,7 +43,7 @@ RUN yum update -y \ && bundle install \ && bundle update i18n \ && gem uninstall -i /usr/share/gems i18n --version 1.8.11 \ - && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* glib2-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* sqlite-libs-* sqlite-devel-* sqlite-* libxml2-* libxml2-devel-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* nettle-* libsolv-* file-libs-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* json-c-* binutils-* libtiff-devel-* libtiff-* || true + && rpm -e --nodeps python3-pip-wheel python3-urllib3-* python3-requests-* python3-libxml2-* python3-dmidecode-* subscription-manager-* libwebp-* libwebp-devel-* libjpeg-turbo-devel-* libjpeg-turbo-* mariadb-connector-c-config-* mariadb-connector-c-* mariadb-connector-c-devel-* rsync-* libX11-* libX11-common-* libX11-devel-* libX11-xcb-* dbus-daemon-* tar-* qt5-srpm-macros-* perl-parent-* git-* bsdtar-* openssh-clients-* binutils-* libtiff-devel-* libtiff-* || true RUN groupadd -r $FLUENT_USER && \ useradd -r -g $FLUENT_USER $FLUENT_USER && \ From 28f01e2f130a283be335ebabda1fa3c93d556efe Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:55:39 +0530 Subject: [PATCH 19/21] 1.1.11 (#87) --- Gemfile.lock | 14 +++++++------- VERSION | 2 +- docker/Gemfile.lock | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a5850dd..2b5801b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-kubernetes-objects (1.1.10) + fluent-plugin-kubernetes-objects (1.1.11) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.9.3) @@ -11,7 +11,7 @@ GEM specs: addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) - concurrent-ruby (1.1.9) + concurrent-ruby (1.1.10) cool.io (1.7.1) crack (0.4.5) rexml @@ -22,12 +22,12 @@ GEM ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - fluentd (1.14.4) + fluentd (1.14.5) bundler cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) - serverengine (>= 2.2.2, < 3.0.0) + serverengine (>= 2.2.5, < 3.0.0) sigdump (~> 0.2.2) strptime (>= 0.2.4, < 1.0.0) tzinfo (>= 1.0, < 3.0) @@ -58,7 +58,7 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) minitest (5.14.4) - msgpack (1.4.4) + msgpack (1.4.5) multi_json (1.15.0) netrc (0.11.0) power_assert (2.0.1) @@ -85,7 +85,7 @@ GEM power_assert tzinfo (2.0.4) concurrent-ruby (~> 1.0) - tzinfo-data (1.2021.5) + tzinfo-data (1.2022.1) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext @@ -110,4 +110,4 @@ DEPENDENCIES webmock (~> 3.5) BUNDLED WITH - 2.2.30 + 2.2.32 diff --git a/VERSION b/VERSION index 9695327..5165303 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.10 \ No newline at end of file +1.1.11 \ No newline at end of file diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index bb87688..46b5a6d 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-objects (1.1.10) + fluent-plugin-kubernetes-objects (1.1.11) fluentd (>= 1.9.1) http_parser.rb (= 0.5.3) kubeclient (~> 4.9.3) @@ -43,7 +43,7 @@ GEM prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) - fluent-plugin-splunk-hec (1.2.11) + fluent-plugin-splunk-hec (1.2.12) fluentd (>= 1.4) multi_json (~> 1.13) net-http-persistent (~> 3.1) From 944ea4fb61193f2872c225de2673d51d445bd90c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Apr 2022 14:19:51 +0200 Subject: [PATCH 20/21] Bump yajl-ruby from 1.4.1 to 1.4.2 (#89) Bumps [yajl-ruby](https://github.com/brianmario/yajl-ruby) from 1.4.1 to 1.4.2. - [Release notes](https://github.com/brianmario/yajl-ruby/releases) - [Changelog](https://github.com/brianmario/yajl-ruby/blob/master/CHANGELOG.md) - [Commits](https://github.com/brianmario/yajl-ruby/commits) --- updated-dependencies: - dependency-name: yajl-ruby dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2b5801b..6bb2573 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -95,7 +95,7 @@ GEM crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) webrick (1.7.0) - yajl-ruby (1.4.1) + yajl-ruby (1.4.2) PLATFORMS ruby From f0aac243525ed876c6da194eef85c0006b4a7912 Mon Sep 17 00:00:00 2001 From: harshit-splunk <89519921+harshit-splunk@users.noreply.github.com> Date: Fri, 29 Apr 2022 12:27:36 +0530 Subject: [PATCH 21/21] 1.1.12 (#91) --- Gemfile.lock | 20 ++++++------- VERSION | 2 +- docker/Gemfile.lock | 38 ++++++++++++------------ fluent-plugin-kubernetes-objects.gemspec | 2 +- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6bb2573..67e9f1c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,9 +1,9 @@ PATH remote: . specs: - fluent-plugin-kubernetes-objects (1.1.11) + fluent-plugin-kubernetes-objects (1.1.12) fluentd (>= 1.9.1) - http_parser.rb (= 0.5.3) + http_parser.rb (= 0.8.0) kubeclient (~> 4.9.3) GEM @@ -22,7 +22,7 @@ GEM ffi-compiler (1.0.1) ffi (>= 1.0.0) rake - fluentd (1.14.5) + fluentd (1.14.6) bundler cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.9.0) @@ -46,8 +46,8 @@ GEM http-form_data (2.3.0) http-parser (1.2.3) ffi-compiler (>= 1.0, < 2.0) - http_parser.rb (0.5.3) - jsonpath (1.1.0) + http_parser.rb (0.8.0) + jsonpath (1.1.2) multi_json kubeclient (4.9.3) http (>= 3.0, < 5.0) @@ -57,12 +57,12 @@ GEM mime-types (3.4.1) mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) - minitest (5.14.4) - msgpack (1.4.5) + minitest (5.15.0) + msgpack (1.5.1) multi_json (1.15.0) netrc (0.11.0) power_assert (2.0.1) - public_suffix (4.0.6) + public_suffix (4.0.7) rake (13.0.6) recursive-open-struct (1.1.3) rest-client (2.1.0) @@ -79,9 +79,9 @@ GEM simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) - simplecov_json_formatter (0.1.3) + simplecov_json_formatter (0.1.4) strptime (0.2.5) - test-unit (3.5.0) + test-unit (3.5.3) power_assert tzinfo (2.0.4) concurrent-ruby (~> 1.0) diff --git a/VERSION b/VERSION index 9ee1f78..ccad953 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.11 +1.1.12 diff --git a/docker/Gemfile.lock b/docker/Gemfile.lock index 46b5a6d..5839e94 100644 --- a/docker/Gemfile.lock +++ b/docker/Gemfile.lock @@ -1,17 +1,17 @@ PATH remote: gem specs: - fluent-plugin-kubernetes-objects (1.1.11) + fluent-plugin-kubernetes-objects (1.1.12) fluentd (>= 1.9.1) - http_parser.rb (= 0.5.3) + http_parser.rb (= 0.8.0) kubeclient (~> 4.9.3) GEM remote: https://rubygems.org/ specs: - activemodel (7.0.0) - activesupport (= 7.0.0) - activesupport (7.0.0) + activemodel (7.0.2.4) + activesupport (= 7.0.2.4) + activesupport (7.0.2.4) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -22,7 +22,7 @@ GEM attr_required (1.0.1) bigdecimal (3.0.2) bindata (2.4.10) - concurrent-ruby (1.1.9) + concurrent-ruby (1.1.10) connection_pool (2.2.5) cool.io (1.7.1) domain_name (0.5.20190701) @@ -43,18 +43,18 @@ GEM prometheus-client (>= 2.1.0) fluent-plugin-record-modifier (2.1.0) fluentd (>= 1.0, < 2) - fluent-plugin-splunk-hec (1.2.12) + fluent-plugin-splunk-hec (1.2.13) fluentd (>= 1.4) multi_json (~> 1.13) - net-http-persistent (~> 3.1) + net-http-persistent (~> 4.0) openid_connect (~> 1.1.8) prometheus-client (>= 2.1.0) - fluentd (1.14.3) + fluentd (1.14.6) bundler cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.9.0) msgpack (>= 1.3.1, < 2.0.0) - serverengine (>= 2.2.2, < 3.0.0) + serverengine (>= 2.2.5, < 3.0.0) sigdump (~> 0.2.2) strptime (>= 0.2.4, < 1.0.0) tzinfo (>= 1.0, < 3.0) @@ -72,9 +72,9 @@ GEM http-form_data (2.3.0) http-parser (1.2.3) ffi-compiler (>= 1.0, < 2.0) - http_parser.rb (0.5.3) + http_parser.rb (0.8.0) httpclient (2.8.3) - i18n (1.8.11) + i18n (1.10.0) concurrent-ruby (~> 1.0) json-jwt (1.13.0) activesupport (>= 4.2) @@ -95,9 +95,9 @@ GEM mime-types-data (3.2022.0105) mini_mime (1.1.2) minitest (5.15.0) - msgpack (1.4.2) + msgpack (1.5.1) multi_json (1.15.0) - net-http-persistent (3.1.0) + net-http-persistent (4.0.1) connection_pool (~> 2.2) netrc (0.11.0) oj (3.10.18) @@ -111,8 +111,8 @@ GEM validate_email validate_url webfinger (>= 1.0.1) - prometheus-client (2.1.0) - public_suffix (4.0.6) + prometheus-client (4.0.0) + public_suffix (4.0.7) rack (2.2.3) rack-oauth2 (1.19.0) activesupport @@ -127,7 +127,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - serverengine (2.2.4) + serverengine (2.2.5) sigdump (~> 0.2.2) sigdump (0.2.4) strptime (0.2.5) @@ -137,7 +137,7 @@ GEM httpclient (>= 2.4) tzinfo (2.0.4) concurrent-ruby (~> 1.0) - tzinfo-data (1.2021.5) + tzinfo-data (1.2022.1) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext @@ -152,7 +152,7 @@ GEM activesupport httpclient (>= 2.4) webrick (1.7.0) - yajl-ruby (1.4.1) + yajl-ruby (1.4.2) PLATFORMS ruby diff --git a/fluent-plugin-kubernetes-objects.gemspec b/fluent-plugin-kubernetes-objects.gemspec index e115f71..1d077bc 100644 --- a/fluent-plugin-kubernetes-objects.gemspec +++ b/fluent-plugin-kubernetes-objects.gemspec @@ -23,7 +23,7 @@ Gem::Specification.new do |spec| spec.required_ruby_version = '>= 2.3.0' spec.add_runtime_dependency 'fluentd', '>= 1.9.1' spec.add_runtime_dependency 'kubeclient', '~> 4.9.3' - spec.add_runtime_dependency 'http_parser.rb', '= 0.5.3' + spec.add_runtime_dependency 'http_parser.rb', '= 0.8.0' spec.add_development_dependency 'bundler', '~> 2.0' spec.add_development_dependency 'minitest', '~> 5.11'