diff --git a/README.md b/README.md index 7dad2d6..524b729 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ Ready to use log management solution for Kubernetes. Efficiently store big amoun Status is **alpha**. However we (Flant) use it in our production Kubernetes deployments since September, 2017. -Some data may be dropped in alpha's updates. Be careful, when updating! All info will be published in release notes. -**Data's structure will be stable in beta version** (planned on April 2018). +Some data may be dropped in alpha's updates. Be careful, when updating! All info will be published in release notes. +**Data's structure will be stable in beta version** (planned on 2019). Loghouse-dashboard UI demo in action (~3 Mb): @@ -48,9 +48,11 @@ To install loghouse, you need to have [Helm](https://github.com/kubernetes/helm) ``` # helm fetch loghouse/loghouse --untar # vim loghouse/values.yaml -# helm install -n loghouse loghouse +# helm install --namespace loghouse -n loghouse loghouse ``` +Note: use `--timeout 1200` flag in case slow image pulling. + 2.2. Using specific parameters *(check variables in chart's [values.yaml](charts/loghouse/values.yaml) — not documented yet)*: ``` diff --git a/charts/loghouse/.helmignore b/charts/loghouse/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/charts/loghouse/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/loghouse/Chart.yaml b/charts/loghouse/Chart.yaml index 4588f06..6d7e74d 100644 --- a/charts/loghouse/Chart.yaml +++ b/charts/loghouse/Chart.yaml @@ -1,5 +1,6 @@ name: loghouse -version: 0.2.2 +version: 0.2.3 +appVersion: 0.2.3 description: Loghouse (Fluentd, Clickhouse, Tabix, Loghouse) for collect and prepare logs for Kubernetes cluster. keywords: @@ -8,6 +9,8 @@ keywords: - tabix - loghouse - kubernetes +home: https://github.com/flant/loghouse +icon: https://cdn.rawgit.com/flant/loghouse/master/docs/logo.png maintainers: - name: Dmitry Stolyarov email: dmitry.stolyarov@flant.com @@ -15,3 +18,6 @@ maintainers: email: andrey.sidorov@flant.com - name: Sergey Gnuskov email: sergey.gnuskov@flant.com +- name: Ivan Mikheykin + email: ivan.mikheykin@flant.com + diff --git a/charts/loghouse/templates/NOTES.txt b/charts/loghouse/templates/NOTES.txt new file mode 100644 index 0000000..f53f24f --- /dev/null +++ b/charts/loghouse/templates/NOTES.txt @@ -0,0 +1,15 @@ +LOGHOUSE chart is installed in ns/{{.Release.Namespace}}. + +Ingress: {{ if .Values.ingress.enable }}ENABLED{{else}}no{{end}} +Tabix: {{ if .Values.enable_tabix }}ENABLED{{else}}no{{end}} +Clickhouse storage: {{ if .Values.storage.pvc }}pvc/{{.Values.storage.pvc.name}}{{end}}{{if .Values.storage.hostpath}}{{.Values.storage.hostpath}}{{end}}{{if .Values.storage.emptyDir}}emptyDir{{end}} + + +{{- if not .Values.clickhouse.node_selector -}} +Production note: use clickhouse.node_selector to specify node for clickshouse server +{{- end -}} + +Use .Values.docker_path if docker logs is not in /var/lib/docker. +e.g. /dind/docker/containers if dind is used. + + diff --git a/charts/loghouse/templates/_helpers.tpl b/charts/loghouse/templates/_helpers.tpl index ee1eb6e..525e5e2 100644 --- a/charts/loghouse/templates/_helpers.tpl +++ b/charts/loghouse/templates/_helpers.tpl @@ -7,4 +7,16 @@ CronJob api version {{- else -}} "batch/v2alpha1" {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} + + +{{/* +Images version. This version can be set from cli: --set app.version=master. +*/}} +{{- define "app.version" -}} +{{- if .Values.app.version -}} +{{- .Values.app.version -}} +{{- else -}} +{{- .Chart.AppVersion -}} +{{- end -}} +{{- end -}} diff --git a/charts/loghouse/templates/clickhouse/clickhouse.yaml b/charts/loghouse/templates/clickhouse/clickhouse.yaml index d1c1e9f..a59fc87 100644 --- a/charts/loghouse/templates/clickhouse/clickhouse.yaml +++ b/charts/loghouse/templates/clickhouse/clickhouse.yaml @@ -14,21 +14,26 @@ spec: labels: component: clickhouse spec: + securityContext: + runAsUser: 105 + fsGroup: 106 initContainers: - - name: chown + - name: config image: alpine:3.6 - command: ['/bin/sh', '-c', 'chown 105:106 -R /var/lib/clickhouse'] + command: + - '/bin/sh' + - '-c' + - cd /etc/clickhouse-server; + ln -sf /etc/clickhouse-server-config/config.xml; + ln -sf /etc/clickhouse-server-config/users.xml volumeMounts: -{{- if .Values.storage.pvc }} - - name: {{ .Values.storage.pvc.name }} -{{- end }} -{{- if .Values.storage.hostpath }} - - name: hostpath -{{- end }} - mountPath: /var/lib/clickhouse + - mountPath: /etc/clickhouse-server/ + name: config-directory + - mountPath: /etc/clickhouse-server-config/ + name: config-volume containers: - name: clickhouse - image: flant/loghouse-clickhouse:{{ .Chart.Version }} + image: flant/loghouse-clickhouse:{{ template "app.version" $ }} imagePullPolicy: {{ .Values.imagePullPolicy }} ports: - name: http @@ -74,8 +79,14 @@ spec: - name: {{ .Values.storage.pvc.name }} mountPath: /var/lib/clickhouse/ {{- end }} - - name: config-volume - mountPath: /etc/clickhouse-server/ +{{- if .Values.storage.emptyDir }} + - name: data + mountPath: /var/lib/clickhouse/ +{{- end }} + - mountPath: /etc/clickhouse-server/ + name: config-directory + - mountPath: /etc/clickhouse-server-config/ + name: config-volume volumes: {{- if .Values.storage.hostpath }} - name: hostpath @@ -86,10 +97,19 @@ spec: - name: {{ .Values.storage.pvc.name }} persistentVolumeClaim: claimName: {{ .Values.storage.pvc.name }} +{{- end }} +{{- if .Values.storage.emptyDir }} + - name: data + emptyDir: + sizeLimit: "0" {{- end }} - name: config-volume configMap: name: clickhouse-config + - name: config-directory + emptyDir: + sizeLimit: "0" + {{- if .Values.tolerations }} tolerations: - operator: {{ .Values.tolerations.operator | quote }} diff --git a/charts/loghouse/templates/fluentd/fluentd.yaml b/charts/loghouse/templates/fluentd/fluentd.yaml index 8ce628f..cdb193d 100644 --- a/charts/loghouse/templates/fluentd/fluentd.yaml +++ b/charts/loghouse/templates/fluentd/fluentd.yaml @@ -19,7 +19,7 @@ spec: serviceAccountName: fluentd containers: - name: fluentd - image: flant/loghouse-fluentd:{{ .Chart.Version }} + image: flant/loghouse-fluentd:{{ template "app.version" $ }} imagePullPolicy: {{ .Values.imagePullPolicy }} env: - name: FLUENTD_ARGS diff --git a/charts/loghouse/templates/loghouse/loghouse-cronjob.yaml b/charts/loghouse/templates/loghouse/loghouse-cronjob.yaml index 58dc338..8d9928c 100644 --- a/charts/loghouse/templates/loghouse/loghouse-cronjob.yaml +++ b/charts/loghouse/templates/loghouse/loghouse-cronjob.yaml @@ -19,7 +19,7 @@ spec: command: [ '/bin/sh', '-c', 'while ! nc -z clickhouse 8123; do sleep 1; done' ] containers: - name: cron - image: flant/loghouse-dashboard:{{ .Chart.Version }} + image: flant/loghouse-dashboard:{{ template "app.version" $ }} imagePullPolicy: {{ .Values.imagePullPolicy }} command: ['/bin/bash', '-l', '-c', 'rake create_logs_tables'] env: diff --git a/charts/loghouse/templates/loghouse/loghouse-init-db.yaml b/charts/loghouse/templates/loghouse/loghouse-init-db.yaml index adbcb03..234df7a 100644 --- a/charts/loghouse/templates/loghouse/loghouse-init-db.yaml +++ b/charts/loghouse/templates/loghouse/loghouse-init-db.yaml @@ -19,7 +19,7 @@ spec: command: [ '/bin/sh', '-c', 'while ! nc -z clickhouse 8123; do sleep 1; done' ] containers: - name: init - image: flant/loghouse-fluentd:{{ .Chart.Version }} + image: flant/loghouse-fluentd:{{ template "app.version" $ }} imagePullPolicy: {{ .Values.imagePullPolicy }} command: ['/bin/bash', '-l', '-c', 'clickhouse-client --host=${CLICKHOUSE_URL} --port=9000 --user=${CLICKHOUSE_USERNAME} --password=${CLICKHOUSE_PASSWORD} --query="CREATE DATABASE ${CLICKHOUSE_DATABASE};"'] env: diff --git a/charts/loghouse/templates/loghouse/loghouse-init-tables.yaml b/charts/loghouse/templates/loghouse/loghouse-init-tables.yaml index 1fd0f65..c6a55c7 100644 --- a/charts/loghouse/templates/loghouse/loghouse-init-tables.yaml +++ b/charts/loghouse/templates/loghouse/loghouse-init-tables.yaml @@ -7,7 +7,7 @@ metadata: "helm.sh/hook": post-install,post-upgrade "helm.sh/weight": "5" spec: - activeDeadlineSeconds: 60 + activeDeadlineSeconds: 300 template: metadata: name: {{ .Chart.Name }}-init-tables @@ -19,7 +19,7 @@ spec: command: [ '/bin/sh', '-c', 'while ! nc -z clickhouse 8123; do sleep 1; done' ] containers: - name: init-tables - image: flant/loghouse-dashboard:{{ .Chart.Version }} + image: flant/loghouse-dashboard:{{ template "app.version" $ }} imagePullPolicy: {{ .Values.imagePullPolicy }} command: ['/bin/bash', '-l', '-c', 'rake create_logs_tables'] env: diff --git a/charts/loghouse/templates/loghouse/loghouse.yaml b/charts/loghouse/templates/loghouse/loghouse.yaml index 4b95633..3f93de7 100644 --- a/charts/loghouse/templates/loghouse/loghouse.yaml +++ b/charts/loghouse/templates/loghouse/loghouse.yaml @@ -37,7 +37,7 @@ spec: mountPath: /etc/nginx/nginx.conf subPath: nginx.conf - name: backend - image: flant/loghouse-dashboard:{{ .Chart.Version }} + image: flant/loghouse-dashboard:{{ template "app.version" $ }} imagePullPolicy: {{ .Values.imagePullPolicy }} command: ["bundle", "exec", "puma"] ports: diff --git a/charts/loghouse/templates/tabix/tabix.yaml b/charts/loghouse/templates/tabix/tabix.yaml index a688747..055a536 100644 --- a/charts/loghouse/templates/tabix/tabix.yaml +++ b/charts/loghouse/templates/tabix/tabix.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: tabix - image: flant/loghouse-tabix:{{ .Chart.Version }} + image: flant/loghouse-tabix:{{ template "app.version" $ }} imagePullPolicy: {{ .Values.imagePullPolicy }} ports: - name: http diff --git a/charts/loghouse/values.yaml b/charts/loghouse/values.yaml index db4c901..749c068 100644 --- a/charts/loghouse/values.yaml +++ b/charts/loghouse/values.yaml @@ -15,7 +15,7 @@ ingress: clickhouse: host: clickhouse.domain.com path: "/" - tls_secret_name: clickhouse + tls_secret_name: clickhouse loghouse: host: loghouse.domain.com path: "/" @@ -35,6 +35,7 @@ enable_tabix: false # docker_path: /data/docker storage: + emptyDir: true # You can use some storageClass or hostpath for clickhouse data storage # hostpath: /data # OR: @@ -61,15 +62,16 @@ partition_period: 1 # value: 'logging' # effect: 'NoSchedule' -# If you not want install fluentd on master +# If you not want install fluentd on master node install_master: false imagePullPolicy: Always clickhouse: - node_selector: - key: 'node-role/logging' - value: 'clickhouse' + # use nodeSelector for clickhouse to limit clickhouse nodes. + #node_selector: + # key: 'node-role/logging' + # value: 'clickhouse' server: clickhouse port: 9000 user: default