diff --git a/monitor-helm/debian/deb_folder/patches/0001-Add-Makefile-for-helm-charts.patch b/monitor-helm/debian/deb_folder/patches/0001-Add-Makefile-for-helm-charts.patch new file mode 100644 index 0000000..0744620 --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0001-Add-Makefile-for-helm-charts.patch @@ -0,0 +1,62 @@ +From c116dce67296baa7b31cf59a68102b88e008db30 Mon Sep 17 00:00:00 2001 +From: John Kung +Date: Thu, 9 May 2019 11:08:48 -0400 +Subject: [PATCH 1/1] Add Makefile for helm charts + +--- + stable/Makefile | 43 +++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 43 insertions(+) + create mode 100644 stable/Makefile + +diff --git a/stable/Makefile b/stable/Makefile +new file mode 100644 +index 0000000..5cf4447 +--- /dev/null ++++ b/stable/Makefile +@@ -0,0 +1,43 @@ ++# ++# Copyright 2017 The Openstack-Helm Authors. ++# ++# Copyright (c) 2018 Wind River Systems, Inc. ++# ++# SPDX-License-Identifier: Apache-2.0 ++# ++# It's necessary to set this because some environments don't link sh -> bash. ++SHELL := /bin/bash ++TASK := build ++ ++EXCLUDES := helm-toolkit doc tests tools logs tmp ++CHARTS := helm-toolkit $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.))) ++ ++.PHONY: $(EXCLUDES) $(CHARTS) ++ ++all: $(CHARTS) ++ ++$(CHARTS): ++ @if [ -d $@ ]; then \ ++ echo; \ ++ echo "===== Processing [$@] chart ====="; \ ++ make $(TASK)-$@; \ ++ fi ++ ++init-%: ++ if [ -f $*/Makefile ]; then make -C $*; fi ++ if [ -f $*/requirements.yaml ]; then helm dep up $*; fi ++ ++lint-%: init-% ++ if [ -d $* ]; then helm lint $*; fi ++ ++build-%: lint-% ++ if [ -d $* ]; then helm package $*; fi ++ ++clean: ++ @echo "Clean all build artifacts" ++ rm -f */templates/_partials.tpl */templates/_globals.tpl ++ rm -f *tgz */charts/*tgz */requirements.lock ++ rm -rf */charts */tmpcharts ++ ++%: ++ @: +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0002-kibana-workaround-checksum-for-configmap.yaml.patch b/monitor-helm/debian/deb_folder/patches/0002-kibana-workaround-checksum-for-configmap.yaml.patch new file mode 100644 index 0000000..e981875 --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0002-kibana-workaround-checksum-for-configmap.yaml.patch @@ -0,0 +1,28 @@ +From 2315765024f82510212604d16eed37dfd69ff24c Mon Sep 17 00:00:00 2001 +From: John Kung +Date: Thu, 2 May 2019 14:48:20 -0400 +Subject: [PATCH 1/1] kibana: workaround checksum for configmap.yaml + +--- + stable/kibana/templates/deployment.yaml | 5 ----- + 1 file changed, 5 deletions(-) + +diff --git a/stable/kibana/templates/deployment.yaml b/stable/kibana/templates/deployment.yaml +index 33208ea..33fc6f0 100644 +--- a/stable/kibana/templates/deployment.yaml ++++ b/stable/kibana/templates/deployment.yaml +@@ -16,11 +16,6 @@ spec: + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + template: + metadata: +- annotations: +- checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} +-{{- if .Values.podAnnotations }} +-{{ toYaml .Values.podAnnotations | indent 8 }} +-{{- end }} + labels: + app: {{ template "kibana.name" . }} + release: "{{ .Release.Name }}" +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0003-helm-chart-changes-for-stx-monitor.patch b/monitor-helm/debian/deb_folder/patches/0003-helm-chart-changes-for-stx-monitor.patch new file mode 100644 index 0000000..7e23cfb --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0003-helm-chart-changes-for-stx-monitor.patch @@ -0,0 +1,324 @@ +From d22ad4870b78f74298189e383697ae6163ef49d7 Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Wed, 17 Jul 2019 15:13:44 -0400 +Subject: [PATCH 1/1] helm chart changes for stx monitor + +--- + stable/elasticsearch/values.yaml | 2 +- + stable/filebeat/templates/configmap.yaml | 34 ++++++++++++++++++++++++++++++ + stable/filebeat/templates/daemonset.yaml | 15 +++++++++++++ + stable/filebeat/values.yaml | 9 ++------ + stable/kibana/values.yaml | 4 ++-- + stable/logstash/templates/ingress.yaml | 2 +- + stable/logstash/templates/statefulset.yaml | 7 +++--- + stable/logstash/values.yaml | 3 ++- + stable/metricbeat/templates/configmap.yaml | 34 ++++++++++++++++++++++++++++++ + stable/metricbeat/templates/daemonset.yaml | 26 +++++++++++++++++++++++ + stable/metricbeat/values.yaml | 4 ++-- + 11 files changed, 122 insertions(+), 18 deletions(-) + create mode 100755 stable/filebeat/templates/configmap.yaml + create mode 100755 stable/metricbeat/templates/configmap.yaml + +diff --git a/stable/elasticsearch/values.yaml b/stable/elasticsearch/values.yaml +index bbc03dd..6dcd9b0 100644 +--- a/stable/elasticsearch/values.yaml ++++ b/stable/elasticsearch/values.yaml +@@ -33,7 +33,7 @@ podSecurityPolicy: + + image: + repository: "docker.elastic.co/elasticsearch/elasticsearch-oss" +- tag: "6.7.0" ++ tag: "7.2.0" + pullPolicy: "IfNotPresent" + # If specified, use these secrets to access the image + # pullSecrets: +diff --git a/stable/filebeat/templates/configmap.yaml b/stable/filebeat/templates/configmap.yaml +new file mode 100755 +index 0000000..c895965 +--- /dev/null ++++ b/stable/filebeat/templates/configmap.yaml +@@ -0,0 +1,34 @@ ++apiVersion: v1 ++kind: ConfigMap ++metadata: ++ name: {{ template "filebeat.fullname" . }} ++ labels: ++ app: {{ template "filebeat.fullname" . }} ++ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" ++ release: "{{ .Release.Name }}" ++ heritage: "{{ .Release.Service }}" ++data: ++ setup-script.sh: |- ++ #!/bin/bash ++ BEAT='filebeat' ++ BEAT_VER=$($BEAT version | awk '{print $3}') ++ # check if logstash output is enabled, via crude parsing of the .yml file. ++ sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true ++ if [[ $? -eq 0 ]]; then ++ echo "logstash configured, exporting template to elasticsearch" ++ $BEAT export template > /tmp/beat.template.json ++ # remove the lifecycle section of the yaml, as elasticsearch will choke ++ # on it as oss version does not support ilm. ++ sed -i '/lifecycle/,+3d' /tmp/beat.template.json ++ curl -XPUT -H "Content-Type: application/json" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/_template/{$BEAT}-{$BEAT_VER} -d@/tmp/beat.template.json ++ # prime the index as a write index ++ curl -XPUT -H "Content-Type: application/json" -d "{ ++ \"aliases\": { ++ \"{BEAT}-{$BEAT_VER}\": { ++ \"is_write_index\": \"true\" ++ } ++ } ++ }" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/{$BEAT}-{$BEAT_VER}-000001 ++ else ++ echo "logstash not configured, not exporting template, should be done for us." ++ fi +diff --git a/stable/filebeat/templates/daemonset.yaml b/stable/filebeat/templates/daemonset.yaml +index 2b8e265..00a5519 100644 +--- a/stable/filebeat/templates/daemonset.yaml ++++ b/stable/filebeat/templates/daemonset.yaml +@@ -38,6 +38,21 @@ spec: + priorityClassName: "{{ .Values.priorityClassName }}" + {{- end }} + initContainers: ++ - name: "setup-script" ++ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" ++ imagePullPolicy: {{ .Values.image.pullPolicy }} ++ command: ++ - /bin/bash ++ - -c ++ - /usr/share/filebeat/setup-script.sh ++ volumeMounts: ++ - mountPath: /usr/share/filebeat/setup-script.sh ++ name: setupscript ++ subPath: setup-script.sh ++ - name: filebeat-config ++ mountPath: /usr/share/filebeat/filebeat.yml ++ readOnly: true ++ subPath: filebeat.yml + {{- if .Values.indexTemplateLoad }} + - name: "load-es-template" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +diff --git a/stable/filebeat/values.yaml b/stable/filebeat/values.yaml +index 1907ffa..b4b766f 100644 +--- a/stable/filebeat/values.yaml ++++ b/stable/filebeat/values.yaml +@@ -1,15 +1,10 @@ + image: + repository: docker.elastic.co/beats/filebeat-oss +- tag: 6.7.0 ++ tag: 7.2.0 + pullPolicy: IfNotPresent + + config: + filebeat.config: +- prospectors: +- # Mounted `filebeat-prospectors` configmap: +- path: ${path.config}/prospectors.d/*.yml +- # Reload prospectors configs as they change: +- reload.enabled: false + modules: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: +@@ -18,7 +13,7 @@ config: + processors: + - add_cloud_metadata: + +- filebeat.prospectors: ++ filebeat.inputs: + - type: log + enabled: true + paths: +diff --git a/stable/kibana/values.yaml b/stable/kibana/values.yaml +index a3310f9..4ce4b2a 100644 +--- a/stable/kibana/values.yaml ++++ b/stable/kibana/values.yaml +@@ -1,6 +1,6 @@ + image: + repository: "docker.elastic.co/kibana/kibana-oss" +- tag: "6.7.0" ++ tag: "7.2.0" + pullPolicy: "IfNotPresent" + + testFramework: +@@ -25,7 +25,7 @@ files: + ## Default Kibana configuration from kibana-docker. + server.name: kibana + server.host: "0" +- elasticsearch.url: http://elasticsearch:9200 ++ elasticsearch.hosts: http://elasticsearch:9200 + + ## Custom config properties below + ## Ref: https://www.elastic.co/guide/en/kibana/current/settings.html +diff --git a/stable/logstash/templates/ingress.yaml b/stable/logstash/templates/ingress.yaml +index d924504..0e26f89 100644 +--- a/stable/logstash/templates/ingress.yaml ++++ b/stable/logstash/templates/ingress.yaml +@@ -33,6 +33,6 @@ spec: + - path: {{ $ingressPath }} + backend: + serviceName: {{ $fullName }} +- servicePort: http ++ servicePort: 9600 + {{- end }} + {{- end }} +diff --git a/stable/logstash/templates/statefulset.yaml b/stable/logstash/templates/statefulset.yaml +index d2bbc32..cca0050 100644 +--- a/stable/logstash/templates/statefulset.yaml ++++ b/stable/logstash/templates/statefulset.yaml +@@ -25,11 +25,8 @@ spec: + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} +- annotations: +- checksum/patterns: {{ include (print $.Template.BasePath "/patterns-config.yaml") . | sha256sum }} +- checksum/templates: {{ include (print $.Template.BasePath "/files-config.yaml") . | sha256sum }} +- checksum/pipeline: {{ include (print $.Template.BasePath "/pipeline-config.yaml") . | sha256sum }} + {{- if .Values.podAnnotations }} ++ annotations: + ## Custom pod annotations + {{- range $key, $value := .Values.podAnnotations }} + {{ $key }}: {{ $value | quote }} +@@ -46,6 +43,8 @@ spec: + imagePullSecrets: + {{ toYaml .Values.image.pullSecrets | indent 8 }} + {{- end }} ++ hostNetwork: true ++ dnsPolicy: ClusterFirstWithHostNet + containers: + + ## logstash +diff --git a/stable/logstash/values.yaml b/stable/logstash/values.yaml +index a9eb158..9a452b1 100644 +--- a/stable/logstash/values.yaml ++++ b/stable/logstash/values.yaml +@@ -10,7 +10,7 @@ terminationGracePeriodSeconds: 30 + + image: + repository: docker.elastic.co/logstash/logstash-oss +- tag: 6.7.0 ++ tag: 7.2.0 + pullPolicy: IfNotPresent + ## Add secrets manually via kubectl on kubernetes cluster and reference here + # pullSecrets: +@@ -67,6 +67,7 @@ ingress: + path: / + hosts: + - logstash.cluster.local ++ servicePort: 5044 + tls: [] + # - secretName: logstash-tls + # hosts: +diff --git a/stable/metricbeat/templates/configmap.yaml b/stable/metricbeat/templates/configmap.yaml +new file mode 100755 +index 0000000..21890b4 +--- /dev/null ++++ b/stable/metricbeat/templates/configmap.yaml +@@ -0,0 +1,34 @@ ++apiVersion: v1 ++kind: ConfigMap ++metadata: ++ name: {{ template "metricbeat.fullname" . }} ++ labels: ++ app: {{ template "metricbeat.fullname" . }} ++ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" ++ release: "{{ .Release.Name }}" ++ heritage: "{{ .Release.Service }}" ++data: ++ setup-script.sh: |- ++ #!/bin/bash ++ BEAT='metricbeat' ++ BEAT_VER=$($BEAT version | awk '{print $3}') ++ # check if logstash output is enabled, via crude parsing of the .yml file. ++ sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true ++ if [[ $? -eq 0 ]]; then ++ echo "logstash configured, exporting template to elasticsearch" ++ $BEAT export template > /tmp/beat.template.json ++ # remove the lifecycle section of the yaml, as elasticsearch will choke ++ # on it as oss version does not support ilm. ++ sed -i '/lifecycle/,+3d' /tmp/beat.template.json ++ curl -XPUT -H "Content-Type: application/json" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/_template/{$BEAT}-{$BEAT_VER} -d@/tmp/beat.template.json ++ # prime the index as a write index ++ curl -XPUT -H "Content-Type: application/json" -d "{ ++ \"aliases\": { ++ \"{BEAT}-{$BEAT_VER}\": { ++ \"is_write_index\": \"true\" ++ } ++ } ++ }" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/{$BEAT}-{$BEAT_VER}-000001 ++ else ++ echo "logstash not configured, not exporting template, should be done for us." ++ fi +diff --git a/stable/metricbeat/templates/daemonset.yaml b/stable/metricbeat/templates/daemonset.yaml +index eadfb96..e3d7437 100644 +--- a/stable/metricbeat/templates/daemonset.yaml ++++ b/stable/metricbeat/templates/daemonset.yaml +@@ -32,6 +32,23 @@ spec: + {{- end }} + {{- end }} + spec: ++ initContainers: ++ - name: "setup-script" ++ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" ++ imagePullPolicy: {{ .Values.image.pullPolicy }} ++ command: ++ - /bin/bash ++ - -c ++ - /usr/share/metricbeat/setup-script.sh ++ volumeMounts: ++ - mountPath: /usr/share/metricbeat/setup-script.sh ++ name: setupscript ++ subPath: setup-script.sh ++ - name: config ++ mountPath: /usr/share/metricbeat/metricbeat.yml ++ readOnly: true ++ subPath: metricbeat.yml ++ env: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +@@ -62,6 +79,7 @@ spec: + securityContext: + runAsUser: 0 + resources: ++ + {{- if .Values.daemonset.resources }} + {{ toYaml .Values.daemonset.resources | indent 10 }} + {{- else if .Values.resources }} +@@ -85,6 +103,10 @@ spec: + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock ++#Mount the script so we can see it from running containter ++ - mountPath: /usr/share/metricbeat/setup-script.sh ++ name: setupscript ++ subPath: setup-script.sh + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | indent 8 }} + {{- end }} +@@ -108,6 +130,10 @@ spec: + - name: dockersock + hostPath: + path: /var/run/docker.sock ++ - name: setupscript ++ configMap: ++ name: {{ template "metricbeat.fullname" . }} ++ defaultMode: 0755 + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | indent 6 }} + {{- end }} +diff --git a/stable/metricbeat/values.yaml b/stable/metricbeat/values.yaml +index 5094639..2908a1b 100644 +--- a/stable/metricbeat/values.yaml ++++ b/stable/metricbeat/values.yaml +@@ -1,6 +1,6 @@ + image: +- repository: docker.elastic.co/beats/metricbeat +- tag: 6.7.0 ++ repository: docker.elastic.co/beats/metricbeat-oss ++ tag: 7.2.0 + pullPolicy: IfNotPresent + + # The instances created by daemonset retrieve most metrics from the host +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0004-ipv6-helm-chart-changes.patch b/monitor-helm/debian/deb_folder/patches/0004-ipv6-helm-chart-changes.patch new file mode 100644 index 0000000..d555dfe --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0004-ipv6-helm-chart-changes.patch @@ -0,0 +1,267 @@ +From 33fc3cb13670c8e76e7a93455865c81b9f13b0e0 Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Tue, 3 Sep 2019 10:43:50 -0400 +Subject: [PATCH 1/1] ipv6 helm chart changes + +--- + .../elasticsearch/templates/client-deployment.yaml | 2 +- + .../elasticsearch/templates/data-statefulset.yaml | 2 +- + .../templates/master-statefulset.yaml | 2 +- + stable/filebeat/templates/configmap.yaml | 47 +++++++++++++++++----- + stable/filebeat/templates/daemonset.yaml | 21 ++++++++++ + stable/metricbeat/templates/configmap.yaml | 47 +++++++++++++++++----- + stable/metricbeat/templates/daemonset.yaml | 21 ++++++++++ + 7 files changed, 119 insertions(+), 23 deletions(-) + +diff --git a/stable/elasticsearch/templates/client-deployment.yaml b/stable/elasticsearch/templates/client-deployment.yaml +index 047e3c8..5625ee0 100644 +--- a/stable/elasticsearch/templates/client-deployment.yaml ++++ b/stable/elasticsearch/templates/client-deployment.yaml +@@ -102,7 +102,7 @@ spec: + resourceFieldRef: + resource: limits.cpu + - name: ES_JAVA_OPTS +- value: "-Djava.net.preferIPv4Stack=true -Xms{{ .Values.client.heapSize }} -Xmx{{ .Values.client.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.client.additionalJavaOpts }}" ++ value: "-Djava.net.preferIPv6Addresses=true -Xms{{ .Values.client.heapSize }} -Xmx{{ .Values.client.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.client.additionalJavaOpts }}" + {{- range $key, $value := .Values.cluster.env }} + - name: {{ $key }} + value: {{ $value | quote }} +diff --git a/stable/elasticsearch/templates/data-statefulset.yaml b/stable/elasticsearch/templates/data-statefulset.yaml +index a407803..44b6be1 100644 +--- a/stable/elasticsearch/templates/data-statefulset.yaml ++++ b/stable/elasticsearch/templates/data-statefulset.yaml +@@ -120,7 +120,7 @@ spec: + resourceFieldRef: + resource: limits.cpu + - name: ES_JAVA_OPTS +- value: "-Djava.net.preferIPv4Stack=true -Xms{{ .Values.data.heapSize }} -Xmx{{ .Values.data.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.data.additionalJavaOpts }}" ++ value: "-Djava.net.preferIPv6Addresses=true -Xms{{ .Values.data.heapSize }} -Xmx{{ .Values.data.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.data.additionalJavaOpts }}" + {{- range $key, $value := .Values.cluster.env }} + - name: {{ $key }} + value: {{ $value | quote }} +diff --git a/stable/elasticsearch/templates/master-statefulset.yaml b/stable/elasticsearch/templates/master-statefulset.yaml +index 048535c..7c48fe3 100644 +--- a/stable/elasticsearch/templates/master-statefulset.yaml ++++ b/stable/elasticsearch/templates/master-statefulset.yaml +@@ -124,7 +124,7 @@ spec: + resourceFieldRef: + resource: limits.cpu + - name: ES_JAVA_OPTS +- value: "-Djava.net.preferIPv4Stack=true -Xms{{ .Values.master.heapSize }} -Xmx{{ .Values.master.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.master.additionalJavaOpts }}" ++ value: "-Djava.net.preferIPv6Addresses=true -Xms{{ .Values.master.heapSize }} -Xmx{{ .Values.master.heapSize }} {{ .Values.cluster.additionalJavaOpts }} {{ .Values.master.additionalJavaOpts }}" + {{- range $key, $value := .Values.cluster.env }} + - name: {{ $key }} + value: {{ $value | quote }} +diff --git a/stable/filebeat/templates/configmap.yaml b/stable/filebeat/templates/configmap.yaml +index c895965..9433176 100644 +--- a/stable/filebeat/templates/configmap.yaml ++++ b/stable/filebeat/templates/configmap.yaml +@@ -10,8 +10,41 @@ metadata: + data: + setup-script.sh: |- + #!/bin/bash ++ # ++ # This is best effort to load the template into elasticsearch ++ # if beats are going to elasticsearch via logstash. ++ # + BEAT='filebeat' + BEAT_VER=$($BEAT version | awk '{print $3}') ++ ++ ADDR=$OUTPUT_ELASTICSEARCH_HOSTS ++ echo $ADDR ++ ++ # Check if this is a map of entries. If so, we'll only export the template ++ # to the first address ++ if [[ ${ADDR:0:1} == [ ]]; then ++ # Remove the square brackets ++ ADDR=${ADDR:1:-1} ++ # Take the first entry in case there are more than one ++ ADDR=${ADDR%%,*} ++ fi ++ echo $ADDR ++ ++ # Check if user has formatted with http:// on front, if not we need to add ++ HTTP='http://' ++ if [[ ${ADDR} == http* ]]; then ++ HTTP='' ++ fi ++ echo $HTTP ++ ++ # Check for ip address containing special characters where -g must be used in curl command ++ # IPV6 Addresses should come in with square braces around the address. ++ CURL_G='' ++ if [[ ${ADDR} == *[* ]]; then ++ CURL_G=' -g ' ++ fi ++ echo $CURL_G ++ + # check if logstash output is enabled, via crude parsing of the .yml file. + sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true + if [[ $? -eq 0 ]]; then +@@ -20,15 +53,9 @@ data: + # remove the lifecycle section of the yaml, as elasticsearch will choke + # on it as oss version does not support ilm. + sed -i '/lifecycle/,+3d' /tmp/beat.template.json +- curl -XPUT -H "Content-Type: application/json" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/_template/{$BEAT}-{$BEAT_VER} -d@/tmp/beat.template.json +- # prime the index as a write index +- curl -XPUT -H "Content-Type: application/json" -d "{ +- \"aliases\": { +- \"{BEAT}-{$BEAT_VER}\": { +- \"is_write_index\": \"true\" +- } +- } +- }" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/{$BEAT}-{$BEAT_VER}-000001 ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json + else +- echo "logstash not configured, not exporting template, should be done for us." ++ echo "logstash not configured, not exporting template, should be done for us." + fi ++ # Above is non-fatal if there is a problem. Always pass. ++ exit 0 +diff --git a/stable/filebeat/templates/daemonset.yaml b/stable/filebeat/templates/daemonset.yaml +index 00a5519..b8536fb 100644 +--- a/stable/filebeat/templates/daemonset.yaml ++++ b/stable/filebeat/templates/daemonset.yaml +@@ -53,6 +53,17 @@ spec: + mountPath: /usr/share/filebeat/filebeat.yml + readOnly: true + subPath: filebeat.yml ++ env: ++ ## make output.elasticsearch parms visible ++ {{- range $key, $value := .Values.config}} ++ {{- $configname := $key | upper | replace "." "_" -}} ++ {{ if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} + {{- if .Values.indexTemplateLoad }} + - name: "load-es-template" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +@@ -95,6 +106,16 @@ spec: + valueFrom: + fieldRef: + fieldPath: spec.nodeName ++ ## make output.elasticsearch parms visible ++ {{- range $key, $value := .Values.config}} ++ {{- $configname := $key | upper | replace "." "_" -}} ++ {{ if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} + {{- if .Values.extraVars }} + {{ toYaml .Values.extraVars | indent 8 }} + {{- end }} +diff --git a/stable/metricbeat/templates/configmap.yaml b/stable/metricbeat/templates/configmap.yaml +index 21890b4..f990f0c 100644 +--- a/stable/metricbeat/templates/configmap.yaml ++++ b/stable/metricbeat/templates/configmap.yaml +@@ -10,8 +10,41 @@ metadata: + data: + setup-script.sh: |- + #!/bin/bash ++ # ++ # This is best effort to load the template into elasticsearch ++ # if beats are going to elasticsearch via logstash. ++ # + BEAT='metricbeat' + BEAT_VER=$($BEAT version | awk '{print $3}') ++ ++ ADDR=$OUTPUT_ELASTICSEARCH_HOSTS ++ echo $ADDR ++ ++ # Check if this is a map of entries. If so, we'll only export the template ++ # to the first address ++ if [[ ${ADDR:0:1} == [ ]]; then ++ # Remove the square brackets ++ ADDR=${ADDR:1:-1} ++ # Take the first entry in case there are more than one ++ ADDR=${ADDR%%,*} ++ fi ++ echo $ADDR ++ ++ # Check if user has formatted with http:// on front, if not we need to add ++ HTTP='http://' ++ if [[ ${ADDR} == http* ]]; then ++ HTTP='' ++ fi ++ echo $HTTP ++ ++ # Check for ip address containing special characters where -g must be used in curl command ++ # IPV6 Addresses should come in with square braces around the address. ++ CURL_G='' ++ if [[ ${ADDR} == *[* ]]; then ++ CURL_G=' -g ' ++ fi ++ echo $CURL_G ++ + # check if logstash output is enabled, via crude parsing of the .yml file. + sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true + if [[ $? -eq 0 ]]; then +@@ -20,15 +53,9 @@ data: + # remove the lifecycle section of the yaml, as elasticsearch will choke + # on it as oss version does not support ilm. + sed -i '/lifecycle/,+3d' /tmp/beat.template.json +- curl -XPUT -H "Content-Type: application/json" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/_template/{$BEAT}-{$BEAT_VER} -d@/tmp/beat.template.json +- # prime the index as a write index +- curl -XPUT -H "Content-Type: application/json" -d "{ +- \"aliases\": { +- \"{BEAT}-{$BEAT_VER}\": { +- \"is_write_index\": \"true\" +- } +- } +- }" http://{$STX_ELASTICSEARCH_CLIENT_SERVICE_HOST}:{$STX_ELASTICSEARCH_CLIENT_SERVICE_PORT_HTTP}/{$BEAT}-{$BEAT_VER}-000001 ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json + else +- echo "logstash not configured, not exporting template, should be done for us." ++ echo "logstash not configured, not exporting template, should be done for us." + fi ++ # Above is non-fatal if there is a problem. Always pass. ++ exit 0 +diff --git a/stable/metricbeat/templates/daemonset.yaml b/stable/metricbeat/templates/daemonset.yaml +index e3d7437..1ae3a34 100644 +--- a/stable/metricbeat/templates/daemonset.yaml ++++ b/stable/metricbeat/templates/daemonset.yaml +@@ -49,6 +49,17 @@ spec: + readOnly: true + subPath: metricbeat.yml + env: ++ ## make output.elasticsearch parms visible ++ {{- range $key, $value := .Values.daemonset.config}} ++ {{- $configname := $key | upper | replace "." "_" -}} ++ {{ if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} ++ + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +@@ -73,6 +84,16 @@ spec: + valueFrom: + fieldRef: + fieldPath: spec.nodeName ++ ## make output.elasticsearch parms visible ++ {{- range $key, $value := .Values.daemonset.config}} ++ {{- $configname := $key | upper | replace "." "_" -}} ++ {{ if eq $configname "OUTPUT_ELASTICSEARCH" -}} ++ {{- range $key2, $value2 := $value}} ++ - name: "{{ $configname }}_{{ $key2 | upper }}" ++ value: {{ $value2 | quote }} ++ {{- end }} ++ {{- end }} ++ {{- end }} + {{- if .Values.extraEnv }} + {{ toYaml .Values.extraEnv | indent 8 }} + {{- end }} +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0005-decouple-config.patch b/monitor-helm/debian/deb_folder/patches/0005-decouple-config.patch new file mode 100644 index 0000000..718e0f7 --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0005-decouple-config.patch @@ -0,0 +1,97 @@ +From ac9bce2e01870d8623dc7910cd68079d56f917ca Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Wed, 11 Sep 2019 18:07:50 -0400 +Subject: [PATCH 1/1] decouple config + +--- + stable/elasticsearch/templates/configmap.yaml | 8 ++++++++ + stable/elasticsearch/values.yaml | 1 + + stable/filebeat/templates/configmap.yaml | 2 +- + stable/metricbeat/templates/configmap.yaml | 2 +- + stable/metricbeat/values.yaml | 6 ++++-- + 5 files changed, 15 insertions(+), 4 deletions(-) + +diff --git a/stable/elasticsearch/templates/configmap.yaml b/stable/elasticsearch/templates/configmap.yaml +index 977ee9a..6bc2065 100644 +--- a/stable/elasticsearch/templates/configmap.yaml ++++ b/stable/elasticsearch/templates/configmap.yaml +@@ -87,6 +87,14 @@ data: + gateway.recover_after_time: ${RECOVER_AFTER_TIME:5m} + gateway.recover_after_master_nodes: ${RECOVER_AFTER_MASTER_NODES:2} + gateway.recover_after_data_nodes: ${RECOVER_AFTER_DATA_NODES:1} ++ ++{{- if .Values.cluster.auto_config_initial_masters }} ++ cluster.initial_master_nodes: ++ {{- range $i := until (.Values.master.replicas|int) }} ++ - {{template "elasticsearch.master.fullname" $ }}-{{ $i }} ++ {{- end }} ++{{- end }} ++ + {{- with .Values.cluster.config }} + {{ toYaml . | indent 4 }} + {{- end }} +diff --git a/stable/elasticsearch/values.yaml b/stable/elasticsearch/values.yaml +index 6dcd9b0..37321dc 100644 +--- a/stable/elasticsearch/values.yaml ++++ b/stable/elasticsearch/values.yaml +@@ -70,6 +70,7 @@ cluster: + plugins: [] + # - ingest-attachment + # - mapper-size ++ auto_config_initial_masters: true + + client: + name: client +diff --git a/stable/filebeat/templates/configmap.yaml b/stable/filebeat/templates/configmap.yaml +index 9433176..8377e5a 100644 +--- a/stable/filebeat/templates/configmap.yaml ++++ b/stable/filebeat/templates/configmap.yaml +@@ -22,7 +22,7 @@ data: + + # Check if this is a map of entries. If so, we'll only export the template + # to the first address +- if [[ ${ADDR:0:1} == [ ]]; then ++ if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then + # Remove the square brackets + ADDR=${ADDR:1:-1} + # Take the first entry in case there are more than one +diff --git a/stable/metricbeat/templates/configmap.yaml b/stable/metricbeat/templates/configmap.yaml +index f990f0c..4249e7f 100644 +--- a/stable/metricbeat/templates/configmap.yaml ++++ b/stable/metricbeat/templates/configmap.yaml +@@ -22,7 +22,7 @@ data: + + # Check if this is a map of entries. If so, we'll only export the template + # to the first address +- if [[ ${ADDR:0:1} == [ ]]; then ++ if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then + # Remove the square brackets + ADDR=${ADDR:1:-1} + # Take the first entry in case there are more than one +diff --git a/stable/metricbeat/values.yaml b/stable/metricbeat/values.yaml +index 2908a1b..c8f758c 100644 +--- a/stable/metricbeat/values.yaml ++++ b/stable/metricbeat/values.yaml +@@ -113,7 +113,7 @@ deployment: + # Uncomment this to get k8s events: + # - event + period: 10s +- hosts: ["kube-state-metrics:8080"] ++ hosts: ["${KUBE_STATE_METRICS_HOST}:8080"] + # If overrideModules is not empty, metricbeat chart's default modules won't be used at all. + overrideModules: {} + +@@ -122,7 +122,9 @@ plugins: [] + # - kinesis.so + + # additional environment +-# extraEnv: ++extraEnv: ++ - name: KUBE_STATE_METRICS_HOST ++ value: "mon-kube-state-metrics" + # - name: test1 + # value: "test1" + # - name: test2 +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0006-add-system-info.patch b/monitor-helm/debian/deb_folder/patches/0006-add-system-info.patch new file mode 100644 index 0000000..9b8abbb --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0006-add-system-info.patch @@ -0,0 +1,317 @@ +From 4f42518e2e15a65e340e15029c50b53b20e119e9 Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Sat, 14 Sep 2019 12:10:44 -0400 +Subject: [PATCH 1/1] add system info + +--- + stable/filebeat/templates/configmap.yaml | 22 ++++++++++++++--- + stable/filebeat/templates/daemonset.yaml | 36 ++++++++++++++++++++++++++++ + stable/filebeat/values.yaml | 4 ++++ + stable/logstash/templates/statefulset.yaml | 10 ++++++++ + stable/logstash/values.yaml | 3 +++ + stable/metricbeat/templates/configmap.yaml | 22 ++++++++++++++--- + stable/metricbeat/templates/daemonset.yaml | 37 ++++++++++++++++++++++++++++- + stable/metricbeat/templates/deployment.yaml | 18 ++++++++++++++ + stable/metricbeat/values.yaml | 4 ++++ + 9 files changed, 149 insertions(+), 7 deletions(-) + +diff --git a/stable/filebeat/templates/configmap.yaml b/stable/filebeat/templates/configmap.yaml +index 8377e5a..c08a4bc 100644 +--- a/stable/filebeat/templates/configmap.yaml ++++ b/stable/filebeat/templates/configmap.yaml +@@ -18,7 +18,9 @@ data: + BEAT_VER=$($BEAT version | awk '{print $3}') + + ADDR=$OUTPUT_ELASTICSEARCH_HOSTS +- echo $ADDR ++ ESPATH=$OUTPUT_ELASTICSEARCH_PATH ++ ++ echo $ADDR$ESPATH + + # Check if this is a map of entries. If so, we'll only export the template + # to the first address +@@ -28,7 +30,7 @@ data: + # Take the first entry in case there are more than one + ADDR=${ADDR%%,*} + fi +- echo $ADDR ++ echo $ADDR$ESPATH + + # Check if user has formatted with http:// on front, if not we need to add + HTTP='http://' +@@ -53,7 +55,21 @@ data: + # remove the lifecycle section of the yaml, as elasticsearch will choke + # on it as oss version does not support ilm. + sed -i '/lifecycle/,+3d' /tmp/beat.template.json +- curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json ++ ++ # "unset" is the special system name when none has been set through ++ # overrides.. We key on that to know what template name and index pattern to set ++ if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then ++ # replace the standard index pattern with one including our system name, ++ # which will match our created indices ++ sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json ++ # give the template a name with the system name appended. ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json ++ else ++ # apply a higher order to this template in case there are templates with system names ++ # which should be applied first ++ sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json ++ fi + else + echo "logstash not configured, not exporting template, should be done for us." + fi +diff --git a/stable/filebeat/templates/daemonset.yaml b/stable/filebeat/templates/daemonset.yaml +index b8536fb..b1833a9 100644 +--- a/stable/filebeat/templates/daemonset.yaml ++++ b/stable/filebeat/templates/daemonset.yaml +@@ -64,6 +64,24 @@ spec: + {{- end }} + {{- end }} + {{- end }} ++ ## make system name and index parms visible ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}" ++ {{- end }} + {{- if .Values.indexTemplateLoad }} + - name: "load-es-template" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +@@ -116,6 +134,24 @@ spec: + {{- end }} + {{- end }} + {{- end }} ++ ## make system name and index parms visible ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}" ++ {{- end }} + {{- if .Values.extraVars }} + {{ toYaml .Values.extraVars | indent 8 }} + {{- end }} +diff --git a/stable/filebeat/values.yaml b/stable/filebeat/values.yaml +index b4b766f..3897c82 100644 +--- a/stable/filebeat/values.yaml ++++ b/stable/filebeat/values.yaml +@@ -68,6 +68,10 @@ extraVars: [] + # name: configmap + # key: config.key + ++# Will be used in index name ++systemNameForIndex: "" ++indexNamePrefix: filebeat-%{[agent.version]} ++ + # Add additional volumes and mounts, for example to read other log files on the host + extraVolumes: [] + # - hostPath: +diff --git a/stable/logstash/templates/statefulset.yaml b/stable/logstash/templates/statefulset.yaml +index cca0050..070ef56 100644 +--- a/stable/logstash/templates/statefulset.yaml ++++ b/stable/logstash/templates/statefulset.yaml +@@ -71,6 +71,16 @@ spec: + value: {{ .Values.elasticsearch.host | quote }} + - name: ELASTICSEARCH_PORT + value: {{ .Values.elasticsearch.port | quote }} ++ ## make system name visible for in setting index. ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "" ++ {{- end }} + ## Additional env vars + {{- range $key, $value := .Values.config }} + - name: {{ $key | upper | replace "." "_" }} +diff --git a/stable/logstash/values.yaml b/stable/logstash/values.yaml +index 9a452b1..9ba80c9 100644 +--- a/stable/logstash/values.yaml ++++ b/stable/logstash/values.yaml +@@ -220,6 +220,9 @@ patterns: + # main: |- + # TESTING {"foo":.*}$ + ++# Will be used in index names ++systemNameForIndex: "" ++ + ## Custom files that can be referenced by plugins. + ## Each YAML heredoc will become located in the logstash home directory under + ## the files subdirectory. +diff --git a/stable/metricbeat/templates/configmap.yaml b/stable/metricbeat/templates/configmap.yaml +index 4249e7f..9ae276d 100644 +--- a/stable/metricbeat/templates/configmap.yaml ++++ b/stable/metricbeat/templates/configmap.yaml +@@ -18,7 +18,9 @@ data: + BEAT_VER=$($BEAT version | awk '{print $3}') + + ADDR=$OUTPUT_ELASTICSEARCH_HOSTS +- echo $ADDR ++ ESPATH=$OUTPUT_ELASTICSEARCH_PATH ++ ++ echo $ADDR$ESPATH + + # Check if this is a map of entries. If so, we'll only export the template + # to the first address +@@ -28,7 +30,7 @@ data: + # Take the first entry in case there are more than one + ADDR=${ADDR%%,*} + fi +- echo $ADDR ++ echo $ADDR$ESPATH + + # Check if user has formatted with http:// on front, if not we need to add + HTTP='http://' +@@ -53,7 +55,21 @@ data: + # remove the lifecycle section of the yaml, as elasticsearch will choke + # on it as oss version does not support ilm. + sed -i '/lifecycle/,+3d' /tmp/beat.template.json +- curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json ++ ++ # "unset" is the special system name when none has been set through ++ # overrides.. We key on that to know what template name and index pattern to set ++ if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then ++ # replace the standard index pattern with one including our system name, ++ # which will match our created indices ++ sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json ++ # give the template a name with the system name appended. ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json ++ else ++ # apply a higher order to this template in case there are templates with system names ++ # which should be applied first ++ sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json ++ fi + else + echo "logstash not configured, not exporting template, should be done for us." + fi +diff --git a/stable/metricbeat/templates/daemonset.yaml b/stable/metricbeat/templates/daemonset.yaml +index 1ae3a34..947e501 100644 +--- a/stable/metricbeat/templates/daemonset.yaml ++++ b/stable/metricbeat/templates/daemonset.yaml +@@ -59,7 +59,24 @@ spec: + {{- end }} + {{- end }} + {{- end }} +- ++ ## make system name and index parms visible ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}" ++ {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +@@ -94,6 +111,24 @@ spec: + {{- end }} + {{- end }} + {{- end }} ++ ## make system name and index parms visible ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}" ++ {{- end }} + {{- if .Values.extraEnv }} + {{ toYaml .Values.extraEnv | indent 8 }} + {{- end }} +diff --git a/stable/metricbeat/templates/deployment.yaml b/stable/metricbeat/templates/deployment.yaml +index 47ada04..d1d08e3 100644 +--- a/stable/metricbeat/templates/deployment.yaml ++++ b/stable/metricbeat/templates/deployment.yaml +@@ -55,6 +55,24 @@ spec: + valueFrom: + fieldRef: + fieldPath: spec.nodeName ++ ## make system name and index parms visible ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}-{{$systemNameFormatted}}" ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "unset" ++ - name: INDEX_PATTERN ++ value: "{{ .Values.indexNamePrefix }}-*" ++ - name: INDEX_NAME ++ value: "{{ .Values.indexNamePrefix }}" ++ {{- end }} + {{- if .Values.extraEnv }} + {{ toYaml .Values.extraEnv | indent 8 }} + {{- end }} +diff --git a/stable/metricbeat/values.yaml b/stable/metricbeat/values.yaml +index c8f758c..b59c428 100644 +--- a/stable/metricbeat/values.yaml ++++ b/stable/metricbeat/values.yaml +@@ -130,6 +130,10 @@ extraEnv: + # - name: test2 + # value: "test2" + ++# Will be used in index name ++systemNameForIndex: "" ++indexNamePrefix: metricbeat-%{[agent.version]} ++ + # Add additional volumes and mounts, for example to read other log files on the host + extraVolumes: [] + # - hostPath: +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0007-three-masters.patch b/monitor-helm/debian/deb_folder/patches/0007-three-masters.patch new file mode 100644 index 0000000..7714545 --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0007-three-masters.patch @@ -0,0 +1,40 @@ +From 876e674e26400dcbbee5ee52b6cd7c9b430063f7 Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Mon, 23 Sep 2019 13:29:06 -0400 +Subject: [PATCH 1/1] three masters + +--- + stable/elasticsearch/templates/data-statefulset.yaml | 2 +- + stable/elasticsearch/values.yaml | 3 +++ + 2 files changed, 4 insertions(+), 1 deletion(-) + +diff --git a/stable/elasticsearch/templates/data-statefulset.yaml b/stable/elasticsearch/templates/data-statefulset.yaml +index 44b6be1..69b54cd 100644 +--- a/stable/elasticsearch/templates/data-statefulset.yaml ++++ b/stable/elasticsearch/templates/data-statefulset.yaml +@@ -114,7 +114,7 @@ spec: + - name: DISCOVERY_SERVICE + value: {{ template "elasticsearch.fullname" . }}-discovery + - name: NODE_MASTER +- value: "false" ++ value: {{ .Values.data.combinedDataAndMaster | quote }} + - name: PROCESSORS + valueFrom: + resourceFieldRef: +diff --git a/stable/elasticsearch/values.yaml b/stable/elasticsearch/values.yaml +index 37321dc..a4b65b7 100644 +--- a/stable/elasticsearch/values.yaml ++++ b/stable/elasticsearch/values.yaml +@@ -222,6 +222,9 @@ data: + drain: # drain the node before stopping it and re-integrate it into the cluster after start + enabled: true + ++ # If below is true, data nodes will also be master capable ++ combinedDataAndMaster: false ++ + ## Sysctl init container to setup vm.max_map_count + # see https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html + # and https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration-memory.html#mlockall +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0008-Update-stx-monitor-for-kubernetes-API-1.16.patch b/monitor-helm/debian/deb_folder/patches/0008-Update-stx-monitor-for-kubernetes-API-1.16.patch new file mode 100644 index 0000000..4795ba4 --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0008-Update-stx-monitor-for-kubernetes-API-1.16.patch @@ -0,0 +1,316 @@ +From f82b9b85ac03f59fd1ce9b7eb0a9ed2afac47995 Mon Sep 17 00:00:00 2001 +From: John Kung +Date: Sun, 6 Oct 2019 12:50:45 -0400 +Subject: [PATCH 1/1] 0008-Update-stx-monitor-for-kubernetes-API-1.16 + +--- + stable/elasticsearch/templates/client-deployment.yaml | 6 +++++- + stable/elasticsearch/templates/client-ingress.yaml | 2 +- + stable/elasticsearch/templates/data-statefulset.yaml | 7 ++++++- + stable/elasticsearch/templates/master-statefulset.yaml | 7 ++++++- + stable/elasticsearch/templates/podsecuritypolicy.yaml | 2 +- + stable/filebeat/templates/daemonset.yaml | 2 +- + stable/filebeat/templates/podsecuritypolicy.yaml | 2 +- + stable/kibana/templates/deployment.yaml | 6 +++++- + stable/kibana/templates/ingress.yaml | 2 +- + stable/kube-state-metrics/templates/deployment.yaml | 5 ++++- + stable/kube-state-metrics/templates/podsecuritypolicy.yaml | 2 +- + stable/logstash/templates/ingress.yaml | 2 +- + stable/logstash/templates/statefulset.yaml | 2 +- + stable/metricbeat/templates/podsecuritypolicy.yaml | 2 +- + stable/nginx-ingress/templates/controller-daemonset.yaml | 6 +++++- + stable/nginx-ingress/templates/controller-deployment.yaml | 6 +++++- + stable/nginx-ingress/templates/default-backend-deployment.yaml | 6 +++++- + stable/nginx-ingress/templates/podsecuritypolicy.yaml | 2 +- + 18 files changed, 51 insertions(+), 18 deletions(-) + +diff --git a/stable/elasticsearch/templates/client-deployment.yaml b/stable/elasticsearch/templates/client-deployment.yaml +index 5625ee0..8bd1b15 100644 +--- a/stable/elasticsearch/templates/client-deployment.yaml ++++ b/stable/elasticsearch/templates/client-deployment.yaml +@@ -1,4 +1,4 @@ +-apiVersion: apps/v1beta1 ++apiVersion: apps/v1 + kind: Deployment + metadata: + labels: +@@ -9,6 +9,10 @@ metadata: + release: {{ .Release.Name }} + name: {{ template "elasticsearch.client.fullname" . }} + spec: ++ selector: ++ matchLabels: ++ app: {{ template "elasticsearch.name" . }} ++ release: {{ .Release.Name }} + replicas: {{ .Values.client.replicas }} + template: + metadata: +diff --git a/stable/elasticsearch/templates/client-ingress.yaml b/stable/elasticsearch/templates/client-ingress.yaml +index 9070cdd..8eb4b9a 100644 +--- a/stable/elasticsearch/templates/client-ingress.yaml ++++ b/stable/elasticsearch/templates/client-ingress.yaml +@@ -1,7 +1,7 @@ + {{- if .Values.client.ingress.enabled -}} + {{- $fullName := include "elasticsearch.client.fullname" . -}} + {{- $ingressPath := .Values.client.ingress.path -}} +-apiVersion: extensions/v1beta1 ++apiVersion: networking.k8s.io/v1beta1 + kind: Ingress + metadata: + name: {{ $fullName }} +diff --git a/stable/elasticsearch/templates/data-statefulset.yaml b/stable/elasticsearch/templates/data-statefulset.yaml +index 69b54cd..ccf7ad2 100644 +--- a/stable/elasticsearch/templates/data-statefulset.yaml ++++ b/stable/elasticsearch/templates/data-statefulset.yaml +@@ -1,4 +1,4 @@ +-apiVersion: apps/v1beta1 ++apiVersion: apps/v1 + kind: StatefulSet + metadata: + labels: +@@ -11,6 +11,11 @@ metadata: + spec: + serviceName: {{ template "elasticsearch.data.fullname" . }} + replicas: {{ .Values.data.replicas }} ++ selector: ++ matchLabels: ++ app: {{ template "elasticsearch.name" . }} ++ component: "{{ .Values.data.name }}" ++ release: {{ .Release.Name }} + template: + metadata: + labels: +diff --git a/stable/elasticsearch/templates/master-statefulset.yaml b/stable/elasticsearch/templates/master-statefulset.yaml +index 7c48fe3..aa7014d 100644 +--- a/stable/elasticsearch/templates/master-statefulset.yaml ++++ b/stable/elasticsearch/templates/master-statefulset.yaml +@@ -1,4 +1,4 @@ +-apiVersion: apps/v1beta1 ++apiVersion: apps/v1 + kind: StatefulSet + metadata: + labels: +@@ -11,6 +11,11 @@ metadata: + spec: + serviceName: {{ template "elasticsearch.master.fullname" . }} + replicas: {{ .Values.master.replicas }} ++ selector: ++ matchLabels: ++ app: {{ template "elasticsearch.name" . }} ++ component: "{{ .Values.master.name }}" ++ release: {{ .Release.Name }} + template: + metadata: + labels: +diff --git a/stable/elasticsearch/templates/podsecuritypolicy.yaml b/stable/elasticsearch/templates/podsecuritypolicy.yaml +index ee38e35..fd5f663 100644 +--- a/stable/elasticsearch/templates/podsecuritypolicy.yaml ++++ b/stable/elasticsearch/templates/podsecuritypolicy.yaml +@@ -1,5 +1,5 @@ + {{- if .Values.podSecurityPolicy.enabled }} +-apiVersion: extensions/v1beta1 ++apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: {{ template "elasticsearch.fullname" . }} +diff --git a/stable/filebeat/templates/daemonset.yaml b/stable/filebeat/templates/daemonset.yaml +index b1833a9..22b46df 100644 +--- a/stable/filebeat/templates/daemonset.yaml ++++ b/stable/filebeat/templates/daemonset.yaml +@@ -1,4 +1,4 @@ +-apiVersion: apps/v1beta2 ++apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: {{ template "filebeat.fullname" . }} +diff --git a/stable/filebeat/templates/podsecuritypolicy.yaml b/stable/filebeat/templates/podsecuritypolicy.yaml +index 3a11b38..25a0f06 100644 +--- a/stable/filebeat/templates/podsecuritypolicy.yaml ++++ b/stable/filebeat/templates/podsecuritypolicy.yaml +@@ -1,6 +1,6 @@ + {{- if .Values.rbac.create -}} + {{- if .Values.podSecurityPolicy.enabled }} +-apiVersion: extensions/v1beta1 ++apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: {{ template "filebeat.fullname" . }} +diff --git a/stable/kibana/templates/deployment.yaml b/stable/kibana/templates/deployment.yaml +index 33fc6f0..c5229ae 100644 +--- a/stable/kibana/templates/deployment.yaml ++++ b/stable/kibana/templates/deployment.yaml +@@ -1,4 +1,4 @@ +-apiVersion: apps/v1beta1 ++apiVersion: apps/v1 + kind: Deployment + metadata: + labels: +@@ -13,6 +13,10 @@ metadata: + {{- end }} + spec: + replicas: {{ .Values.replicaCount }} ++ selector: ++ matchLabels: ++ app: {{ template "kibana.name" . }} ++ release: {{ .Release.Name }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + template: + metadata: +diff --git a/stable/kibana/templates/ingress.yaml b/stable/kibana/templates/ingress.yaml +index de14ae9..9d97e26 100644 +--- a/stable/kibana/templates/ingress.yaml ++++ b/stable/kibana/templates/ingress.yaml +@@ -1,7 +1,7 @@ + {{- if .Values.ingress.enabled -}} + {{- $serviceName := include "kibana.fullname" . -}} + {{- $servicePort := .Values.service.externalPort -}} +-apiVersion: extensions/v1beta1 ++apiVersion: networking.k8s.io/v1beta1 + kind: Ingress + metadata: + labels: +diff --git a/stable/kube-state-metrics/templates/deployment.yaml b/stable/kube-state-metrics/templates/deployment.yaml +index b251758..ce02f8e 100644 +--- a/stable/kube-state-metrics/templates/deployment.yaml ++++ b/stable/kube-state-metrics/templates/deployment.yaml +@@ -1,4 +1,4 @@ +-apiVersion: extensions/v1beta1 ++apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ template "kube-state-metrics.fullname" . }} +@@ -8,6 +8,9 @@ metadata: + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + spec: ++ selector: ++ matchLabels: ++ app: {{ template "kube-state-metrics.name" . }} + replicas: {{ .Values.replicas }} + template: + metadata: +diff --git a/stable/kube-state-metrics/templates/podsecuritypolicy.yaml b/stable/kube-state-metrics/templates/podsecuritypolicy.yaml +index d195a5f..4ca46ac 100644 +--- a/stable/kube-state-metrics/templates/podsecuritypolicy.yaml ++++ b/stable/kube-state-metrics/templates/podsecuritypolicy.yaml +@@ -1,5 +1,5 @@ + {{- if .Values.podSecurityPolicy.enabled }} +-apiVersion: extensions/v1beta1 ++apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: {{ template "kube-state-metrics.fullname" . }} +diff --git a/stable/logstash/templates/ingress.yaml b/stable/logstash/templates/ingress.yaml +index 0e26f89..d107a32 100644 +--- a/stable/logstash/templates/ingress.yaml ++++ b/stable/logstash/templates/ingress.yaml +@@ -1,7 +1,7 @@ + {{- if .Values.ingress.enabled -}} + {{- $fullName := include "logstash.fullname" . -}} + {{- $ingressPath := .Values.ingress.path -}} +-apiVersion: extensions/v1beta1 ++apiVersion: networking.k8s.io/v1beta1 + kind: Ingress + metadata: + name: {{ $fullName }} +diff --git a/stable/logstash/templates/statefulset.yaml b/stable/logstash/templates/statefulset.yaml +index 070ef56..1a1a3ef 100644 +--- a/stable/logstash/templates/statefulset.yaml ++++ b/stable/logstash/templates/statefulset.yaml +@@ -1,4 +1,4 @@ +-apiVersion: apps/v1beta2 ++apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: {{ template "logstash.fullname" . }} +diff --git a/stable/metricbeat/templates/podsecuritypolicy.yaml b/stable/metricbeat/templates/podsecuritypolicy.yaml +index 29fb2a9..90231e2 100644 +--- a/stable/metricbeat/templates/podsecuritypolicy.yaml ++++ b/stable/metricbeat/templates/podsecuritypolicy.yaml +@@ -1,5 +1,5 @@ + {{- if .Values.rbac.pspEnabled }} +-apiVersion: extensions/v1beta1 ++apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: {{ template "metricbeat.fullname" . }} +diff --git a/stable/nginx-ingress/templates/controller-daemonset.yaml b/stable/nginx-ingress/templates/controller-daemonset.yaml +index a05062e..c6e8271 100644 +--- a/stable/nginx-ingress/templates/controller-daemonset.yaml ++++ b/stable/nginx-ingress/templates/controller-daemonset.yaml +@@ -1,5 +1,5 @@ + {{- if eq .Values.controller.kind "DaemonSet" }} +-apiVersion: extensions/v1beta1 ++apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: +@@ -10,6 +10,10 @@ metadata: + release: {{ .Release.Name }} + name: {{ template "nginx-ingress.controller.fullname" . }} + spec: ++ selector: ++ matchLabels: ++ app: {{ template "nginx-ingress.name" . }} ++ release: {{ .Release.Name }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + updateStrategy: + {{ toYaml .Values.controller.updateStrategy | indent 4 }} +diff --git a/stable/nginx-ingress/templates/controller-deployment.yaml b/stable/nginx-ingress/templates/controller-deployment.yaml +index 7d78507..f653c8a 100644 +--- a/stable/nginx-ingress/templates/controller-deployment.yaml ++++ b/stable/nginx-ingress/templates/controller-deployment.yaml +@@ -1,5 +1,5 @@ + {{- if eq .Values.controller.kind "Deployment" }} +-apiVersion: extensions/v1beta1 ++apiVersion: apps/v1 + kind: Deployment + metadata: + labels: +@@ -10,6 +10,10 @@ metadata: + release: {{ .Release.Name }} + name: {{ template "nginx-ingress.controller.fullname" . }} + spec: ++ selector: ++ matchLabels: ++ app: {{ template "nginx-ingress.name" . }} ++ release: {{ .Release.Name }} + replicas: {{ .Values.controller.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + strategy: +diff --git a/stable/nginx-ingress/templates/default-backend-deployment.yaml b/stable/nginx-ingress/templates/default-backend-deployment.yaml +index 93ea613..399b798 100644 +--- a/stable/nginx-ingress/templates/default-backend-deployment.yaml ++++ b/stable/nginx-ingress/templates/default-backend-deployment.yaml +@@ -1,5 +1,5 @@ + {{- if .Values.defaultBackend.enabled }} +-apiVersion: extensions/v1beta1 ++apiVersion: apps/v1 + kind: Deployment + metadata: + labels: +@@ -10,6 +10,10 @@ metadata: + release: {{ .Release.Name }} + name: {{ template "nginx-ingress.defaultBackend.fullname" . }} + spec: ++ selector: ++ matchLabels: ++ app: {{ template "nginx-ingress.name" . }} ++ release: {{ .Release.Name }} + replicas: {{ .Values.defaultBackend.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + template: +diff --git a/stable/nginx-ingress/templates/podsecuritypolicy.yaml b/stable/nginx-ingress/templates/podsecuritypolicy.yaml +index 9222daf..b99aee8 100644 +--- a/stable/nginx-ingress/templates/podsecuritypolicy.yaml ++++ b/stable/nginx-ingress/templates/podsecuritypolicy.yaml +@@ -1,5 +1,5 @@ + {{- if .Values.podSecurityPolicy.enabled}} +-apiVersion: extensions/v1beta1 ++apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: {{ template "nginx-ingress.fullname" . }} +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0009-add-curator-as-of-2019-10-10.patch b/monitor-helm/debian/deb_folder/patches/0009-add-curator-as-of-2019-10-10.patch new file mode 100644 index 0000000..ab57f97 --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0009-add-curator-as-of-2019-10-10.patch @@ -0,0 +1,397 @@ +From 21cd4d9720064f89843551e7da4c1e0528b6cbf5 Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Thu, 10 Oct 2019 15:43:20 -0400 +Subject: [PATCH 1/1] add curator as of 2019-10-10 + +--- + stable/elasticsearch-curator/Chart.yaml | 6 +-- + stable/elasticsearch-curator/OWNERS | 6 +-- + stable/elasticsearch-curator/README.md | 34 ++++++++++--- + .../ci/initcontainer-values.yaml | 9 ++++ + .../elasticsearch-curator/templates/_helpers.tpl | 22 +++++++++ + .../elasticsearch-curator/templates/cronjob.yaml | 10 ++++ + stable/elasticsearch-curator/templates/psp.yml | 35 +++++++++++++ + stable/elasticsearch-curator/templates/role.yaml | 23 +++++++++ + .../templates/rolebinding.yaml | 21 ++++++++ + .../templates/serviceaccount.yaml | 12 +++++ + stable/elasticsearch-curator/values.yaml | 57 ++++++++++++++++++++-- + 11 files changed, 218 insertions(+), 17 deletions(-) + create mode 100644 stable/elasticsearch-curator/ci/initcontainer-values.yaml + create mode 100644 stable/elasticsearch-curator/templates/psp.yml + create mode 100644 stable/elasticsearch-curator/templates/role.yaml + create mode 100644 stable/elasticsearch-curator/templates/rolebinding.yaml + create mode 100644 stable/elasticsearch-curator/templates/serviceaccount.yaml + +diff --git a/stable/elasticsearch-curator/Chart.yaml b/stable/elasticsearch-curator/Chart.yaml +index 24a37ce..7a8e0a7 100644 +--- a/stable/elasticsearch-curator/Chart.yaml ++++ b/stable/elasticsearch-curator/Chart.yaml +@@ -2,7 +2,7 @@ apiVersion: v1 + appVersion: "5.5.4" + description: A Helm chart for Elasticsearch Curator + name: elasticsearch-curator +-version: 1.3.2 ++version: 2.0.2 + home: https://github.com/elastic/curator + keywords: + - curator +@@ -12,7 +12,7 @@ sources: + - https://github.com/kubernetes/charts/elasticsearch-curator + - https://github.com/pires/docker-elasticsearch-curator + maintainers: +- - name: tmestdagh +- email: mestdagh.tom@gmail.com ++ - name: desaintmartin ++ email: cedric.dsm@gmail.com + - name: gianrubio + email: gianrubio@gmail.com +diff --git a/stable/elasticsearch-curator/OWNERS b/stable/elasticsearch-curator/OWNERS +index d8c0ba0..89df1c0 100644 +--- a/stable/elasticsearch-curator/OWNERS ++++ b/stable/elasticsearch-curator/OWNERS +@@ -1,6 +1,6 @@ + approvers: +- - tmestdagh ++ - desaintmartin + - gianrubio + reviewers: +- - tmestdagh +- - gianrubio +\ No newline at end of file ++ - desaintmartin ++ - gianrubio +diff --git a/stable/elasticsearch-curator/README.md b/stable/elasticsearch-curator/README.md +index 0a9f311..2057b85 100644 +--- a/stable/elasticsearch-curator/README.md ++++ b/stable/elasticsearch-curator/README.md +@@ -23,6 +23,17 @@ To install the chart, use the following: + $ helm install stable/elasticsearch-curator + ``` + ++## Upgrading an existing Release to a new major version ++ ++A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an ++incompatible breaking change needing manual actions. ++ ++### To 2.0.0 ++ ++v2.0.0 uses docker image from `elasticsearch-curator` author, which differs in its way to install curator. ++ ++If you have a hardcoded `command` value, please update it to follow the new `curator` executable path: `/curator/curator` (which is not in PATH). ++ + ## Configuration + + The following table lists the configurable parameters of the docker-registry chart and +@@ -31,8 +42,8 @@ their default values. + | Parameter | Description | Default | + | :----------------------------------- | :---------------------------------------------------------- | :------------------------------------------- | + | `image.pullPolicy` | Container pull policy | `IfNotPresent` | +-| `image.repository` | Container image to use | `quay.io/pires/docker-elasticsearch-curator` | +-| `image.tag` | Container image tag to deploy | `5.5.4` | ++| `image.repository` | Container image to use | `untergeek/curator` | ++| `image.tag` | Container image tag to deploy | `5.7.6` | + | `hooks` | Whether to run job on selected hooks | `{ "install": false, "upgrade": false }` | + | `cronjob.schedule` | Schedule for the CronJob | `0 1 * * *` | + | `cronjob.annotations` | Annotations to add to the cronjob | {} | +@@ -43,15 +54,22 @@ their default values. + | `dryrun` | Run Curator in dry-run mode | `false` | + | `env` | Environment variables to add to the cronjob container | {} | + | `envFromSecrets` | Environment variables from secrets to the cronjob container | {} | +-| `envFromSecrets.*.from.secret` | - `secretKeyRef.name` used for environment variable | | +-| `envFromSecrets.*.from.key` | - `secretKeyRef.key` used for environment variable | | +-| `command` | Command to execute | ["curator"] | +-| `configMaps.action_file_yml` | Contents of the Curator action_file.yml | See values.yaml | +-| `configMaps.config_yml` | Contents of the Curator config.yml (overrides config) | See values.yaml | ++| `envFromSecrets.*.from.secret` | - `secretKeyRef.name` used for environment variable | | ++| `envFromSecrets.*.from.key` | - `secretKeyRef.key` used for environment variable | | ++| `command` | Command to execute | ["/curator/curator"] | ++| `configMaps.action_file_yml` | Contents of the Curator action_file.yml | See values.yaml | ++| `configMaps.config_yml` | Contents of the Curator config.yml (overrides config) | See values.yaml | + | `resources` | Resource requests and limits | {} | + | `priorityClassName` | priorityClassName | `nil` | + | `extraVolumeMounts` | Mount extra volume(s), | | + | `extraVolumes` | Extra volumes | | +-| `securityContext` | Configure PodSecurityContext | ++| `extraInitContainers` | Init containers to add to the cronjob container | {} | ++| `securityContext` | Configure PodSecurityContext | `false` | ++| `rbac.enabled` | Enable RBAC resources | `false` | ++| `psp.create` | Create pod security policy resources | `false` | ++| `serviceAccount.create` | Create a default serviceaccount for elasticsearch curator | `true` | ++| `serviceAccount.name` | Name for elasticsearch curator serviceaccount | `""` | ++ ++ + Specify each parameter using the `--set key=value[,key=value]` argument to + `helm install`. +diff --git a/stable/elasticsearch-curator/ci/initcontainer-values.yaml b/stable/elasticsearch-curator/ci/initcontainer-values.yaml +new file mode 100644 +index 0000000..578becf +--- /dev/null ++++ b/stable/elasticsearch-curator/ci/initcontainer-values.yaml +@@ -0,0 +1,9 @@ ++extraInitContainers: ++ test: ++ image: alpine:latest ++ command: ++ - "/bin/sh" ++ - "-c" ++ args: ++ - | ++ true +diff --git a/stable/elasticsearch-curator/templates/_helpers.tpl b/stable/elasticsearch-curator/templates/_helpers.tpl +index c786fb5..8018c5d 100644 +--- a/stable/elasticsearch-curator/templates/_helpers.tpl ++++ b/stable/elasticsearch-curator/templates/_helpers.tpl +@@ -12,6 +12,17 @@ Return the appropriate apiVersion for cronjob APIs. + {{- end -}} + + {{/* ++Return the appropriate apiVersion for podsecuritypolicy. ++*/}} ++{{- define "podsecuritypolicy.apiVersion" -}} ++{{- if semverCompare "<1.10-0" .Capabilities.KubeVersion.GitVersion -}} ++{{- print "extensions/v1beta1" -}} ++{{- else -}} ++{{- print "policy/v1beta1" -}} ++{{- end -}} ++{{- end -}} ++ ++{{/* + Expand the name of the chart. + */}} + {{- define "elasticsearch-curator.name" -}} +@@ -42,3 +53,14 @@ Create chart name and version as used by the chart label. + {{- define "elasticsearch-curator.chart" -}} + {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} + {{- end -}} ++ ++{{/* ++Create the name of the service account to use ++*/}} ++{{- define "elasticsearch-curator.serviceAccountName" -}} ++{{- if .Values.serviceAccount.create -}} ++ {{ default (include "elasticsearch-curator.fullname" .) .Values.serviceAccount.name }} ++{{- else -}} ++ {{ default "default" .Values.serviceAccount.name }} ++{{- end -}} ++{{- end -}} +diff --git a/stable/elasticsearch-curator/templates/cronjob.yaml b/stable/elasticsearch-curator/templates/cronjob.yaml +index d0388f4..37274f6 100644 +--- a/stable/elasticsearch-curator/templates/cronjob.yaml ++++ b/stable/elasticsearch-curator/templates/cronjob.yaml +@@ -53,6 +53,16 @@ spec: + imagePullSecrets: + - name: {{ .Values.image.pullSecret }} + {{- end }} ++{{- if .Values.extraInitContainers }} ++ initContainers: ++{{- range $key, $value := .Values.extraInitContainers }} ++ - name: "{{ $key }}" ++{{ toYaml $value | indent 12 }} ++{{- end }} ++{{- end }} ++ {{- if .Values.rbac.enabled }} ++ serviceAccountName: {{ template "elasticsearch-curator.serviceAccountName" .}} ++ {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +diff --git a/stable/elasticsearch-curator/templates/psp.yml b/stable/elasticsearch-curator/templates/psp.yml +new file mode 100644 +index 0000000..5f62985 +--- /dev/null ++++ b/stable/elasticsearch-curator/templates/psp.yml +@@ -0,0 +1,35 @@ ++{{- if .Values.psp.create }} ++apiVersion: {{ template "podsecuritypolicy.apiVersion" . }} ++kind: PodSecurityPolicy ++metadata: ++ labels: ++ app: {{ template "elasticsearch-curator.name" . }} ++ chart: {{ template "elasticsearch-curator.chart" . }} ++ release: {{ .Release.Name }} ++ heritage: {{ .Release.Service }} ++ name: {{ template "elasticsearch-curator.fullname" . }}-psp ++spec: ++ privileged: true ++ #requiredDropCapabilities: ++ volumes: ++ - 'configMap' ++ - 'secret' ++ hostNetwork: false ++ hostIPC: false ++ hostPID: false ++ runAsUser: ++ rule: 'RunAsAny' ++ seLinux: ++ rule: 'RunAsAny' ++ supplementalGroups: ++ rule: 'MustRunAs' ++ ranges: ++ - min: 1 ++ max: 65535 ++ fsGroup: ++ rule: 'MustRunAs' ++ ranges: ++ - min: 1 ++ max: 65535 ++ readOnlyRootFilesystem: false ++{{- end }} +diff --git a/stable/elasticsearch-curator/templates/role.yaml b/stable/elasticsearch-curator/templates/role.yaml +new file mode 100644 +index 0000000..8867f67 +--- /dev/null ++++ b/stable/elasticsearch-curator/templates/role.yaml +@@ -0,0 +1,23 @@ ++{{- if .Values.rbac.enabled }} ++kind: Role ++apiVersion: rbac.authorization.k8s.io/v1 ++metadata: ++ labels: ++ app: {{ template "elasticsearch-curator.name" . }} ++ chart: {{ template "elasticsearch-curator.chart" . }} ++ heritage: {{ .Release.Service }} ++ release: {{ .Release.Name }} ++ component: elasticsearch-curator-configmap ++ name: {{ template "elasticsearch-curator.name" . }}-role ++rules: ++- apiGroups: [""] ++ resources: ["configmaps"] ++ verbs: ["update", "patch"] ++{{- if .Values.psp.create }} ++- apiGroups: ["extensions"] ++ resources: ["podsecuritypolicies"] ++ verbs: ["use"] ++ resourceNames: ++ - {{ template "elasticsearch-curator.fullname" . }}-psp ++{{- end -}} ++{{- end -}} +diff --git a/stable/elasticsearch-curator/templates/rolebinding.yaml b/stable/elasticsearch-curator/templates/rolebinding.yaml +new file mode 100644 +index 0000000..d25d2e1 +--- /dev/null ++++ b/stable/elasticsearch-curator/templates/rolebinding.yaml +@@ -0,0 +1,21 @@ ++{{- if .Values.rbac.enabled -}} ++kind: RoleBinding ++apiVersion: rbac.authorization.k8s.io/v1 ++metadata: ++ labels: ++ app: {{ template "elasticsearch-curator.name" . }} ++ chart: {{ template "elasticsearch-curator.chart" . }} ++ heritage: {{ .Release.Service }} ++ release: {{ .Release.Name }} ++ component: elasticsearch-curator-configmap ++ name: {{ template "elasticsearch-curator.name" . }}-rolebinding ++roleRef: ++ kind: Role ++ name: {{ template "elasticsearch-curator.name" . }}-role ++ apiGroup: rbac.authorization.k8s.io ++subjects: ++ - kind: ServiceAccount ++ name: {{ template "elasticsearch-curator.serviceAccountName" . }} ++ namespace: {{ .Release.Namespace }} ++{{- end -}} ++ +diff --git a/stable/elasticsearch-curator/templates/serviceaccount.yaml b/stable/elasticsearch-curator/templates/serviceaccount.yaml +new file mode 100644 +index 0000000..ad9c5c9 +--- /dev/null ++++ b/stable/elasticsearch-curator/templates/serviceaccount.yaml +@@ -0,0 +1,12 @@ ++{{- if and .Values.serviceAccount.create .Values.rbac.enabled }} ++apiVersion: v1 ++kind: ServiceAccount ++metadata: ++ name: {{ template "elasticsearch-curator.serviceAccountName" .}} ++ labels: ++ app: {{ template "elasticsearch-curator.fullname" . }} ++ chart: {{ template "elasticsearch-curator.chart" . }} ++ release: "{{ .Release.Name }}" ++ heritage: "{{ .Release.Service }}" ++{{- end }} ++ +diff --git a/stable/elasticsearch-curator/values.yaml b/stable/elasticsearch-curator/values.yaml +index 3779be1..460f2a4 100644 +--- a/stable/elasticsearch-curator/values.yaml ++++ b/stable/elasticsearch-curator/values.yaml +@@ -13,9 +13,25 @@ cronjob: + pod: + annotations: {} + ++rbac: ++ # Specifies whether RBAC should be enabled ++ enabled: false ++ ++serviceAccount: ++ # Specifies whether a ServiceAccount should be created ++ create: true ++ # The name of the ServiceAccount to use. ++ # If not set and create is true, a name is generated using the fullname template ++ name: ++ ++ ++psp: ++ # Specifies whether a podsecuritypolicy should be created ++ create: false ++ + image: +- repository: quay.io/pires/docker-elasticsearch-curator +- tag: 5.5.4 ++ repository: untergeek/curator ++ tag: 5.7.6 + pullPolicy: IfNotPresent + + hooks: +@@ -25,7 +41,7 @@ hooks: + # run curator in dry-run mode + dryrun: false + +-command: ["curator"] ++command: ["/curator/curator"] + env: {} + + configMaps: +@@ -101,5 +117,40 @@ priorityClassName: "" + # mountPath: /certs + # readOnly: true + ++# Add your own init container or uncomment and modify the given example. ++extraInitContainers: {} ++ ## Don't configure S3 repository till Elasticsearch is reachable. ++ ## Ensure that it is available at http://elasticsearch:9200 ++ ## ++ # elasticsearch-s3-repository: ++ # image: jwilder/dockerize:latest ++ # imagePullPolicy: "IfNotPresent" ++ # command: ++ # - "/bin/sh" ++ # - "-c" ++ # args: ++ # - | ++ # ES_HOST=elasticsearch ++ # ES_PORT=9200 ++ # ES_REPOSITORY=backup ++ # S3_REGION=us-east-1 ++ # S3_BUCKET=bucket ++ # S3_BASE_PATH=backup ++ # S3_COMPRESS=true ++ # S3_STORAGE_CLASS=standard ++ # apk add curl --no-cache && \ ++ # dockerize -wait http://${ES_HOST}:${ES_PORT} --timeout 120s && \ ++ # cat < +Date: Fri, 18 Oct 2019 17:01:14 -0400 +Subject: [PATCH 1/1] Update kube-state-metrics 1.8.0, to commit 09daf19 + +--- + stable/kube-state-metrics/Chart.yaml | 7 +- + stable/kube-state-metrics/OWNERS | 8 ++ + stable/kube-state-metrics/README.md | 96 ++++++++++++---------- + stable/kube-state-metrics/templates/NOTES.txt | 2 +- + .../kube-state-metrics/templates/clusterrole.yaml | 38 +++++++-- + .../templates/clusterrolebinding.yaml | 8 +- + .../kube-state-metrics/templates/deployment.yaml | 52 +++++++++--- + .../templates/podsecuritypolicy.yaml | 10 +-- + .../templates/psp-clusterrole.yaml | 8 +- + .../templates/psp-clusterrolebinding.yaml | 8 +- + stable/kube-state-metrics/templates/service.yaml | 22 +++-- + .../templates/serviceaccount.yaml | 10 +-- + .../templates/servicemonitor.yaml | 24 ++++++ + stable/kube-state-metrics/values.yaml | 24 +++++- + 14 files changed, 220 insertions(+), 97 deletions(-) + create mode 100644 stable/kube-state-metrics/OWNERS + create mode 100644 stable/kube-state-metrics/templates/servicemonitor.yaml + +diff --git a/stable/kube-state-metrics/Chart.yaml b/stable/kube-state-metrics/Chart.yaml +index 2f0f39d..283e1ac 100644 +--- a/stable/kube-state-metrics/Chart.yaml ++++ b/stable/kube-state-metrics/Chart.yaml +@@ -5,11 +5,14 @@ keywords: + - metric + - monitoring + - prometheus +-version: 0.16.0 +-appVersion: 1.5.0 ++- kubernetes ++version: 2.4.1 ++appVersion: 1.8.0 + home: https://github.com/kubernetes/kube-state-metrics/ + sources: + - https://github.com/kubernetes/kube-state-metrics/ + maintainers: + - name: fiunchinho + email: jose@armesto.net ++- name: tariq1890 ++ email: tariq.ibrahim@mulesoft.com +diff --git a/stable/kube-state-metrics/OWNERS b/stable/kube-state-metrics/OWNERS +new file mode 100644 +index 0000000..6ffd97d +--- /dev/null ++++ b/stable/kube-state-metrics/OWNERS +@@ -0,0 +1,8 @@ ++approvers: ++- fiunchinho ++- tariq1890 ++- mrueg ++reviewers: ++- fiunchinho ++- tariq1890 ++- mrueg +diff --git a/stable/kube-state-metrics/README.md b/stable/kube-state-metrics/README.md +index 94ad049..6c7f364 100644 +--- a/stable/kube-state-metrics/README.md ++++ b/stable/kube-state-metrics/README.md +@@ -12,45 +12,57 @@ $ helm install stable/kube-state-metrics + + ## Configuration + +-| Parameter | Description | Default | +-|---------------------------------------|---------------------------------------------------------|---------------------------------------------| +-| `image.repository` | The image repository to pull from | k8s.gcr.io/kube-state-metrics | +-| `image.tag` | The image tag to pull from | `v1.5.0` | +-| `image.pullPolicy` | Image pull policy | IfNotPresent | +-| `replicas` | Number of replicas | 1 | +-| `service.port` | The port of the container | 8080 | +-| `prometheusScrape` | Whether or not enable prom scrape | true | +-| `rbac.create` | If true, create & use RBAC resources | true | +-| `serviceAccount.create` | If true, and rbac true, create & use serviceAccount | true | +-| `serviceAccount.name` | If not set & create is true, use template fullname | | +-| `serviceAccount.imagePullSecrets` | Specify image pull secrets field | `[]` | +-| `podSecurityPolicy.enabled` | If true, create & use PodSecurityPolicy resources | false | +-| `podSecurityPolicy.annotations` | Specify pod annotations in the pod security policy | {} | +-| `securityContext.enabled` | Enable security context | `true` | +-| `securityContext.fsGroup` | Group ID for the container | `65534` | +-| `securityContext.runAsUser` | User ID for the container | `65534` | +-| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +-| `nodeSelector` | Node labels for pod assignment | {} | +-| `tolerations` | Tolerations for pod assignment | [] | +-| `podAnnotations` | Annotations to be added to the pod | {} | +-| `resources` | kube-state-metrics resource requests and limits | {} | +-| `collectors.configmaps` | Enable the configmaps collector. | true | +-| `collectors.cronjobs` | Enable the cronjobs collector. | true | +-| `collectors.daemonsets` | Enable the daemonsets collector. | true | +-| `collectors.deployments` | Enable the deployments collector. | true | +-| `collectors.endpoints` | Enable the endpoints collector. | true | +-| `collectors.horizontalpodautoscalers` | Enable the horizontalpodautoscalers collector. | true | +-| `collectors.jobs` | Enable the jobs collector. | true | +-| `collectors.limitranges` | Enable the limitranges collector. | true | +-| `collectors.namespaces` | Enable the namespaces collector. | true | +-| `collectors.nodes` | Enable the nodes collector. | true | +-| `collectors.persistentvolumeclaims` | Enable the persistentvolumeclaims collector. | true | +-| `collectors.persistentvolumes` | Enable the persistentvolumes collector. | true | +-| `collectors.poddisruptionbudgets` | Enable the poddisruptionbudgets collector. | true | +-| `collectors.pods` | Enable the pods collector. | true | +-| `collectors.replicasets` | Enable the replicasets collector. | true | +-| `collectors.replicationcontrollers` | Enable the replicationcontrollers collector. | true | +-| `collectors.resourcequotas` | Enable the resourcequotas collector. | true | +-| `collectors.secrets` | Enable the secrets collector. | true | +-| `collectors.services` | Enable the services collector. | true | +-| `collectors.statefulsets` | Enable the statefulsets collector. | true | ++| Parameter | Description | Default | ++|:----------------------------------------|:--------------------------------------------------------------------------------------|:-------------------------------------------| ++| `image.repository` | The image repository to pull from | quay.io/coreos/kube-state-metrics | ++| `image.tag` | The image tag to pull from | `v1.8.0` | ++| `image.pullPolicy` | Image pull policy | `IfNotPresent` | ++| `replicas` | Number of replicas | `1` | ++| `service.port` | The port of the container | `8080` | ++| `service.annotations` | Annotations to be added to the service | `{}` ++| `customLabels` | Custom labels to apply to service, deployment and pods | `{}` | ++| `hostNetwork` | Whether or not to use the host network | `false` | ++| `prometheusScrape` | Whether or not enable prom scrape | `true` | ++| `rbac.create` | If true, create & use RBAC resources | `true` | ++| `serviceAccount.create` | If true, create & use serviceAccount | `true` | ++| `serviceAccount.name` | If not set & create is true, use template fullname | | ++| `serviceAccount.imagePullSecrets` | Specify image pull secrets field | `[]` | ++| `podSecurityPolicy.enabled` | If true, create & use PodSecurityPolicy resources | `false` | ++| `podSecurityPolicy.annotations` | Specify pod annotations in the pod security policy | {} | ++| `securityContext.enabled` | Enable security context | `true` | ++| `securityContext.fsGroup` | Group ID for the container | `65534` | ++| `securityContext.runAsUser` | User ID for the container | `65534` | ++| `priorityClassName` | Name of Priority Class to assign pods | `nil` | ++| `nodeSelector` | Node labels for pod assignment | {} | ++| `affinity` | Affinity settings for pod assignment | {} | ++| `tolerations` | Tolerations for pod assignment | [] | ++| `podAnnotations` | Annotations to be added to the pod | {} | ++| `resources` | kube-state-metrics resource requests and limits | {} | ++| `collectors.certificatesigningrequests` | Enable the certificatesigningrequests collector. | `true` | ++| `collectors.configmaps` | Enable the configmaps collector. | `true` | ++| `collectors.cronjobs` | Enable the cronjobs collector. | `true` | ++| `collectors.daemonsets` | Enable the daemonsets collector. | `true` | ++| `collectors.deployments` | Enable the deployments collector. | `true` | ++| `collectors.endpoints` | Enable the endpoints collector. | `true` | ++| `collectors.horizontalpodautoscalers` | Enable the horizontalpodautoscalers collector. | `true` | ++| `collectors.ingresses` | Enable the ingresses collector. | `true` | ++| `collectors.jobs` | Enable the jobs collector. | `true` | ++| `collectors.limitranges` | Enable the limitranges collector. | `true` | ++| `collectors.namespaces` | Enable the namespaces collector. | `true` | ++| `collectors.nodes` | Enable the nodes collector. | `true` | ++| `collectors.persistentvolumeclaims` | Enable the persistentvolumeclaims collector. | `true` | ++| `collectors.persistentvolumes` | Enable the persistentvolumes collector. | `true` | ++| `collectors.poddisruptionbudgets` | Enable the poddisruptionbudgets collector. | `true` | ++| `collectors.pods` | Enable the pods collector. | `true` | ++| `collectors.replicasets` | Enable the replicasets collector. | `true` | ++| `collectors.replicationcontrollers` | Enable the replicationcontrollers collector. | `true` | ++| `collectors.resourcequotas` | Enable the resourcequotas collector. | `true` | ++| `collectors.secrets` | Enable the secrets collector. | `true` | ++| `collectors.services` | Enable the services collector. | `true` | ++| `collectors.statefulsets` | Enable the statefulsets collector. | `true` | ++| `collectors.storageclasses` | Enable the storageclasses collector. | `true` | ++| `collectors.verticalpodautoscalers` | Enable the verticalpodautoscalers collector. | `false` | ++| `prometheus.monitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | ++| `prometheus.monitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | `{}` | ++| `prometheus.monitor.namespace` | Namespace where servicemonitor resource should be created | `the same namespace as kube-state-metrics` | ++| `prometheus.monitor.honorLabels` | Honor metric labels | `false` | +diff --git a/stable/kube-state-metrics/templates/NOTES.txt b/stable/kube-state-metrics/templates/NOTES.txt +index 8e8d9fe..d804011 100644 +--- a/stable/kube-state-metrics/templates/NOTES.txt ++++ b/stable/kube-state-metrics/templates/NOTES.txt +@@ -1,6 +1,6 @@ + kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. + The exposed metrics can be found here: +-https://github.com/kubernetes/kube-state-metrics/tree/master/Documentation#documentation. ++https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + + The metrics are exported on the HTTP endpoint /metrics on the listening port. + In your case, {{ template "kube-state-metrics.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }}/metrics +diff --git a/stable/kube-state-metrics/templates/clusterrole.yaml b/stable/kube-state-metrics/templates/clusterrole.yaml +index 803b73b..4f68a1f 100644 +--- a/stable/kube-state-metrics/templates/clusterrole.yaml ++++ b/stable/kube-state-metrics/templates/clusterrole.yaml +@@ -3,12 +3,18 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: ClusterRole + metadata: + labels: +- app: {{ template "kube-state-metrics.name" . }} +- chart: {{ .Chart.Name }}-{{ .Chart.Version }} +- heritage: {{ .Release.Service }} +- release: {{ .Release.Name }} ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} ++ app.kubernetes.io/managed-by: {{ .Release.Service }} ++ app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "kube-state-metrics.fullname" . }} + rules: ++{{ if .Values.collectors.certificatesigningrequests }} ++- apiGroups: ["certificates.k8s.io"] ++ resources: ++ - certificatesigningrequests ++ verbs: ["list", "watch"] ++{{ end -}} + {{ if .Values.collectors.configmaps }} + - apiGroups: [""] + resources: +@@ -22,13 +28,13 @@ rules: + verbs: ["list", "watch"] + {{ end -}} + {{ if .Values.collectors.daemonsets }} +-- apiGroups: ["extensions"] ++- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] + {{ end -}} + {{ if .Values.collectors.deployments }} +-- apiGroups: ["extensions"] ++- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +@@ -45,6 +51,12 @@ rules: + - horizontalpodautoscalers + verbs: ["list", "watch"] + {{ end -}} ++{{ if .Values.collectors.ingresses }} ++- apiGroups: ["extensions", "networking.k8s.io"] ++ resources: ++ - ingresses ++ verbs: ["list", "watch"] ++{{ end -}} + {{ if .Values.collectors.jobs }} + - apiGroups: ["batch"] + resources: +@@ -94,7 +106,7 @@ rules: + verbs: ["list", "watch"] + {{ end -}} + {{ if .Values.collectors.replicasets }} +-- apiGroups: ["extensions"] ++- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +@@ -129,4 +141,16 @@ rules: + - statefulsets + verbs: ["list", "watch"] + {{ end -}} ++{{ if .Values.collectors.storageclasses }} ++- apiGroups: ["storage.k8s.io"] ++ resources: ++ - storageclasses ++ verbs: ["list", "watch"] ++{{ end -}} ++{{ if .Values.collectors.verticalpodautoscalers }} ++- apiGroups: ["autoscaling.k8s.io"] ++ resources: ++ - verticalpodautoscalers ++ verbs: ["list", "watch"] ++{{ end -}} + {{- end -}} +diff --git a/stable/kube-state-metrics/templates/clusterrolebinding.yaml b/stable/kube-state-metrics/templates/clusterrolebinding.yaml +index e57b051..b4586fb 100644 +--- a/stable/kube-state-metrics/templates/clusterrolebinding.yaml ++++ b/stable/kube-state-metrics/templates/clusterrolebinding.yaml +@@ -3,10 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: ClusterRoleBinding + metadata: + labels: +- app: {{ template "kube-state-metrics.name" . }} +- chart: {{ .Chart.Name }}-{{ .Chart.Version }} +- heritage: {{ .Release.Service }} +- release: {{ .Release.Name }} ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} ++ app.kubernetes.io/managed-by: {{ .Release.Service }} ++ app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "kube-state-metrics.fullname" . }} + roleRef: + apiGroup: rbac.authorization.k8s.io +diff --git a/stable/kube-state-metrics/templates/deployment.yaml b/stable/kube-state-metrics/templates/deployment.yaml +index ce02f8e..492df24 100644 +--- a/stable/kube-state-metrics/templates/deployment.yaml ++++ b/stable/kube-state-metrics/templates/deployment.yaml +@@ -3,28 +3,33 @@ kind: Deployment + metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: +- app: {{ template "kube-state-metrics.name" . }} +- chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" +- release: "{{ .Release.Name }}" +- heritage: "{{ .Release.Service }}" ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" ++ app.kubernetes.io/instance: "{{ .Release.Name }}" ++ app.kubernetes.io/managed-by: "{{ .Release.Service }}" ++{{- if .Values.customLabels }} ++{{ toYaml .Values.customLabels | indent 4 }} ++{{- end }} + spec: + selector: + matchLabels: +- app: {{ template "kube-state-metrics.name" . }} ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + replicas: {{ .Values.replicas }} + template: + metadata: + labels: +- app: {{ template "kube-state-metrics.name" . }} +- release: "{{ .Release.Name }}" ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ app.kubernetes.io/instance: "{{ .Release.Name }}" ++{{- if .Values.customLabels }} ++{{ toYaml .Values.customLabels | indent 8 }} ++{{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: +-{{ if .Values.rbac.create }} ++ hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} +-{{ end }} + {{- if .Values.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.securityContext.fsGroup }} +@@ -36,6 +41,9 @@ spec: + containers: + - name: {{ .Chart.Name }} + args: ++{{ if .Values.collectors.certificatesigningrequests }} ++ - --collectors=certificatesigningrequests ++{{ end }} + {{ if .Values.collectors.configmaps }} + - --collectors=configmaps + {{ end }} +@@ -54,6 +62,9 @@ spec: + {{ if .Values.collectors.horizontalpodautoscalers }} + - --collectors=horizontalpodautoscalers + {{ end }} ++{{ if .Values.collectors.ingresses }} ++ - --collectors=ingresses ++{{ end }} + {{ if .Values.collectors.jobs }} + - --collectors=jobs + {{ end }} +@@ -96,6 +107,12 @@ spec: + {{ if .Values.collectors.statefulsets }} + - --collectors=statefulsets + {{ end }} ++{{ if .Values.collectors.storageclasses }} ++ - --collectors=storageclasses ++{{ end }} ++{{ if .Values.collectors.verticalpodautoscalers }} ++ - --collectors=verticalpodautoscalers ++{{ end }} + {{ if .Values.namespace }} + - --namespace={{ .Values.namespace }} + {{ end }} +@@ -103,14 +120,26 @@ spec: + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + ports: + - containerPort: 8080 +- readinessProbe: ++ livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 ++ readinessProbe: ++ httpGet: ++ path: / ++ port: 8080 ++ initialDelaySeconds: 5 ++ timeoutSeconds: 5 ++{{- if .Values.resources }} + resources: +-{{ toYaml .Values.resources | indent 12 }} ++{{ toYaml .Values.resources | indent 10 }} ++{{- end }} ++{{- if .Values.affinity }} ++ affinity: ++{{ toYaml .Values.affinity | indent 8 }} ++{{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: + {{ toYaml .Values.nodeSelector | indent 8 }} +@@ -119,4 +148,3 @@ spec: + tolerations: + {{ toYaml .Values.tolerations | indent 8 }} + {{- end }} +- +diff --git a/stable/kube-state-metrics/templates/podsecuritypolicy.yaml b/stable/kube-state-metrics/templates/podsecuritypolicy.yaml +index 4ca46ac..aeff117 100644 +--- a/stable/kube-state-metrics/templates/podsecuritypolicy.yaml ++++ b/stable/kube-state-metrics/templates/podsecuritypolicy.yaml +@@ -4,12 +4,12 @@ kind: PodSecurityPolicy + metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: +- app: {{ template "kube-state-metrics.name" . }} +- chart: {{ .Chart.Name }}-{{ .Chart.Version }} +- heritage: {{ .Release.Service }} +- release: {{ .Release.Name }} +- annotations: ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} ++ app.kubernetes.io/managed-by: {{ .Release.Service }} ++ app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.podSecurityPolicy.annotations }} ++ annotations: + {{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} + {{- end }} + spec: +diff --git a/stable/kube-state-metrics/templates/psp-clusterrole.yaml b/stable/kube-state-metrics/templates/psp-clusterrole.yaml +index c43f90d..dcd65e1 100644 +--- a/stable/kube-state-metrics/templates/psp-clusterrole.yaml ++++ b/stable/kube-state-metrics/templates/psp-clusterrole.yaml +@@ -3,10 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: +- app: {{ template "kube-state-metrics.name" . }} +- chart: {{ .Chart.Name }}-{{ .Chart.Version }} +- heritage: {{ .Release.Service }} +- release: {{ .Release.Name }} ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} ++ app.kubernetes.io/managed-by: {{ .Release.Service }} ++ app.kubernetes.io/instance: {{ .Release.Name }} + name: psp-{{ template "kube-state-metrics.fullname" . }} + rules: + - apiGroups: ['extensions'] +diff --git a/stable/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/stable/kube-state-metrics/templates/psp-clusterrolebinding.yaml +index bfca12c..7418618 100644 +--- a/stable/kube-state-metrics/templates/psp-clusterrolebinding.yaml ++++ b/stable/kube-state-metrics/templates/psp-clusterrolebinding.yaml +@@ -3,10 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: +- app: {{ template "kube-state-metrics.name" . }} +- chart: {{ .Chart.Name }}-{{ .Chart.Version }} +- heritage: {{ .Release.Service }} +- release: {{ .Release.Name }} ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} ++ app.kubernetes.io/managed-by: {{ .Release.Service }} ++ app.kubernetes.io/instance: {{ .Release.Name }} + name: psp-{{ template "kube-state-metrics.fullname" . }} + roleRef: + apiGroup: rbac.authorization.k8s.io +diff --git a/stable/kube-state-metrics/templates/service.yaml b/stable/kube-state-metrics/templates/service.yaml +index b6daacd..a880bf9 100644 +--- a/stable/kube-state-metrics/templates/service.yaml ++++ b/stable/kube-state-metrics/templates/service.yaml +@@ -3,14 +3,20 @@ kind: Service + metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: +- app: {{ template "kube-state-metrics.name" . }} +- chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" +- release: "{{ .Release.Name }}" +- heritage: "{{ .Release.Service }}" +- {{- if .Values.prometheusScrape }} ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" ++ app.kubernetes.io/instance: "{{ .Release.Name }}" ++ app.kubernetes.io/managed-by: "{{ .Release.Service }}" ++{{- if .Values.customLabels }} ++{{ toYaml .Values.customLabels | indent 4 }} ++{{- end }} + annotations: ++ {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' +- {{- end }} ++ {{- end }} ++ {{- if .Values.service.annotations }} ++ {{- toYaml .Values.service.annotations | nindent 4 }} ++ {{- end }} + spec: + type: "{{ .Values.service.type }}" + ports: +@@ -25,5 +31,5 @@ spec: + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" + {{- end }} + selector: +- app: {{ template "kube-state-metrics.name" . }} +- release: {{ .Release.Name }} ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ app.kubernetes.io/instance: {{ .Release.Name }} +diff --git a/stable/kube-state-metrics/templates/serviceaccount.yaml b/stable/kube-state-metrics/templates/serviceaccount.yaml +index edac3b9..e2bf7c0 100644 +--- a/stable/kube-state-metrics/templates/serviceaccount.yaml ++++ b/stable/kube-state-metrics/templates/serviceaccount.yaml +@@ -1,15 +1,13 @@ +-{{- if .Values.rbac.create -}} + {{- if .Values.serviceAccount.create -}} + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: +- app: {{ template "kube-state-metrics.name" . }} +- chart: {{ .Chart.Name }}-{{ .Chart.Version }} +- heritage: {{ .Release.Service }} +- release: {{ .Release.Name }} ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} ++ app.kubernetes.io/managed-by: {{ .Release.Service }} ++ app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "kube-state-metrics.fullname" . }} + imagePullSecrets: + {{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} + {{- end -}} +-{{- end -}} +diff --git a/stable/kube-state-metrics/templates/servicemonitor.yaml b/stable/kube-state-metrics/templates/servicemonitor.yaml +new file mode 100644 +index 0000000..6405bd4 +--- /dev/null ++++ b/stable/kube-state-metrics/templates/servicemonitor.yaml +@@ -0,0 +1,24 @@ ++{{- if .Values.prometheus.monitor.enabled }} ++apiVersion: monitoring.coreos.com/v1 ++kind: ServiceMonitor ++metadata: ++ name: {{ template "kube-state-metrics.fullname" . }} ++ labels: ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" ++ app.kubernetes.io/instance: "{{ .Release.Name }}" ++ app.kubernetes.io/managed-by: "{{ .Release.Service }}" ++ {{- if .Values.prometheus.monitor.additionalLabels }} ++{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} ++ {{- end }} ++spec: ++ selector: ++ matchLabels: ++ app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} ++ app.kubernetes.io/instance: {{ .Release.Name }} ++ endpoints: ++ - port: http ++ {{- if .Values.prometheus.monitor.honorLabels }} ++ honorLabels: true ++ {{- end }} ++{{- end }} +diff --git a/stable/kube-state-metrics/values.yaml b/stable/kube-state-metrics/values.yaml +index 86aab9b..2bf5b30 100644 +--- a/stable/kube-state-metrics/values.yaml ++++ b/stable/kube-state-metrics/values.yaml +@@ -1,8 +1,8 @@ + # Default values for kube-state-metrics. + prometheusScrape: true + image: +- repository: k8s.gcr.io/kube-state-metrics +- tag: v1.5.0 ++ repository: quay.io/coreos/kube-state-metrics ++ tag: v1.8.0 + pullPolicy: IfNotPresent + + replicas: 1 +@@ -13,6 +13,11 @@ service: + type: ClusterIP + nodePort: 0 + loadBalancerIP: "" ++ annotations: {} ++ ++customLabels: {} ++ ++hostNetwork: false + + rbac: + # If true, create & use RBAC resources +@@ -28,6 +33,13 @@ serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + ++prometheus: ++ monitor: ++ enabled: false ++ additionalLabels: {} ++ namespace: "" ++ honorLabels: false ++ + ## Specify if a Pod Security Policy for kube-state-metrics must be created + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + ## +@@ -53,6 +65,10 @@ securityContext: + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + ++## Affinity settings for pod assignment ++## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ ++affinity: {} ++ + ## Tolerations for pod assignment + ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] +@@ -66,12 +82,14 @@ podAnnotations: {} + # Available collectors for kube-state-metrics. By default all available + # collectors are enabled. + collectors: ++ certificatesigningrequests: true + configmaps: true + cronjobs: true + daemonsets: true + deployments: true + endpoints: true + horizontalpodautoscalers: true ++ ingresses: true + jobs: true + limitranges: true + namespaces: true +@@ -86,6 +104,8 @@ collectors: + secrets: true + services: true + statefulsets: true ++ storageclasses: true ++ verticalpodautoscalers: false + + # Namespace to be enabled for collecting resources. By default all namespaces are collected. + # namespace: "" +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0011-update-init-container-env-to-include-node-name.patch b/monitor-helm/debian/deb_folder/patches/0011-update-init-container-env-to-include-node-name.patch new file mode 100644 index 0000000..97c6190 --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0011-update-init-container-env-to-include-node-name.patch @@ -0,0 +1,55 @@ +From 35a6b56e47710ea306608f44382b76022e236be0 Mon Sep 17 00:00:00 2001 +From: Matt Peters +Date: Fri, 25 Oct 2019 11:20:37 -0500 +Subject: [PATCH] update init container env to include node name + +Update the init containers to have access to the same set of +environment variables as the runtime containers. Specifically +NODE_NAME is required to ensure the name parameter is set +properly. +--- + stable/filebeat/templates/daemonset.yaml | 8 ++++++++ + stable/metricbeat/templates/daemonset.yaml | 8 ++++++++ + 2 files changed, 16 insertions(+) + +diff --git a/stable/filebeat/templates/daemonset.yaml b/stable/filebeat/templates/daemonset.yaml +index 22b46df..2429892 100644 +--- a/stable/filebeat/templates/daemonset.yaml ++++ b/stable/filebeat/templates/daemonset.yaml +@@ -54,6 +54,14 @@ spec: + readOnly: true + subPath: filebeat.yml + env: ++ - name: POD_NAMESPACE ++ valueFrom: ++ fieldRef: ++ fieldPath: metadata.namespace ++ - name: NODE_NAME ++ valueFrom: ++ fieldRef: ++ fieldPath: spec.nodeName + ## make output.elasticsearch parms visible + {{- range $key, $value := .Values.config}} + {{- $configname := $key | upper | replace "." "_" -}} +diff --git a/stable/metricbeat/templates/daemonset.yaml b/stable/metricbeat/templates/daemonset.yaml +index 947e501..247304e 100644 +--- a/stable/metricbeat/templates/daemonset.yaml ++++ b/stable/metricbeat/templates/daemonset.yaml +@@ -49,6 +49,14 @@ spec: + readOnly: true + subPath: metricbeat.yml + env: ++ - name: POD_NAMESPACE ++ valueFrom: ++ fieldRef: ++ fieldPath: metadata.namespace ++ - name: NODE_NAME ++ valueFrom: ++ fieldRef: ++ fieldPath: spec.nodeName + ## make output.elasticsearch parms visible + {{- range $key, $value := .Values.daemonset.config}} + {{- $configname := $key | upper | replace "." "_" -}} +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0012-Add-imagePullSecrets.patch b/monitor-helm/debian/deb_folder/patches/0012-Add-imagePullSecrets.patch new file mode 100644 index 0000000..755e39a --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0012-Add-imagePullSecrets.patch @@ -0,0 +1,35 @@ +From 1f506385234b0f1ef088afdecb6002b3e9c84305 Mon Sep 17 00:00:00 2001 +From: Angie Wang +Date: Thu, 23 Jan 2020 14:37:40 -0500 +Subject: [PATCH 1/1] Add imagePullSecrets + +Add imagePullSecrets for the missing chart. +--- + stable/metricbeat/templates/serviceaccount.yaml | 2 ++ + stable/metricbeat/values.yaml | 2 ++ + 2 files changed, 4 insertions(+) + +diff --git a/stable/metricbeat/templates/serviceaccount.yaml b/stable/metricbeat/templates/serviceaccount.yaml +index c53fc5a..c2378ec 100644 +--- a/stable/metricbeat/templates/serviceaccount.yaml ++++ b/stable/metricbeat/templates/serviceaccount.yaml +@@ -8,4 +8,6 @@ metadata: + chart: {{ template "metricbeat.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} ++imagePullSecrets: ++{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} + {{- end -}} +diff --git a/stable/metricbeat/values.yaml b/stable/metricbeat/values.yaml +index b59c428..a784e7a 100644 +--- a/stable/metricbeat/values.yaml ++++ b/stable/metricbeat/values.yaml +@@ -167,3 +167,5 @@ serviceAccount: + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: ++ # Reference to one or more secrets to be used when pulling images ++ imagePullSecrets: [] +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0013-removed-unused-images.patch b/monitor-helm/debian/deb_folder/patches/0013-removed-unused-images.patch new file mode 100644 index 0000000..5085dda --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0013-removed-unused-images.patch @@ -0,0 +1,181 @@ +From 2d54de3ae3858aa6c67ba1de25860c440f5be8a2 Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Thu, 12 Dec 2019 09:40:13 -0500 +Subject: [PATCH 1/1] removed unused images + +--- + stable/kibana/templates/tests/test-configmap.yaml | 35 ------------------- + stable/kibana/templates/tests/test.yaml | 42 ----------------------- + stable/kibana/values.yaml | 4 --- + stable/logstash/templates/statefulset.yaml | 32 ----------------- + stable/logstash/values.yaml | 5 --- + 5 files changed, 118 deletions(-) + delete mode 100644 stable/kibana/templates/tests/test-configmap.yaml + delete mode 100644 stable/kibana/templates/tests/test.yaml + +diff --git a/stable/kibana/templates/tests/test-configmap.yaml b/stable/kibana/templates/tests/test-configmap.yaml +deleted file mode 100644 +index 912755e..0000000 +--- a/stable/kibana/templates/tests/test-configmap.yaml ++++ /dev/null +@@ -1,35 +0,0 @@ +-apiVersion: v1 +-kind: ConfigMap +-metadata: +- name: {{ template "kibana.fullname" . }}-test +- labels: +- app: {{ template "kibana.fullname" . }} +- chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" +- heritage: "{{ .Release.Service }}" +- release: "{{ .Release.Name }}" +-data: +- run.sh: |- +- @test "Test Status" { +- {{- if .Values.service.selector }} +- skip "Can't guarentee pod names with selector" +- {{- else }} +- {{- $port := .Values.service.externalPort }} +- url="http://{{ template "kibana.fullname" . }}{{ if $port }}:{{ $port }}{{ end }}/api{{ .Values.livenessProbe.path }}" +- +- # retry for 1 minute +- run curl -s -o /dev/null -I -w "%{http_code}" --retry 30 --retry-delay 2 $url +- +- code=$(curl -s -o /dev/null -I -w "%{http_code}" $url) +- body=$(curl $url) +- if [ "$code" == "503" ] +- then +- skip "Kibana Unavailable (503), can't get status - see pod logs: $body" +- fi +- +- result=$(echo $body | jq -cr '.status.statuses[]') +- [ "$result" != "" ] +- +- result=$(echo $body | jq -cr '.status.statuses[] | select(.state != "green")') +- [ "$result" == "" ] +- {{- end }} +- } +diff --git a/stable/kibana/templates/tests/test.yaml b/stable/kibana/templates/tests/test.yaml +deleted file mode 100644 +index 8a518fd..0000000 +--- a/stable/kibana/templates/tests/test.yaml ++++ /dev/null +@@ -1,42 +0,0 @@ +-apiVersion: v1 +-kind: Pod +-metadata: +- name: {{ template "kibana.fullname" . }}-test +- labels: +- app: {{ template "kibana.fullname" . }} +- chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" +- heritage: "{{ .Release.Service }}" +- release: "{{ .Release.Name }}" +- annotations: +- "helm.sh/hook": test-success +-spec: +- initContainers: +- - name: test-framework +- image: "{{ .Values.testFramework.image}}:{{ .Values.testFramework.tag }}" +- command: +- - "bash" +- - "-c" +- - | +- set -ex +- # copy bats to tools dir +- cp -R /usr/local/libexec/ /tools/bats/ +- volumeMounts: +- - mountPath: /tools +- name: tools +- containers: +- - name: {{ .Release.Name }}-test +- image: "dwdraju/alpine-curl-jq" +- command: ["/tools/bats/bats", "-t", "/tests/run.sh"] +- volumeMounts: +- - mountPath: /tests +- name: tests +- readOnly: true +- - mountPath: /tools +- name: tools +- volumes: +- - name: tests +- configMap: +- name: {{ template "kibana.fullname" . }}-test +- - name: tools +- emptyDir: {} +- restartPolicy: Never +diff --git a/stable/kibana/values.yaml b/stable/kibana/values.yaml +index 4ce4b2a..7bc9cf5 100644 +--- a/stable/kibana/values.yaml ++++ b/stable/kibana/values.yaml +@@ -3,10 +3,6 @@ image: + tag: "7.2.0" + pullPolicy: "IfNotPresent" + +-testFramework: +- image: "dduportal/bats" +- tag: "0.4.0" +- + commandline: + args: [] + +diff --git a/stable/logstash/templates/statefulset.yaml b/stable/logstash/templates/statefulset.yaml +index 1a1a3ef..4749bb1 100644 +--- a/stable/logstash/templates/statefulset.yaml ++++ b/stable/logstash/templates/statefulset.yaml +@@ -91,38 +91,6 @@ spec: + volumeMounts: + {{ toYaml .Values.volumeMounts | indent 12 }} + +-{{- if .Values.exporter.logstash.enabled }} +- ## logstash-exporter +- - name: {{ .Chart.Name }}-exporter +- image: "{{ .Values.exporter.logstash.image.repository }}:{{ .Values.exporter.logstash.image.tag }}" +- imagePullPolicy: {{ .Values.exporter.logstash.image.pullPolicy }} +- command: ["/bin/sh", "-c"] +- ## Delay start of logstash-exporter to give logstash more time to come online. +- args: +- - >- +- sleep 60; +- exec /logstash_exporter +- --logstash.endpoint=http://localhost:{{ .Values.exporter.logstash.target.port }} +- --web.listen-address=:{{ .Values.exporter.logstash.port }} +- ports: +- - name: ls-exporter +- containerPort: {{ .Values.exporter.logstash.port }} +- protocol: TCP +- livenessProbe: +-{{ toYaml .Values.exporter.logstash.livenessProbe | indent 12 }} +- readinessProbe: +-{{ toYaml .Values.exporter.logstash.readinessProbe | indent 12 }} +- {{- with .Values.exporter.logstash.config }} +- env: +- {{- range $key, $value := . }} +- - name: {{ $key | upper | replace "." "_" }} +- value: {{ $value | quote }} +- {{- end }} +- {{- end }} +- resources: +-{{ toYaml .Values.exporter.logstash.resources | indent 12 }} +-{{- end }} +- + {{- with .Values.nodeSelector }} + nodeSelector: + {{ toYaml . | indent 8 }} +diff --git a/stable/logstash/values.yaml b/stable/logstash/values.yaml +index 9ba80c9..b8b8de2 100644 +--- a/stable/logstash/values.yaml ++++ b/stable/logstash/values.yaml +@@ -169,11 +169,6 @@ volumes: [] + + exporter: + logstash: +- enabled: false +- image: +- repository: bonniernews/logstash_exporter +- tag: v0.1.2 +- pullPolicy: IfNotPresent + env: {} + resources: {} + path: /metrics +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0014-Add-rbac-replicasets-to-apps-apigroup-commit-1717e2d.patch b/monitor-helm/debian/deb_folder/patches/0014-Add-rbac-replicasets-to-apps-apigroup-commit-1717e2d.patch new file mode 100644 index 0000000..3004aec --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0014-Add-rbac-replicasets-to-apps-apigroup-commit-1717e2d.patch @@ -0,0 +1,24 @@ +From 6bed0291665738b2b1f9cdceff3eaaec46c9409f Mon Sep 17 00:00:00 2001 +From: Abdelfattah Saafan +Date: Wed, 22 Jan 2020 11:19:39 -0500 +Subject: [PATCH 1/1] add rbac replicasets to apps + +--- + stable/metricbeat/templates/clusterrole.yaml | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/stable/metricbeat/templates/clusterrole.yaml b/stable/metricbeat/templates/clusterrole.yaml +index 5ca886c..bd028fc 100644 +--- a/stable/metricbeat/templates/clusterrole.yaml ++++ b/stable/metricbeat/templates/clusterrole.yaml +@@ -30,6 +30,7 @@ rules: + resources: + - statefulsets + - deployments ++ - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0015-script-flexibility.patch b/monitor-helm/debian/deb_folder/patches/0015-script-flexibility.patch new file mode 100644 index 0000000..adbcab8 --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0015-script-flexibility.patch @@ -0,0 +1,547 @@ +From efa0ac8a6ecfe0482f669cf5574687eaf34d41df Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Wed, 22 Jan 2020 09:56:22 -0500 +Subject: [PATCH 1/1] script flexibility + +--- + stable/filebeat/templates/configmap.yaml | 73 ++------------------------ + stable/filebeat/templates/daemonset.yaml | 14 +++-- + stable/filebeat/values.yaml | 84 ++++++++++++++++++++++++++++++ + stable/logstash/templates/statefulset.yaml | 33 ++++++++++++ + stable/logstash/values.yaml | 10 ++++ + stable/metricbeat/templates/configmap.yaml | 73 ++------------------------ + stable/metricbeat/templates/daemonset.yaml | 19 +++---- + stable/metricbeat/values.yaml | 83 +++++++++++++++++++++++++++++ + 8 files changed, 240 insertions(+), 149 deletions(-) + +diff --git a/stable/filebeat/templates/configmap.yaml b/stable/filebeat/templates/configmap.yaml +index c08a4bc..340f2b9 100644 +--- a/stable/filebeat/templates/configmap.yaml ++++ b/stable/filebeat/templates/configmap.yaml +@@ -1,77 +1,14 @@ + apiVersion: v1 + kind: ConfigMap + metadata: +- name: {{ template "filebeat.fullname" . }} ++ name: {{ template "filebeat.fullname" . }}-files + labels: + app: {{ template "filebeat.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + data: +- setup-script.sh: |- +- #!/bin/bash +- # +- # This is best effort to load the template into elasticsearch +- # if beats are going to elasticsearch via logstash. +- # +- BEAT='filebeat' +- BEAT_VER=$($BEAT version | awk '{print $3}') +- +- ADDR=$OUTPUT_ELASTICSEARCH_HOSTS +- ESPATH=$OUTPUT_ELASTICSEARCH_PATH +- +- echo $ADDR$ESPATH +- +- # Check if this is a map of entries. If so, we'll only export the template +- # to the first address +- if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then +- # Remove the square brackets +- ADDR=${ADDR:1:-1} +- # Take the first entry in case there are more than one +- ADDR=${ADDR%%,*} +- fi +- echo $ADDR$ESPATH +- +- # Check if user has formatted with http:// on front, if not we need to add +- HTTP='http://' +- if [[ ${ADDR} == http* ]]; then +- HTTP='' +- fi +- echo $HTTP +- +- # Check for ip address containing special characters where -g must be used in curl command +- # IPV6 Addresses should come in with square braces around the address. +- CURL_G='' +- if [[ ${ADDR} == *[* ]]; then +- CURL_G=' -g ' +- fi +- echo $CURL_G +- +- # check if logstash output is enabled, via crude parsing of the .yml file. +- sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true +- if [[ $? -eq 0 ]]; then +- echo "logstash configured, exporting template to elasticsearch" +- $BEAT export template > /tmp/beat.template.json +- # remove the lifecycle section of the yaml, as elasticsearch will choke +- # on it as oss version does not support ilm. +- sed -i '/lifecycle/,+3d' /tmp/beat.template.json +- +- # "unset" is the special system name when none has been set through +- # overrides.. We key on that to know what template name and index pattern to set +- if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then +- # replace the standard index pattern with one including our system name, +- # which will match our created indices +- sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json +- # give the template a name with the system name appended. +- curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json +- else +- # apply a higher order to this template in case there are templates with system names +- # which should be applied first +- sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json +- curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json +- fi +- else +- echo "logstash not configured, not exporting template, should be done for us." +- fi +- # Above is non-fatal if there is a problem. Always pass. +- exit 0 ++{{- range $key, $value := .Values.files }} ++ {{ $key }}: |- ++{{ $value | indent 4 }} ++{{- end }} +diff --git a/stable/filebeat/templates/daemonset.yaml b/stable/filebeat/templates/daemonset.yaml +index 2429892..ea76007 100644 +--- a/stable/filebeat/templates/daemonset.yaml ++++ b/stable/filebeat/templates/daemonset.yaml +@@ -44,11 +44,10 @@ spec: + command: + - /bin/bash + - -c +- - /usr/share/filebeat/setup-script.sh ++ - /usr/share/filebeat/files/setup-script.sh + volumeMounts: +- - mountPath: /usr/share/filebeat/setup-script.sh +- name: setupscript +- subPath: setup-script.sh ++ - name: files ++ mountPath: /usr/share/filebeat/files + - name: filebeat-config + mountPath: /usr/share/filebeat/filebeat.yml + readOnly: true +@@ -90,6 +89,9 @@ spec: + - name: INDEX_NAME + value: "{{ .Values.indexNamePrefix }}" + {{- end }} ++{{- if .Values.extraVars }} ++{{ toYaml .Values.extraVars | indent 8 }} ++{{- end }} + {{- if .Values.indexTemplateLoad }} + - name: "load-es-template" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +@@ -221,6 +223,10 @@ spec: + hostPath: + path: /var/lib/filebeat + type: DirectoryOrCreate ++ - name: files ++ configMap: ++ name: {{ template "filebeat.fullname" . }}-files ++ defaultMode: 0755 + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | indent 6 }} + {{- end }} +diff --git a/stable/filebeat/values.yaml b/stable/filebeat/values.yaml +index 3897c82..2490242 100644 +--- a/stable/filebeat/values.yaml ++++ b/stable/filebeat/values.yaml +@@ -90,6 +90,90 @@ extraInitContainers: [] + # - echo + # - hello + ++files: ++ setup-script.sh: |- ++ #!/bin/bash ++ # ++ # This is best effort to load the template into elasticsearch ++ # if beats are going to elasticsearch via logstash. ++ # ++ BEAT='filebeat' ++ BEAT_VER=$($BEAT version | awk '{print $3}') ++ ++ ADDR=$OUTPUT_ELASTICSEARCH_HOSTS ++ ESPATH=$OUTPUT_ELASTICSEARCH_PATH ++ ++ echo $ADDR$ESPATH ++ ++ # Check if this is a map of entries. If so, we'll only export the template ++ # to the first address ++ if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then ++ # Remove the square brackets ++ ADDR=${ADDR:1:-1} ++ # Take the first entry in case there are more than one ++ ADDR=${ADDR%%,*} ++ fi ++ echo $ADDR$ESPATH ++ ++ # Check if user has formatted with http:// on front, if not we need to add ++ HTTP='http://' ++ if [[ ${ADDR} == http* ]]; then ++ HTTP='' ++ fi ++ echo $HTTP ++ ++ # Check for ip address containing special characters where -g must be used in curl command ++ # IPV6 Addresses should come in with square braces around the address. ++ CURL_G='' ++ if [[ ${ADDR} == *[* ]]; then ++ CURL_G=' -g ' ++ fi ++ echo $CURL_G ++ ++ # check if logstash output is enabled, via crude parsing of the .yml file. ++ sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true ++ if [[ $? -eq 0 ]]; then ++ echo "logstash configured, exporting template to elasticsearch" ++ $BEAT export template > /tmp/beat.template.json ++ # remove the lifecycle section of the yaml, as elasticsearch will choke ++ # on it as oss version does not support ilm. ++ sed -i '/lifecycle/,+3d' /tmp/beat.template.json ++ ++ # "unset" is the special system name when none has been set through ++ # overrides.. We key on that to know what template name and index pattern to set ++ if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then ++ # replace the standard index pattern with one including our system name, ++ # which will match our created indices ++ sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json ++ # give the template a name with the system name appended. ++ ++ # check if the template exists first before creating ++ curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX | grep "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX" ++ if [[ $? -ne 0 ]]; then ++ echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template not found, creating..." ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json ++ else ++ echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template found, not creating..." ++ fi ++ else ++ # apply a higher order to this template in case there are templates with system names ++ # which should be applied first ++ curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER | grep "$BEAT-$BEAT_VER" ++ if [[ $? -ne 0 ]]; then ++ echo "$BEAT-$BEAT_VER template not found, creating..." ++ sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json ++ else ++ echo "$BEAT-$BEAT_VER template found, not creating..." ++ fi ++ fi ++ else ++ echo "logstash not configured, not exporting template, should be done for us." ++ fi ++ # Above is non-fatal if there is a problem. Always pass. ++ exit 0 ++ ++ + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little +diff --git a/stable/logstash/templates/statefulset.yaml b/stable/logstash/templates/statefulset.yaml +index 4749bb1..d34dcec 100644 +--- a/stable/logstash/templates/statefulset.yaml ++++ b/stable/logstash/templates/statefulset.yaml +@@ -45,6 +45,36 @@ spec: + {{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet ++ initContainers: ++{{- if .Values.extraInitContainers }} ++{{ toYaml .Values.extraInitContainers | indent 8 }} ++ env: ++ ## Elasticsearch output ++ - name: ELASTICSEARCH_HOST ++ value: {{ .Values.elasticsearch.host | quote }} ++ - name: ELASTICSEARCH_PORT ++ value: {{ .Values.elasticsearch.port | quote }} ++ ## make system name visible for in setting index. ++ {{- if .Values.systemNameForIndex}} ++ {{ $systemNameNoSpecial := regexReplaceAll "[^A-Za-z0-9-]+" .Values.systemNameForIndex "" }} ++ {{ $systemNameFormatted := $systemNameNoSpecial | lower | trunc 200 }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: {{ printf "-%s" $systemNameFormatted | quote }} ++ {{- else }} ++ - name: SYSTEM_NAME_FOR_INDEX ++ value: "" ++ {{- end }} ++ ## Additional env vars ++ {{- range $key, $value := .Values.config }} ++ - name: {{ $key | upper | replace "." "_" }} ++ value: {{ $value | quote }} ++ {{- end }} ++ {{- if .Values.extraEnv }} ++{{ .Values.extraEnv | toYaml | indent 12 }} ++ {{- end }} ++ volumeMounts: ++{{ toYaml .Values.volumeMounts | indent 12 }} ++{{- end }} + containers: + + ## logstash +@@ -86,6 +116,9 @@ spec: + - name: {{ $key | upper | replace "." "_" }} + value: {{ $value | quote }} + {{- end }} ++ {{- if .Values.extraEnv }} ++{{ .Values.extraEnv | toYaml | indent 12 }} ++ {{- end }} + resources: + {{ toYaml .Values.resources | indent 12 }} + volumeMounts: +diff --git a/stable/logstash/values.yaml b/stable/logstash/values.yaml +index b8b8de2..2fdebde 100644 +--- a/stable/logstash/values.yaml ++++ b/stable/logstash/values.yaml +@@ -167,6 +167,16 @@ volumes: [] + # hostPath: + # path: /tmp + ++extraEnv: [] ++ ++extraInitContainers: [] ++ # - name: echo ++ # image: busybox ++ # imagePullPolicy: Always ++ # args: ++ # - echo ++ # - hello ++ + exporter: + logstash: + env: {} +diff --git a/stable/metricbeat/templates/configmap.yaml b/stable/metricbeat/templates/configmap.yaml +index 9ae276d..8b281b4 100644 +--- a/stable/metricbeat/templates/configmap.yaml ++++ b/stable/metricbeat/templates/configmap.yaml +@@ -1,77 +1,14 @@ + apiVersion: v1 + kind: ConfigMap + metadata: +- name: {{ template "metricbeat.fullname" . }} ++ name: {{ template "metricbeat.fullname" . }}-files + labels: + app: {{ template "metricbeat.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + data: +- setup-script.sh: |- +- #!/bin/bash +- # +- # This is best effort to load the template into elasticsearch +- # if beats are going to elasticsearch via logstash. +- # +- BEAT='metricbeat' +- BEAT_VER=$($BEAT version | awk '{print $3}') +- +- ADDR=$OUTPUT_ELASTICSEARCH_HOSTS +- ESPATH=$OUTPUT_ELASTICSEARCH_PATH +- +- echo $ADDR$ESPATH +- +- # Check if this is a map of entries. If so, we'll only export the template +- # to the first address +- if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then +- # Remove the square brackets +- ADDR=${ADDR:1:-1} +- # Take the first entry in case there are more than one +- ADDR=${ADDR%%,*} +- fi +- echo $ADDR$ESPATH +- +- # Check if user has formatted with http:// on front, if not we need to add +- HTTP='http://' +- if [[ ${ADDR} == http* ]]; then +- HTTP='' +- fi +- echo $HTTP +- +- # Check for ip address containing special characters where -g must be used in curl command +- # IPV6 Addresses should come in with square braces around the address. +- CURL_G='' +- if [[ ${ADDR} == *[* ]]; then +- CURL_G=' -g ' +- fi +- echo $CURL_G +- +- # check if logstash output is enabled, via crude parsing of the .yml file. +- sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true +- if [[ $? -eq 0 ]]; then +- echo "logstash configured, exporting template to elasticsearch" +- $BEAT export template > /tmp/beat.template.json +- # remove the lifecycle section of the yaml, as elasticsearch will choke +- # on it as oss version does not support ilm. +- sed -i '/lifecycle/,+3d' /tmp/beat.template.json +- +- # "unset" is the special system name when none has been set through +- # overrides.. We key on that to know what template name and index pattern to set +- if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then +- # replace the standard index pattern with one including our system name, +- # which will match our created indices +- sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json +- # give the template a name with the system name appended. +- curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json +- else +- # apply a higher order to this template in case there are templates with system names +- # which should be applied first +- sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json +- curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json +- fi +- else +- echo "logstash not configured, not exporting template, should be done for us." +- fi +- # Above is non-fatal if there is a problem. Always pass. +- exit 0 ++{{- range $key, $value := .Values.files }} ++ {{ $key }}: |- ++{{ $value | indent 4 }} ++{{- end }} +diff --git a/stable/metricbeat/templates/daemonset.yaml b/stable/metricbeat/templates/daemonset.yaml +index 247304e..581c120 100644 +--- a/stable/metricbeat/templates/daemonset.yaml ++++ b/stable/metricbeat/templates/daemonset.yaml +@@ -39,11 +39,10 @@ spec: + command: + - /bin/bash + - -c +- - /usr/share/metricbeat/setup-script.sh ++ - /usr/share/metricbeat/files/setup-script.sh + volumeMounts: +- - mountPath: /usr/share/metricbeat/setup-script.sh +- name: setupscript +- subPath: setup-script.sh ++ - name: files ++ mountPath: /usr/share/metricbeat/files + - name: config + mountPath: /usr/share/metricbeat/metricbeat.yml + readOnly: true +@@ -85,6 +84,9 @@ spec: + - name: INDEX_NAME + value: "{{ .Values.indexNamePrefix }}" + {{- end }} ++{{- if .Values.extraEnv }} ++{{ toYaml .Values.extraEnv | indent 8 }} ++{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +@@ -168,9 +170,8 @@ spec: + - name: dockersock + mountPath: /var/run/docker.sock + #Mount the script so we can see it from running containter +- - mountPath: /usr/share/metricbeat/setup-script.sh +- name: setupscript +- subPath: setup-script.sh ++ - name: files ++ mountPath: /usr/share/metricbeat/files + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | indent 8 }} + {{- end }} +@@ -194,9 +195,9 @@ spec: + - name: dockersock + hostPath: + path: /var/run/docker.sock +- - name: setupscript ++ - name: files + configMap: +- name: {{ template "metricbeat.fullname" . }} ++ name: {{ template "metricbeat.fullname" . }}-files + defaultMode: 0755 + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | indent 6 }} +diff --git a/stable/metricbeat/values.yaml b/stable/metricbeat/values.yaml +index b59c428..8f80bb6 100644 +--- a/stable/metricbeat/values.yaml ++++ b/stable/metricbeat/values.yaml +@@ -144,6 +144,89 @@ extraVolumeMounts: [] + # mountPath: /host/var/log + # readOnly: true + ++files: ++ setup-script.sh: |- ++ #!/bin/bash ++ # ++ # This is best effort to load the template into elasticsearch ++ # if beats are going to elasticsearch via logstash. ++ # ++ BEAT='metricbeat' ++ BEAT_VER=$($BEAT version | awk '{print $3}') ++ ++ ADDR=$OUTPUT_ELASTICSEARCH_HOSTS ++ ESPATH=$OUTPUT_ELASTICSEARCH_PATH ++ ++ echo $ADDR$ESPATH ++ ++ # Check if this is a map of entries. If so, we'll only export the template ++ # to the first address ++ if [ ${ADDR: 0:1} == [ ] && [ ${ADDR: -1} == ] ]; then ++ # Remove the square brackets ++ ADDR=${ADDR:1:-1} ++ # Take the first entry in case there are more than one ++ ADDR=${ADDR%%,*} ++ fi ++ echo $ADDR$ESPATH ++ ++ # Check if user has formatted with http:// on front, if not we need to add ++ HTTP='http://' ++ if [[ ${ADDR} == http* ]]; then ++ HTTP='' ++ fi ++ echo $HTTP ++ ++ # Check for ip address containing special characters where -g must be used in curl command ++ # IPV6 Addresses should come in with square braces around the address. ++ CURL_G='' ++ if [[ ${ADDR} == *[* ]]; then ++ CURL_G=' -g ' ++ fi ++ echo $CURL_G ++ ++ # check if logstash output is enabled, via crude parsing of the .yml file. ++ sed -e '/output.logstash/,/enabled:/!d' ${BEAT}.yml | grep -i true ++ if [[ $? -eq 0 ]]; then ++ echo "logstash configured, exporting template to elasticsearch" ++ $BEAT export template > /tmp/beat.template.json ++ # remove the lifecycle section of the yaml, as elasticsearch will choke ++ # on it as oss version does not support ilm. ++ sed -i '/lifecycle/,+3d' /tmp/beat.template.json ++ ++ # "unset" is the special system name when none has been set through ++ # overrides.. We key on that to know what template name and index pattern to set ++ if [[ $SYSTEM_NAME_FOR_INDEX != unset ]]; then ++ # replace the standard index pattern with one including our system name, ++ # which will match our created indices ++ sed -i "s/$BEAT-$BEAT_VER/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX/g" /tmp/beat.template.json ++ # give the template a name with the system name appended. ++ ++ # check if the template exists first before creating ++ curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX | grep "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX" ++ if [[ $? -ne 0 ]]; then ++ echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template not found, creating..." ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX -d@/tmp/beat.template.json ++ else ++ echo "$BEAT-$BEAT_VER$SYSTEM_NAME_FOR_INDEX template found, not creating..." ++ fi ++ else ++ # apply a higher order to this template in case there are templates with system names ++ # which should be applied first ++ curl -v $CURL_G -XGET -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER | grep "$BEAT-$BEAT_VER" ++ if [[ $? -ne 0 ]]; then ++ echo "$BEAT-$BEAT_VER template not found, creating..." ++ sed -i "s/\"order\": 1/\"order\": 2/g" /tmp/beat.template.json ++ curl -v $CURL_G -XPUT -H "Content-Type: application/json" $HTTP$ADDR$ESPATH/_template/$BEAT-$BEAT_VER -d@/tmp/beat.template.json ++ else ++ echo "$BEAT-$BEAT_VER template found, not creating..." ++ fi ++ fi ++ else ++ echo "logstash not configured, not exporting template, should be done for us." ++ fi ++ # Above is non-fatal if there is a problem. Always pass. ++ exit 0 ++ + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0016-use-main-container-image-for-initcontainer.patch b/monitor-helm/debian/deb_folder/patches/0016-use-main-container-image-for-initcontainer.patch new file mode 100644 index 0000000..d051d5b --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0016-use-main-container-image-for-initcontainer.patch @@ -0,0 +1,44 @@ +From 1e1ee29b195958f0c3313af86b6dae8f04720277 Mon Sep 17 00:00:00 2001 +From: Kevin Smith +Date: Thu, 26 Mar 2020 15:01:07 -0400 +Subject: [PATCH 1/1] use main container image for initcontainer + +--- + stable/logstash/templates/statefulset.yaml | 4 ++++ + stable/logstash/values.yaml | 5 +++++ + 2 files changed, 9 insertions(+) + +diff --git a/stable/logstash/templates/statefulset.yaml b/stable/logstash/templates/statefulset.yaml +index d34dcec..66f6313 100644 +--- a/stable/logstash/templates/statefulset.yaml ++++ b/stable/logstash/templates/statefulset.yaml +@@ -48,6 +48,10 @@ spec: + initContainers: + {{- if .Values.extraInitContainers }} + {{ toYaml .Values.extraInitContainers | indent 8 }} ++{{- if .Values.useMainImageForInitContainers }} ++ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" ++ imagePullPolicy: {{ .Values.image.pullPolicy }} ++{{- end }} + env: + ## Elasticsearch output + - name: ELASTICSEARCH_HOST +diff --git a/stable/logstash/values.yaml b/stable/logstash/values.yaml +index 2fdebde..2167601 100644 +--- a/stable/logstash/values.yaml ++++ b/stable/logstash/values.yaml +@@ -169,6 +169,11 @@ volumes: [] + + extraEnv: [] + ++# Init container will use the same image as the main ++# container if this is true. If true, do not specify ++# the image in the extraInitContainers block. If false, ++# will need to specify the image. ++useMainImageForInitContainers: true + extraInitContainers: [] + # - name: echo + # image: busybox +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/0017-stable-nginx-ingress-allow-nodePort-for-tcp-udp-serv.patch b/monitor-helm/debian/deb_folder/patches/0017-stable-nginx-ingress-allow-nodePort-for-tcp-udp-serv.patch new file mode 100644 index 0000000..5368924 --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0017-stable-nginx-ingress-allow-nodePort-for-tcp-udp-serv.patch @@ -0,0 +1,58 @@ +From 872dd678b0ea08be23700337665ea2125dab4d0d Mon Sep 17 00:00:00 2001 +From: Matt Peters +Date: Thu, 23 Apr 2020 18:39:16 -0500 +Subject: [PATCH] [stable/nginx-ingress] allow nodePort for tcp/udp services + +--- + stable/nginx-ingress/templates/controller-service.yaml | 10 ++++++++++ + stable/nginx-ingress/values.yaml | 4 ++++ + 2 files changed, 14 insertions(+) + +diff --git a/stable/nginx-ingress/templates/controller-service.yaml b/stable/nginx-ingress/templates/controller-service.yaml +index a81c81e..6389c14 100644 +--- a/stable/nginx-ingress/templates/controller-service.yaml ++++ b/stable/nginx-ingress/templates/controller-service.yaml +@@ -60,12 +60,22 @@ spec: + port: {{ $key }} + protocol: TCP + targetPort: "{{ $key }}-tcp" ++ {{- if $.Values.controller.service.nodePorts.tcp }} ++ {{- if index $.Values.controller.service.nodePorts.tcp $key }} ++ nodePort: {{ index $.Values.controller.service.nodePorts.tcp $key }} ++ {{- end }} ++ {{- end }} + {{- end }} + {{- range $key, $value := .Values.udp }} + - name: "{{ $key }}-udp" + port: {{ $key }} + protocol: UDP + targetPort: "{{ $key }}-udp" ++ {{- if $.Values.controller.service.nodePorts.udp }} ++ {{- if index $.Values.controller.service.nodePorts.udp $key }} ++ nodePort: {{ index $.Values.controller.service.nodePorts.udp $key }} ++ {{- end }} ++ {{- end }} + {{- end }} + selector: + app: {{ template "nginx-ingress.name" . }} +diff --git a/stable/nginx-ingress/values.yaml b/stable/nginx-ingress/values.yaml +index 80bc1a6..71bbcb3 100644 +--- a/stable/nginx-ingress/values.yaml ++++ b/stable/nginx-ingress/values.yaml +@@ -192,9 +192,13 @@ controller: + # nodePorts: + # http: 32080 + # https: 32443 ++ # tcp: ++ # 8080: 32808 + nodePorts: + http: "" + https: "" ++ tcp: {} ++ udp: {} + + extraContainers: [] + ## Additional containers to be added to the controller pod. +-- +2.24.0 + diff --git a/monitor-helm/debian/deb_folder/patches/0018-Update-nginx-ingress-chart-for-Helm-v3.patch b/monitor-helm/debian/deb_folder/patches/0018-Update-nginx-ingress-chart-for-Helm-v3.patch new file mode 100644 index 0000000..9b5ebfa --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/0018-Update-nginx-ingress-chart-for-Helm-v3.patch @@ -0,0 +1,22 @@ +From 8f31965ee80e72ef57b50481c2eec6ba30d89d13 Mon Sep 17 00:00:00 2001 +From: Jim Gauld +Date: Thu, 9 Apr 2020 10:50:50 -0400 +Subject: [PATCH] Update nginx-ingress chart for Helm v3 + +Signed-off-by: Jim Gauld +--- + stable/nginx-ingress/Chart.yaml | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/stable/nginx-ingress/Chart.yaml b/stable/nginx-ingress/Chart.yaml +index a9d64ed..6574c3f 100644 +--- a/stable/nginx-ingress/Chart.yaml ++++ b/stable/nginx-ingress/Chart.yaml +@@ -1,3 +1,4 @@ ++apiVersion: v1 + name: nginx-ingress + version: 1.4.0 + appVersion: 0.23.0 +-- +1.8.3.1 + diff --git a/monitor-helm/debian/deb_folder/patches/series b/monitor-helm/debian/deb_folder/patches/series new file mode 100644 index 0000000..15f9b52 --- /dev/null +++ b/monitor-helm/debian/deb_folder/patches/series @@ -0,0 +1,18 @@ +0001-Add-Makefile-for-helm-charts.patch +0002-kibana-workaround-checksum-for-configmap.yaml.patch +0003-helm-chart-changes-for-stx-monitor.patch +0004-ipv6-helm-chart-changes.patch +0005-decouple-config.patch +0006-add-system-info.patch +0007-three-masters.patch +0008-Update-stx-monitor-for-kubernetes-API-1.16.patch +0009-add-curator-as-of-2019-10-10.patch +0010-Update-kube-state-metrics-1.8.0-to-commit-09daf19.patch +0011-update-init-container-env-to-include-node-name.patch +0012-Add-imagePullSecrets.patch +0013-removed-unused-images.patch +0014-Add-rbac-replicasets-to-apps-apigroup-commit-1717e2d.patch +0015-script-flexibility.patch +0016-use-main-container-image-for-initcontainer.patch +0017-stable-nginx-ingress-allow-nodePort-for-tcp-udp-serv.patch +0018-Update-nginx-ingress-chart-for-Helm-v3.patch diff --git a/monitor-helm/debian/deb_folder/rules b/monitor-helm/debian/deb_folder/rules index bad0564..a7f9a8c 100755 --- a/monitor-helm/debian/deb_folder/rules +++ b/monitor-helm/debian/deb_folder/rules @@ -8,27 +8,6 @@ export HELM_FOLDER = $(ROOT)/usr/lib/helm dh $@ override_dh_auto_build: - # Move the source files from the extracted root directory to build root. - mv helm-charts/* . - # Apply the openstack-helm patches. - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0001-Add-Makefile-for-helm-charts.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0002-kibana-workaround-checksum-for-configmap.yaml.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0003-helm-chart-changes-for-stx-monitor.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0004-ipv6-helm-chart-changes.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0005-decouple-config.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0006-add-system-info.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0007-three-masters.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0008-Update-stx-monitor-for-kubernetes-API-1.16.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0009-add-curator-as-of-2019-10-10.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0010-Update-kube-state-metrics-1.8.0-to-commit-09daf19.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0011-update-init-container-env-to-include-node-name.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0012-Add-imagePullSecrets.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0013-removed-unused-images.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0014-Add-rbac-replicasets-to-apps-apigroup-commit-1717e2d.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0015-script-flexibility.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0016-use-main-container-image-for-initcontainer.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0017-stable-nginx-ingress-allow-nodePort-for-tcp-udp-serv.patch - patch --no-backup-if-mismatch --prefix=/tmp/junk -p1 < 0018-Update-nginx-ingress-chart-for-Helm-v3.patch # Host a server for the helm charts. chartmuseum --debug --port=8879 --context-path='/charts' --storage="local" \ --storage-local-rootdir="." & diff --git a/monitor-helm/debian/dl_hook b/monitor-helm/debian/dl_hook deleted file mode 100755 index 45eb7e6..0000000 --- a/monitor-helm/debian/dl_hook +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -x - -PKG_BUILD_NAME=$1 -PKG_BUILD_ROOT=$(realpath `pwd`/${PKG_BUILD_NAME}) -PKG_DIR="monitor-helm" -STX_BASE=$(realpath ${MY_REPO}/stx) -SRC=$(realpath ${STX_BASE}/monitor-armada-app/${PKG_DIR}) - -mkdir -p ${PKG_BUILD_ROOT} -pushd ${PKG_BUILD_ROOT} - -# Local mirror workaround until CGCS_BASE mirror is supported. -STX_MIRROR=$(realpath "/import/mirrors/starlingx") - -# Download monitor helm chart source package. -MON_PKG="helm-charts-92b6289ae93816717a8453cfe62bad51cbdb8ad0.tar.gz" -MON_SRC_PATH=$(realpath ${STX_MIRROR}/downloads/${MON_PKG}) -cp ${MON_SRC_PATH} ${PKG_BUILD_ROOT} - -# Extract the monitor helm chart tar file. -tar xfzvv ${MON_PKG} - -# Copy source files to the build directory. -cp -pr ${SRC}/files/* ${PKG_BUILD_ROOT} diff --git a/monitor-helm/debian/meta_data.yaml b/monitor-helm/debian/meta_data.yaml index 81b78cd..3a6d853 100644 --- a/monitor-helm/debian/meta_data.yaml +++ b/monitor-helm/debian/meta_data.yaml @@ -1,7 +1,13 @@ --- debname: monitor-helm debver: 1.0-1 -dl_hook: dl_hook +dl_path: + name: helm-charts-92b6289ae93816717a8453cfe62bad51cbdb8ad0.tar.gz + url: https://github.com/helm/charts/archive/92b6289ae93816717a8453cfe62bad51cbdb8ad0.tar.gz + md5sum: d32b2ad945a10441d8e1b014bddd8cb8 +src_files: + - files/index.yaml + - files/repositories.yaml revision: dist: $STX_DIST PKG_GITREVCOUNT: true