From 36331e1d097bcec2d0375e0a1672a07a1cec849b Mon Sep 17 00:00:00 2001 From: Daniel Liu Date: Tue, 22 Mar 2022 18:12:44 -0400 Subject: [PATCH] init --- Chart.yaml | 14 +++++ README.md | 3 + requirements.lock | 24 ++++++++ requirements.yaml | 29 +++++++++ templates/NOTES.txt | 3 + templates/_helpers.tpl | 58 ++++++++++++++++++ templates/datasources.yaml | 31 ++++++++++ templates/tests/loki-test-configmap.yaml | 52 ++++++++++++++++ templates/tests/loki-test-pod.yaml | 30 +++++++++ values.yaml | 78 ++++++++++++++++++++++++ 10 files changed, 322 insertions(+) create mode 100644 Chart.yaml create mode 100644 README.md create mode 100644 requirements.lock create mode 100644 requirements.yaml create mode 100644 templates/NOTES.txt create mode 100644 templates/_helpers.tpl create mode 100644 templates/datasources.yaml create mode 100644 templates/tests/loki-test-configmap.yaml create mode 100644 templates/tests/loki-test-pod.yaml create mode 100644 values.yaml diff --git a/Chart.yaml b/Chart.yaml new file mode 100644 index 0000000..54f5076 --- /dev/null +++ b/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: "v1" +name: loki-stack +version: 2.6.1 +appVersion: v2.1.0 +kubeVersion: "^1.10.0-0" +description: "Loki: like Prometheus, but for logs." +home: https://grafana.com/loki +icon: https://raw.githubusercontent.com/grafana/loki/master/docs/sources/logo.png +sources: +- https://github.com/grafana/loki +maintainers: +- name: Loki Maintainers + email: lokiproject@googlegroups.com +engine: gotpl diff --git a/README.md b/README.md new file mode 100644 index 0000000..ea688b9 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ + +# prom-loki + diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..7fa52fc --- /dev/null +++ b/requirements.lock @@ -0,0 +1,24 @@ +dependencies: +- name: loki + repository: https://grafana.github.io/helm-charts + version: 2.10.1 +- name: promtail + repository: https://grafana.github.io/helm-charts + version: 2.2.0 +- name: fluent-bit + repository: https://grafana.github.io/helm-charts + version: 2.3.0 +- name: grafana + repository: https://grafana.github.io/helm-charts + version: 6.21.8 +- name: prometheus + repository: https://prometheus-community.github.io/helm-charts + version: 15.0.4 +- name: filebeat + repository: https://helm.elastic.co + version: 7.17.1 +- name: logstash + repository: https://helm.elastic.co + version: 7.8.1 +digest: sha256:18e9f3020b565448367602142683ed984b872ec4f9e4eb568f193341265ad550 +generated: "2022-03-09T19:18:00.699336741-05:00" diff --git a/requirements.yaml b/requirements.yaml new file mode 100644 index 0000000..f854da5 --- /dev/null +++ b/requirements.yaml @@ -0,0 +1,29 @@ +dependencies: +- name: "loki" + condition: loki.enabled + repository: "https://grafana.github.io/helm-charts" + version: "^2.3.0" +- name: "promtail" + condition: promtail.enabled + repository: "https://grafana.github.io/helm-charts" + version: "^2.2.0" +- name: "fluent-bit" + condition: fluent-bit.enabled + repository: "https://grafana.github.io/helm-charts" + version: "^2.2.0" +- name: "grafana" + condition: grafana.enabled + version: "~6.21.2" + repository: "https://grafana.github.io/helm-charts" +- name: "prometheus" + condition: prometheus.enabled + version: "~15.0.1" + repository: "https://prometheus-community.github.io/helm-charts" +- name: "filebeat" + condition: filebeat.enabled + version: "~7.17.0" + repository: "https://helm.elastic.co" +- name: "logstash" + condition: logstash.enabled + version: "~7.8.0" + repository: "https://helm.elastic.co" diff --git a/templates/NOTES.txt b/templates/NOTES.txt new file mode 100644 index 0000000..d9cdccb --- /dev/null +++ b/templates/NOTES.txt @@ -0,0 +1,3 @@ +The Loki stack has been deployed to your cluster. Loki can now be added as a datasource in Grafana. + +See http://docs.grafana.org/features/datasources/loki/ for more detail. diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl new file mode 100644 index 0000000..1cb7269 --- /dev/null +++ b/templates/_helpers.tpl @@ -0,0 +1,58 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "loki-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "loki-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "loki-stack.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Override the naming defined by the prometheus chart. +Added as a fix for https://github.com/grafana/loki/issues/1169 +*/}} +{{- define "prometheus.fullname" -}} +{{- printf "%s-%s" .Release.Name "prometheus-server" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +The service name to connect to Loki. Defaults to the same logic as "loki.fullname" +*/}} +{{- define "loki.serviceName" -}} +{{- if .Values.loki.serviceName -}} +{{- .Values.loki.serviceName -}} +{{- else if .Values.loki.fullnameOverride -}} +{{- .Values.loki.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "loki" .Values.loki.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/templates/datasources.yaml b/templates/datasources.yaml new file mode 100644 index 0000000..29e2f4f --- /dev/null +++ b/templates/datasources.yaml @@ -0,0 +1,31 @@ +{{- if .Values.grafana.sidecar.datasources.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "loki-stack.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "loki-stack.name" . }} + chart: {{ template "loki-stack.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + grafana_datasource: "1" +data: + loki-stack-datasource.yaml: |- + apiVersion: 1 + datasources: +{{- if .Values.loki.enabled }} + - name: Loki + type: loki + access: proxy + url: http://{{(include "loki.serviceName" .)}}:{{ .Values.loki.service.port }} + version: 1 +{{- end }} +{{- if .Values.prometheus.enabled }} + - name: Prometheus + type: prometheus + access: proxy + url: http://{{ include "prometheus.fullname" .}}:{{ .Values.prometheus.server.service.servicePort }}{{ .Values.prometheus.server.prefixURL }} + version: 1 +{{- end }} +{{- end }} diff --git a/templates/tests/loki-test-configmap.yaml b/templates/tests/loki-test-configmap.yaml new file mode 100644 index 0000000..3e53cfd --- /dev/null +++ b/templates/tests/loki-test-configmap.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "loki-stack.fullname" . }}-test + labels: + app: {{ template "loki-stack.name" . }} + chart: {{ template "loki-stack.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + test.sh: | + #!/usr/bin/env bash + + LOKI_URI="http://${LOKI_SERVICE}:${LOKI_PORT}" + + function setup() { + apk add -u curl jq + until (curl -s ${LOKI_URI}/api/prom/label/app/values | jq -e '.values[] | select(. == "loki")'); do + sleep 1 + done + } + + @test "Has labels" { + curl -s ${LOKI_URI}/api/prom/label | \ + jq -e '.values[] | select(. == "app")' + } + + @test "Query log entry" { + curl -sG ${LOKI_URI}/api/prom/query?limit=10 --data-urlencode 'query={app="loki"}' | \ + jq -e '.streams[].entries | length >= 1' + } + + @test "Push log entry legacy" { + local timestamp=$(date -Iseconds -u | sed 's/UTC/.000000000+00:00/') + local data=$(jq -n --arg timestamp "${timestamp}" '{"streams": [{"labels": "{app=\"loki-test\"}", "entries": [{"ts": $timestamp, "line": "foobar"}]}]}') + + curl -s -X POST -H "Content-Type: application/json" ${LOKI_URI}/api/prom/push -d "${data}" + + curl -sG ${LOKI_URI}/api/prom/query?limit=1 --data-urlencode 'query={app="loki-test"}' | \ + jq -e '.streams[].entries[].line == "foobar"' + } + + @test "Push log entry" { + local timestamp=$(date +%s000000000) + local data=$(jq -n --arg timestamp "${timestamp}" '{"streams": [{"stream": {"app": "loki-test"}, "values": [[$timestamp, "foobar"]]}]}') + + curl -s -X POST -H "Content-Type: application/json" ${LOKI_URI}/loki/api/v1/push -d "${data}" + + curl -sG ${LOKI_URI}/api/prom/query?limit=1 --data-urlencode 'query={app="loki-test"}' | \ + jq -e '.streams[].entries[].line == "foobar"' + } + diff --git a/templates/tests/loki-test-pod.yaml b/templates/tests/loki-test-pod.yaml new file mode 100644 index 0000000..a153a05 --- /dev/null +++ b/templates/tests/loki-test-pod.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + "helm.sh/hook": test-success + labels: + app: {{ template "loki-stack.name" . }} + chart: {{ template "loki-stack.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "loki-stack.fullname" . }}-test +spec: + containers: + - name: test + image: bats/bats:v1.1.0 + args: + - /var/lib/loki/test.sh + env: + - name: LOKI_SERVICE + value: {{ template "loki.serviceName" . }} + - name: LOKI_PORT + value: "{{ .Values.loki.service.port }}" + volumeMounts: + - name: tests + mountPath: /var/lib/loki + restartPolicy: Never + volumes: + - name: tests + configMap: + name: {{ template "loki-stack.fullname" . }}-test diff --git a/values.yaml b/values.yaml new file mode 100644 index 0000000..6ec1143 --- /dev/null +++ b/values.yaml @@ -0,0 +1,78 @@ + +loki: + enabled: true + +grafana: + enabled: true + + # TODO change this + adminPassword: prom-operator + + ingress: + enabled: true + ingressClassName: nginx + + path: / + pathType: Prefix + hosts: + - grafana.ing + +prometheus: + enabled: true + +filebeat: + enabled: true + filebeatConfig: + filebeat.yml: | + logging.level: debug + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" + output.logstash: + hosts: ["logstash-loki:5044"] + +logstash: + enabled: true + image: grafana/logstash-output-loki + imageTag: 1.0.1 + filters: + main: |- + filter { + if [kubernetes] { + mutate { + add_field => { + "container_name" => "%{[kubernetes][container][name]}" + "namespace" => "%{[kubernetes][namespace]}" + "pod" => "%{[kubernetes][pod][name]}" + } + replace => { "host" => "%{[kubernetes][node][name]}"} + } + } + mutate { + remove_field => ["tags"] + } + } + outputs: + main: |- + output { + loki { + url => "http://loki:3100/loki/api/v1/push" + username => "test" + password => "test" + } + # stdout { codec => rubydebug } + } + +promtail: + enabled: false + +fluent-bit: + enabled: false +