Jsonnet: Add Grafana Enterprise Logs library (#4165)

* feat: Add enterprise-logs Jsonnet library Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * fixup: Use correct image for GEL Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * fix: Separate k8s-libsonnet dependency Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * chore: Simplify Makefile Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * fix: Remove extra comma in jsonnetfile.json Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * fix: Remove unused prerequisite of test evaluation Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * doc: Remove unnecessary comments Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * fix: Remove storageClassName from PVCs Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * feat: Ensure all GEL Pods run as non-root user Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * feat: Configure memberlist for a more stable ring Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * style: Simplify gossip-ring Service ports configuration Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * chore: Update dependency lock Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * chore: Add convenience test target Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * fix: Disable tokengen Job by default Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * doc: Add TODO for utility functions Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * chore: Add tooling for developer testing Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
4 years ago · f38126473c
parent de5d2628f6
commit f38126473c
11 changed files with 532 additions and 0 deletions
--- a/production/ksonnet/enterprise-logs/.gitignore
+++ b/production/ksonnet/enterprise-logs/.gitignore
@ -0,0 +1,3 @@
+.eval
+lib
+vendor
--- a/production/ksonnet/enterprise-logs/Makefile
+++ b/production/ksonnet/enterprise-logs/Makefile
@ -0,0 +1,50 @@
+include variables.mk
+
+.ONESHELL:
+.DELETE_ON_ERROR:
+export SHELL     := bash
+export SHELLOPTS := pipefail:errexit
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rule
+
+# Adapted from https://suva.sh/posts/well-documented-makefiles/
+.PHONY: help
+help: ## Display this help.
+help:
+	@awk 'BEGIN {FS = ": ##"; printf "Usage:\n  make <target>\n\nTargets:\n"} /^[a-zA-Z0-9_\.\-\/% ]+: ##/ { printf "  %-45s %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
+
+jsonnetfile.lock.json: ## Update the locked dependency versions for the library.
+jsonnetfile.lock.json: jsonnetfile.json
+	jb update
+
+test: ## Evaluate the library Jsonnet.
+test: test/.eval
+	:
+
+test/.eval: # Cache testing results.
+test/.eval: test/eval.jsonnet main.libsonnet test/lib/k.libsonnet test/vendor
+	cd $(@D); tmp=$$(mktemp); if tk eval $(<F) | tee $${tmp}; then mv $${tmp} $(@F); fi
+
+test/lib:
+	mkdir -p $@
+
+test/lib/k.libsonnet: # Install implicit k.libsonnet dependency used in testing.
+test/lib/k.libsonnet: test/lib
+	printf "(import 'github.com/jsonnet-libs/k8s-libsonnet/%s/main.libsonnet')" $(K8S_VERSION) > $@
+
+test/jsonnetfile.json: ## Update the jsonnetfile used in testing.
+test/jsonnetfile.json: test/jsonnetfile.jsonnet variables.mk jsonnetfile.json
+	jsonnet --tla-str k8sVersion=$(K8S_VERSION) $< > $@
+
+test/jsonnetfile.lock.json test/vendor: ## Update the locked dependency versions used in testing.
+test/jsonnetfile.lock.json test/vendor: test/jsonnetfile.json
+	cd $(@D); jb update
+
+.PHONY: k3d-cluster-create
+k3d-cluster-create: ## Create a development k3d cluster.
+	./scripts/k3d-cluster create
+
+.PHONY: k3d-cluster-create
+k3d-cluster-delete: ## Delete a development k3d cluster.
+k3d-cluster-delete:
+	./scripts/k3d-cluster delete
--- a/production/ksonnet/enterprise-logs/jsonnetfile.json
+++ b/production/ksonnet/enterprise-logs/jsonnetfile.json
@ -0,0 +1,24 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/loki.git",
+          "subdir": "production/ksonnet/loki"
+        }
+      },
+      "version": "main"
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs",
+          "subdir": "ksonnet-util"
+        }
+      },
+      "version": "master"
+    }
+  ],
+  "legacyImports": true
+}
--- a/production/ksonnet/enterprise-logs/jsonnetfile.lock.json
+++ b/production/ksonnet/enterprise-logs/jsonnetfile.lock.json
@ -0,0 +1,66 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "consul"
+        }
+      },
+      "version": "00795013f5975f518a0a3de99253f9d5590271c8",
+      "sum": "Po3c1Ic96ngrJCtOazic/7OsLkoILOKZWXWyZWl+od8="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "jaeger-agent-mixin"
+        }
+      },
+      "version": "00795013f5975f518a0a3de99253f9d5590271c8",
+      "sum": "DsdBoqgx5kE3zc6fMYnfiGjW2+9Mx2OXFieWm1oFHgY="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "ksonnet-util"
+        }
+      },
+      "version": "00795013f5975f518a0a3de99253f9d5590271c8",
+      "sum": "OxgtIWL4hjvG0xkMwUzZ7Yjs52zUhLhaVQpwHCbqf8A="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "memcached"
+        }
+      },
+      "version": "00795013f5975f518a0a3de99253f9d5590271c8",
+      "sum": "dTOeEux3t9bYSqP2L/uCuLo/wUDpCKH4w+4OD9fePUk="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/loki.git",
+          "subdir": "production/ksonnet/loki"
+        }
+      },
+      "version": "9ea59f2062016d91398387ee2231e2e840af6a06",
+      "sum": "i27fS9sssvYd9Ywyq6uoB52EUWTaOPxo9DczCBVBuaM="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/jsonnet-libs/k8s-libsonnet.git",
+          "subdir": "1.18"
+        }
+      },
+      "version": "91008dbd2ea5734288467d6dcafef7c285c3f7e6",
+      "sum": "x881PM+6ARMsa9OSJcxO6L+4GOBy91clZipjeYbbzpw="
+    }
+  ],
+  "legacyImports": false
+}
--- a/production/ksonnet/enterprise-logs/main.libsonnet
+++ b/production/ksonnet/enterprise-logs/main.libsonnet
@ -0,0 +1,220 @@
+// TODO(jdb): Use cluster_dns_suffix to configure absolute domain names so as to avoid excessive lookups.
+// TODO(jdb): Introduce utility functions for mapping over all containers, all microservices (modules), all apps, etc..
+local k = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet',
+      clusterRole = k.rbac.v1.clusterRole,
+      clusterRoleBinding = k.rbac.v1.clusterRoleBinding,
+      container = k.core.v1.container,
+      configMap = k.core.v1.configMap,
+      deployment = k.apps.v1.deployment,
+      job = k.batch.v1.job,
+      policyRule = k.rbac.v1.policyRule,
+      pvc = k.core.v1.persistentVolumeClaim,
+      service = k.core.v1.service,
+      serviceAccount = k.core.v1.serviceAccount,
+      subject = k.rbac.v1.subject,
+      statefulSet = k.apps.v1.statefulSet;
+local loki = import 'github.com/grafana/loki/production/ksonnet/loki/loki.libsonnet';
+local util = (import 'github.com/grafana/jsonnet-libs/ksonnet-util/util.libsonnet').withK(k) {
+  withNonRootSecurityContext(uid, fsGroup=null)::
+    { spec+: { template+: { spec+: { securityContext: {
+      fsGroup: if fsGroup == null then uid else fsGroup,
+      runAsNonRoot: true,
+      runAsUser: uid,
+    } } } } },
+};
+
+loki {
+  _config+:: {
+    commonArgs+:: {
+      'auth.enabled': 'true',
+      'auth.type': 'enterprise',
+      'cluster-name':
+        if self['auth.type'] == 'enterprise' then
+          error 'must set _config.commonArgs.cluster-name'
+        else null,
+    },
+
+    loki+: {
+      distributor+: {
+        ring: {
+          kvstore: {
+            store: 'memberlist',
+          },
+        },
+      },
+      ingester+: {
+        lifecycler+: {
+          ring: {
+            kvstore: {
+              store: 'memberlist',
+            },
+          },
+        },
+      },
+      memberlist: {
+        retransmit_factor: 2,
+        gossip_interval: '5s',
+        stream_timeout: '5s',
+        abort_if_cluster_join_fails: false,
+        bind_port: 7946,
+        join_members: ['gossip-ring'],
+      },
+    },
+
+    ingester_pvc_size: '50Gi',
+    stateful_ingesters: true,
+
+    querier_pvc_size: '50Gi',
+    stateful_queriers: true,
+
+    compactor_pvc_size: '50Gi',
+  },
+
+  _images+:: {
+    loki: 'grafana/enterprise-logs:v1.1.0',
+    kubectl: 'bitnami/kubectl',
+  },
+
+  admin_api_args:: self._config.commonArgs {
+    'bootstrap.license.path': '/etc/gel-license/license.jwt',
+    target: 'admin-api',
+  },
+  admin_api_container::
+    container.new(name='admin-api', image=self._images.loki)
+    + container.withArgs(util.mapToFlags(self.admin_api_args))
+    + container.withPorts(loki.util.defaultPorts)
+    + container.resources.withLimits({ cpu: '2', memory: '4Gi' })
+    + container.resources.withRequests({ cpu: '500m', memory: '1Gi' })
+    + loki.util.readinessProbe,
+  admin_api_deployment:
+    deployment.new(name='admin-api', replicas=3, containers=[self.admin_api_container])
+    + deployment.spec.selector.withMatchLabelsMixin({ name: 'admin-api' })
+    + deployment.spec.template.metadata.withLabelsMixin({ name: 'admin-api', gossip_ring_member: 'true' })
+    + deployment.spec.template.spec.withTerminationGracePeriodSeconds(15)
+    + util.configVolumeMount('loki', '/etc/loki/config')
+    + util.secretVolumeMount('gel-license', '/etc/gel-license/')
+    + util.withNonRootSecurityContext(uid=10001),
+  admin_api_service:
+    util.serviceFor(self.admin_api_deployment),
+
+  compactor_data_pvc+: { spec+: { storageClassName:: null } },
+  compactor_statefulset+: util.withNonRootSecurityContext(10001),
+
+  // Remove consul in favor of memberlist.
+  consul_config_map:: {},
+  consul_deployment:: null,
+  consul_service:: null,
+
+  distributor_deployment+:
+    deployment.spec.template.metadata.withLabelsMixin({ gossip_ring_member: 'true' })
+    + util.withNonRootSecurityContext(uid=10001),
+
+  gateway_args:: self._config.commonArgs {
+    'bootstrap.license.path': '/etc/gel-license/license.jwt',
+    'gateway.proxy.admin-api.url': 'http://admin-api:%s' % $._config.http_listen_port,
+    'gateway.proxy.compactor.url': 'http://compactor:%s' % $._config.http_listen_port,
+    'gateway.proxy.distributor.url': 'dns:///distributor:9095',
+    'gateway.proxy.ingester.url': 'http://ingester:%s' % $._config.http_listen_port,
+    'gateway.proxy.query-frontend.url': 'http://query-frontend:%s' % $._config.http_listen_port,
+    'gateway.proxy.ruler.url': 'http://ruler:%s' % $._config.http_listen_port,
+    target: 'gateway',
+  },
+  gateway_container::
+    container.new(name='gateway', image=self._images.loki)
+    + container.withArgs(util.mapToFlags(self.gateway_args))
+    + container.withPorts(loki.util.defaultPorts)
+    + container.resources.withLimits({ cpu: '2', memory: '4Gi' })
+    + container.resources.withRequests({ cpu: '500m', memory: '1Gi' })
+    + loki.util.readinessProbe,
+  gateway_deployment:
+    deployment.new(name='gateway', replicas=3, containers=[self.gateway_container])
+    + deployment.spec.template.spec.withTerminationGracePeriodSeconds(15)
+    + deployment.spec.template.spec.securityContext.withFsGroup(10001)
+    + deployment.spec.template.spec.securityContext.withRunAsNonRoot(true)
+    + deployment.spec.template.spec.securityContext.withRunAsUser(10001)
+    + util.configVolumeMount('loki', '/etc/loki/config')
+    + util.withNonRootSecurityContext(uid=10001),
+  // Remove the htpasswd Secret used by the OSS Loki NGINX gateway.
+  gateway_secret:: null,
+  gateway_service:
+    util.serviceFor(self.gateway_deployment),
+
+  ingester_data_pvc+: { spec+: { storageClassName:: null } },
+  ingester_statefulset+:
+    statefulSet.spec.template.metadata.withLabelsMixin({ gossip_ring_member: 'true' })
+    + util.withNonRootSecurityContext(uid=10001),
+
+  querier_data_pvc+: { spec+: { storageClassName:: null } },
+  querier_statefulset+:
+    statefulSet.spec.template.metadata.withLabelsMixin({ gossip_ring_member: 'true' })
+    + util.withNonRootSecurityContext(uid=10001),
+
+  table_manager_deployment+: util.withNonRootSecurityContext(uid=10001),
+
+  tokengen_args:: self._config.commonArgs {
+    target: 'tokengen',
+    'tokengen.token-file': '/shared/admin-token',
+  },
+  tokengen_container::
+    container.new('tokengen', self._images.loki)
+    + container.withPorts(loki.util.defaultPorts)
+    + container.withArgs(util.mapToFlags(self.tokengen_args))
+    + container.withVolumeMounts([
+      { mountPath: '/etc/loki/config', name: 'config' },
+      { mountPath: '/shared', name: 'shared' },
+    ])
+    + container.resources.withLimits({ memory: '4Gi' })
+    + container.resources.withRequests({ cpu: '500m', memory: '500Mi' }),
+  tokengen_create_secret_container::
+    container.new('create-secret', self._images.kubectl)
+    + container.withCommand([
+      '/bin/bash',
+      '-euc',
+      'kubectl create secret generic gel-admin-token --from-file=token=/shared/admin-token --from-literal=grafana-token="self.base64 <(echo :self.cat /shared/admin-token)))"',
+    ])
+    + container.withVolumeMounts([{ mountPath: '/shared', name: 'shared' }]),
+  tokengen_job::
+    job.new('tokengen')
+    + job.spec.withCompletions(1)
+    + job.spec.withParallelism(1)
+    + job.spec.template.spec.withContainers([self.tokengen_create_secret_container])
+    + job.spec.template.spec.withInitContainers([self.tokengen_container])
+    + job.spec.template.spec.withRestartPolicy('Never')
+    + job.spec.template.spec.withServiceAccount('tokengen')
+    + job.spec.template.spec.withServiceAccountName('tokengen')
+    + job.spec.template.spec.withVolumes([
+      { name: 'config', configMap: { name: 'loki' } },
+      { name: 'shared', emptyDir: {} },
+    ])
+    + util.withNonRootSecurityContext(uid=10001),
+  tokengen_service_account:
+    serviceAccount.new('tokengen'),
+  tokengen_cluster_role:
+    clusterRole.new('tokengen')
+    + clusterRole.withRules([
+      policyRule.withApiGroups([''])
+      + policyRule.withResources(['secrets'])
+      + policyRule.withVerbs(['create']),
+    ]),
+  tokengen_cluster_role_binding:
+    clusterRoleBinding.new()
+    + clusterRoleBinding.metadata.withName('tokengen')
+    + clusterRoleBinding.roleRef.withApiGroup('rbac.authorization.k8s.io')
+    + clusterRoleBinding.roleRef.withKind('ClusterRole')
+    + clusterRoleBinding.roleRef.withName('tokengen')
+    + clusterRoleBinding.withSubjects([
+      subject.new()
+      + subject.withName('tokengen')
+      + subject.withKind('ServiceAccount')
+      + { namespace: $._config.namespace },
+    ]),
+
+  gossip_ring_service:
+    service.new(
+      name='gossip-ring',
+      selector={ gossip_ring_member: 'true' },
+      ports=[{ name: 'gossip-ring', port: 7946, protocol: 'TCP', targetPort: 7946 }],
+    )
+    + service.spec.withClusterIp('None')
+    + service.spec.withPublishNotReadyAddresses(true),
+}
--- a/production/ksonnet/enterprise-logs/scripts/k3d-cluster
+++ b/production/ksonnet/enterprise-logs/scripts/k3d-cluster
@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+  cat <<EOF
+Create or delete a dev k3d cluster
+
+Usage:
+  $0 create
+  $0 delete
+EOF
+fi
+
+CLUSTER_NAME=enterprise-logs
+
+case $1 in
+  create)
+    shift
+    # Make creation idempotent.
+    if k3d cluster list "${CLUSTER_NAME}" &>/dev/null; then
+      exit 0
+    fi
+    k3d cluster create "${CLUSTER_NAME}"
+
+    echo -n 'creating'
+    set +e
+    while ! k3d kubeconfig get "${CLUSTER_NAME}" &>/dev/null; do
+      sleep 1
+      echo -n '.'
+    done
+    set -e
+    echo 'done'
+
+    kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml
+    kubectl patch storageclass local-path -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+
+    kubectl config use-context k3d-"${CLUSTER_NAME}"
+    sleep 10 # Lazy sleep instead of checking for readiness of API server to handle all resources.
+    ;;
+  delete)
+    k3d cluster delete "${CLUSTER_NAME}"
+    shift
+    ;;
+esac
--- a/production/ksonnet/enterprise-logs/test/eval.jsonnet
+++ b/production/ksonnet/enterprise-logs/test/eval.jsonnet
@ -0,0 +1,12 @@
+(import '../main.libsonnet') {
+  _config+:: {
+    boltdb_shipper_shared_store: self.storage_backend,
+    gcs_bucket_name: 'test-gcs-bucket-name',
+    storage_backend: 'gcs',
+
+    namespace: 'test-namespace',
+    commonArgs+: {
+      'cluster-name': 'test-cluster-name',
+    },
+  },
+}
--- a/production/ksonnet/enterprise-logs/test/jsonnetfile.json
+++ b/production/ksonnet/enterprise-logs/test/jsonnetfile.json
@ -0,0 +1,33 @@
+{
+   "dependencies": [
+      {
+         "source": {
+            "git": {
+               "remote": "https://github.com/grafana/loki.git",
+               "subdir": "production/ksonnet/loki"
+            }
+         },
+         "version": "main"
+      },
+      {
+         "source": {
+            "git": {
+               "remote": "https://github.com/grafana/jsonnet-libs",
+               "subdir": "ksonnet-util"
+            }
+         },
+         "version": "master"
+      },
+      {
+         "source": {
+            "git": {
+               "remote": "https://github.com/jsonnet-libs/k8s-libsonnet.git",
+               "subdir": "1.18"
+            }
+         },
+         "version": "main"
+      }
+   ],
+   "legacyImports": true,
+   "version": 1
+}
--- a/production/ksonnet/enterprise-logs/test/jsonnetfile.jsonnet
+++ b/production/ksonnet/enterprise-logs/test/jsonnetfile.jsonnet
@ -0,0 +1,12 @@
+function(k8sVersion)
+  (import '../jsonnetfile.json') + {
+    dependencies+: [{
+      source: {
+        git: {
+          remote: 'https://github.com/jsonnet-libs/k8s-libsonnet.git',
+          subdir: k8sVersion,
+        },
+      },
+      version: 'main',
+    }],
+  }
--- a/production/ksonnet/enterprise-logs/test/jsonnetfile.lock.json
+++ b/production/ksonnet/enterprise-logs/test/jsonnetfile.lock.json
@ -0,0 +1,66 @@
+{
+  "version": 1,
+  "dependencies": [
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "consul"
+        }
+      },
+      "version": "7e3954ca9459cad6508ce3675876dbe4f4b33a8f",
+      "sum": "Po3c1Ic96ngrJCtOazic/7OsLkoILOKZWXWyZWl+od8="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "jaeger-agent-mixin"
+        }
+      },
+      "version": "7e3954ca9459cad6508ce3675876dbe4f4b33a8f",
+      "sum": "DsdBoqgx5kE3zc6fMYnfiGjW2+9Mx2OXFieWm1oFHgY="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "ksonnet-util"
+        }
+      },
+      "version": "7e3954ca9459cad6508ce3675876dbe4f4b33a8f",
+      "sum": "OxgtIWL4hjvG0xkMwUzZ7Yjs52zUhLhaVQpwHCbqf8A="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/jsonnet-libs.git",
+          "subdir": "memcached"
+        }
+      },
+      "version": "7e3954ca9459cad6508ce3675876dbe4f4b33a8f",
+      "sum": "dTOeEux3t9bYSqP2L/uCuLo/wUDpCKH4w+4OD9fePUk="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/loki.git",
+          "subdir": "production/ksonnet/loki"
+        }
+      },
+      "version": "cd79adc8ab0c80c55a0f39b43322839c6ffb05d8",
+      "sum": "ihooLlOemtJlNiFzAPYM06ug4kLKtOMTJ0SWRXWNCuA="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/jsonnet-libs/k8s-libsonnet.git",
+          "subdir": "1.18"
+        }
+      },
+      "version": "91008dbd2ea5734288467d6dcafef7c285c3f7e6",
+      "sum": "x881PM+6ARMsa9OSJcxO6L+4GOBy91clZipjeYbbzpw="
+    }
+  ],
+  "legacyImports": false
+}
--- a/production/ksonnet/enterprise-logs/variables.mk
+++ b/production/ksonnet/enterprise-logs/variables.mk
@ -0,0 +1,2 @@
+## Version of Kubernetes used in testing.
+K8S_VERSION = 1.18