mirror of https://github.com/grafana/loki
Brett Jones
4 years ago
parent
2c0afdfce3
commit
c4e376c233
@ -1,14 +0,0 @@ |
||||
package manifests |
||||
|
||||
const ( |
||||
ContainerImage = "docker.io/grafana/loki:2.1.0" |
||||
|
||||
GossipPort = 7946 |
||||
MetricsPort = 3100 |
||||
GRPCPort = 9095 |
||||
|
||||
ServiceNameDistributorHTTP = "loki-distributor-http" |
||||
ServiceNameDistributorGRPC = "loki-distributor-grpc" |
||||
|
||||
ServiceNameGossipRing = "loki-gossip-ring" |
||||
) |
||||
@ -1,116 +0,0 @@ |
||||
package distributor |
||||
|
||||
import ( |
||||
"path" |
||||
|
||||
"github.com/ViaQ/logerr/kverrors" |
||||
"github.com/openshift/loki-operator/internal/lokidto" |
||||
"github.com/openshift/loki-operator/internal/manifests" |
||||
"github.com/openshift/loki-operator/internal/manifests/gossip" |
||||
"gopkg.in/yaml.v2" |
||||
apps "k8s.io/api/apps/v1" |
||||
core "k8s.io/api/core/v1" |
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
||||
) |
||||
|
||||
func newConfigMap(namespace, name string) (*core.ConfigMap, error) { |
||||
config := newLokiConfig(namespace) |
||||
c, err := yaml.Marshal(config) |
||||
if err != nil { |
||||
return nil, kverrors.Wrap(err, "failed to marshal config", "config", config) |
||||
} |
||||
|
||||
return &core.ConfigMap{ |
||||
TypeMeta: metav1.TypeMeta{ |
||||
Kind: "ConfigMap", |
||||
APIVersion: apps.SchemeGroupVersion.String(), |
||||
}, |
||||
ObjectMeta: metav1.ObjectMeta{ |
||||
Name: name, |
||||
Namespace: namespace, |
||||
Labels: commonLabels, |
||||
}, |
||||
Data: map[string]string{ |
||||
"/etc/loki/config/config.yaml": string(c), |
||||
}, |
||||
}, nil |
||||
} |
||||
|
||||
func newLokiConfig(namespace string) lokidto.Config { |
||||
return lokidto.Config{ |
||||
AuthEnabled: true, |
||||
ChunkStoreConfig: &lokidto.ChunkStoreConfig{ |
||||
ChunkCacheConfig: &lokidto.CacheConfig{ |
||||
FIFOCache: &lokidto.FIFOCache{ |
||||
MaxSizeBytes: "1Gi", |
||||
}, |
||||
}, |
||||
}, |
||||
Distributor: &lokidto.Distributor{ |
||||
Ring: lokidto.Ring{ |
||||
Kvstore: &lokidto.Kvstore{ |
||||
Store: "memberlist", |
||||
}, |
||||
}, |
||||
}, |
||||
IngesterClient: &lokidto.IngesterClient{ |
||||
GrpcClientConfig: &lokidto.GrpcClientConfig{ |
||||
MaxRecvMsgSize: 67108864, |
||||
}, |
||||
RemoteTimeout: "1s", |
||||
}, |
||||
LimitsConfig: &lokidto.LimitsConfig{ |
||||
IngestionBurstSizeMb: 20, |
||||
IngestionRateMb: 10, |
||||
IngestionRateStrategy: "global", |
||||
MaxCacheFreshnessPerQuery: "10m", |
||||
MaxGlobalStreamsPerUser: 25000, |
||||
MaxQueryLength: "12000h", |
||||
MaxQueryParallelism: 32, |
||||
RejectOldSamples: true, |
||||
RejectOldSamplesMaxAge: "24h", |
||||
}, |
||||
Memberlist: &lokidto.Memberlist{ |
||||
AbortIfClusterJoinFails: false, |
||||
BindPort: manifests.GossipPort, |
||||
JoinMembers: []string{gossip.RingServiceURL(namespace)}, |
||||
MaxJoinBackoff: "1m", |
||||
MaxJoinRetries: 10, |
||||
MinJoinBackoff: "1s", |
||||
}, |
||||
SchemaConfig: &lokidto.SchemaConfig{ |
||||
Configs: []lokidto.Configs{ |
||||
{ |
||||
From: "2020-10-01", |
||||
Index: &lokidto.Index{ |
||||
Period: "24h", |
||||
Prefix: "index_", |
||||
}, |
||||
ObjectStore: "filesystem", |
||||
Schema: "v11", |
||||
Store: "boltdb", |
||||
}, |
||||
}, |
||||
}, |
||||
Server: &lokidto.Server{ |
||||
GracefulShutdownTimeout: "5s", |
||||
GrpcServerMaxConcurrentStreams: 1000, |
||||
GrpcServerMaxRecvMsgSize: 104857600, |
||||
GrpcServerMaxSendMsgSize: 104857600, |
||||
HTTPListenPort: 3100, |
||||
HTTPServerIdleTimeout: "120s", |
||||
HTTPServerWriteTimeout: "1m", |
||||
}, |
||||
StorageConfig: &lokidto.StorageConfig{ |
||||
BoltDB: &lokidto.BoltDB{ |
||||
Directory: path.Join(dataPath, "/index"), |
||||
}, |
||||
Filesystem: &lokidto.Filesystem{ |
||||
Directory: path.Join(dataPath, "/chunks"), |
||||
}, |
||||
}, |
||||
Tracing: &lokidto.Tracing{ |
||||
Enabled: false, |
||||
}, |
||||
} |
||||
} |
||||
@ -1,178 +0,0 @@ |
||||
package distributor |
||||
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"github.com/openshift/loki-operator/internal/manifests" |
||||
"github.com/openshift/loki-operator/internal/manifests/gossip" |
||||
apps "k8s.io/api/apps/v1" |
||||
core "k8s.io/api/core/v1" |
||||
"k8s.io/apimachinery/pkg/api/resource" |
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
||||
"k8s.io/apimachinery/pkg/labels" |
||||
"k8s.io/apimachinery/pkg/util/intstr" |
||||
"k8s.io/utils/pointer" |
||||
) |
||||
|
||||
const ( |
||||
configVolume = "config" |
||||
dataVolume = "storage" |
||||
dataPath = "/data/loki" |
||||
) |
||||
|
||||
var ( |
||||
commonLabels = map[string]string{ |
||||
"app.kubernetes.io/name": "loki", |
||||
"app.kubernetes.io/provider": "openshift", |
||||
"loki.grafana.com/component": "distributor", |
||||
} |
||||
) |
||||
|
||||
// New creates a new set of distributor objects to be deployed
|
||||
func New(namespace, name string, replicas int) (*Distributor, error) { |
||||
cm, err := newConfigMap(namespace, fmt.Sprintf("%s-distributor", name)) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return &Distributor{ |
||||
deployment: newDeployment(namespace, fmt.Sprintf("%s-distributor", name), replicas), |
||||
configMap: cm, |
||||
services: services(namespace), |
||||
}, nil |
||||
} |
||||
|
||||
// Distributor is a collection of manifests for deploying a working Loki Distributor.
|
||||
type Distributor struct { |
||||
deployment *apps.Deployment |
||||
configMap *core.ConfigMap |
||||
services []core.Service |
||||
} |
||||
|
||||
func (d *Distributor) Deployment() *apps.Deployment { |
||||
return d.deployment |
||||
} |
||||
|
||||
func newDeployment(namespace, name string, replicas int) *apps.Deployment { |
||||
podSpec := core.PodSpec{ |
||||
Volumes: []core.Volume{ |
||||
{ |
||||
Name: configVolume, |
||||
VolumeSource: core.VolumeSource{ |
||||
ConfigMap: &core.ConfigMapVolumeSource{ |
||||
LocalObjectReference: core.LocalObjectReference{ |
||||
Name: configVolume, |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
Name: dataVolume, |
||||
VolumeSource: core.VolumeSource{ |
||||
EmptyDir: &core.EmptyDirVolumeSource{}, |
||||
}, |
||||
}, |
||||
}, |
||||
Containers: []core.Container{ |
||||
{ |
||||
Image: manifests.ContainerImage, |
||||
Name: "loki-distributor", |
||||
Args: []string{ |
||||
"-config.file=/etc/loki/config/config.yaml", |
||||
"-distributor.replication-factor=1", |
||||
"-limits.per-user-override-config=/etc/loki/config/overrides.yaml", |
||||
"-log.level=info", |
||||
"-querier.worker-parallelism=1", |
||||
"-target=distributor", |
||||
}, |
||||
ReadinessProbe: &core.Probe{ |
||||
Handler: core.Handler{ |
||||
HTTPGet: &core.HTTPGetAction{ |
||||
Path: "/ready", |
||||
Port: intstr.FromInt(3100), |
||||
Scheme: core.URISchemeHTTP, |
||||
}, |
||||
}, |
||||
InitialDelaySeconds: 15, |
||||
TimeoutSeconds: 1, |
||||
}, |
||||
LivenessProbe: &core.Probe{ |
||||
Handler: core.Handler{ |
||||
HTTPGet: &core.HTTPGetAction{ |
||||
Path: "/metrics", |
||||
Port: intstr.FromInt(3100), |
||||
Scheme: core.URISchemeHTTP, |
||||
}, |
||||
}, |
||||
TimeoutSeconds: 2, |
||||
PeriodSeconds: 30, |
||||
FailureThreshold: 10, |
||||
}, |
||||
Ports: []core.ContainerPort{ |
||||
{ |
||||
Name: "metrics", |
||||
ContainerPort: manifests.MetricsPort, |
||||
}, |
||||
{ |
||||
Name: "grpc", |
||||
ContainerPort: manifests.GRPCPort, |
||||
}, |
||||
{ |
||||
Name: "gossip-ring", |
||||
ContainerPort: manifests.GossipPort, |
||||
}, |
||||
}, |
||||
Resources: core.ResourceRequirements{ |
||||
Limits: core.ResourceList{ |
||||
core.ResourceMemory: resource.MustParse("1Gi"), |
||||
core.ResourceCPU: resource.MustParse("1000m"), |
||||
}, |
||||
Requests: core.ResourceList{ |
||||
core.ResourceMemory: resource.MustParse("50m"), |
||||
core.ResourceCPU: resource.MustParse("50m"), |
||||
}, |
||||
}, |
||||
VolumeMounts: []core.VolumeMount{ |
||||
{ |
||||
Name: configVolume, |
||||
ReadOnly: false, |
||||
MountPath: "/etc/loki/config", |
||||
}, |
||||
{ |
||||
Name: dataVolume, |
||||
ReadOnly: false, |
||||
MountPath: dataPath, |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
return &apps.Deployment{ |
||||
TypeMeta: metav1.TypeMeta{ |
||||
Kind: "Deployment", |
||||
APIVersion: apps.SchemeGroupVersion.String(), |
||||
}, |
||||
ObjectMeta: metav1.ObjectMeta{ |
||||
Name: name, |
||||
Namespace: namespace, |
||||
Labels: commonLabels, |
||||
}, |
||||
Spec: apps.DeploymentSpec{ |
||||
Replicas: pointer.Int32Ptr(int32(replicas)), |
||||
Selector: &metav1.LabelSelector{ |
||||
MatchLabels: labels.Merge(commonLabels, gossip.Labels()), |
||||
}, |
||||
Template: core.PodTemplateSpec{ |
||||
ObjectMeta: metav1.ObjectMeta{ |
||||
Name: name, |
||||
Labels: labels.Merge(commonLabels, gossip.Labels()), |
||||
}, |
||||
Spec: podSpec, |
||||
}, |
||||
Strategy: apps.DeploymentStrategy{ |
||||
Type: apps.RollingUpdateDeploymentStrategyType, |
||||
}, |
||||
}, |
||||
} |
||||
} |
||||
@ -1,20 +0,0 @@ |
||||
package distributor_test |
||||
|
||||
import ( |
||||
"testing" |
||||
|
||||
"github.com/openshift/loki-operator/internal/manifests/distributor" |
||||
"github.com/stretchr/testify/require" |
||||
"sigs.k8s.io/yaml" |
||||
) |
||||
|
||||
func TestWorks(t *testing.T) { |
||||
ns := "mynamespace" |
||||
name := "myname" |
||||
d, err := distributor.New(ns, name, 1) |
||||
require.NoError(t, err) |
||||
|
||||
deploy, err := yaml.Marshal(d.Deployment()) |
||||
require.NoError(t, err) |
||||
_ = deploy |
||||
} |
||||
@ -1,57 +0,0 @@ |
||||
package distributor |
||||
|
||||
import ( |
||||
"github.com/openshift/loki-operator/internal/manifests" |
||||
apps "k8s.io/api/apps/v1" |
||||
core "k8s.io/api/core/v1" |
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
||||
) |
||||
|
||||
func services(namespace string) []core.Service { |
||||
return []core.Service{ |
||||
{ |
||||
TypeMeta: metav1.TypeMeta{ |
||||
Kind: "Service", |
||||
APIVersion: apps.SchemeGroupVersion.String(), |
||||
}, |
||||
ObjectMeta: metav1.ObjectMeta{ |
||||
Name: manifests.ServiceNameDistributorHTTP, |
||||
Namespace: namespace, |
||||
Labels: commonLabels, |
||||
}, |
||||
Spec: core.ServiceSpec{ |
||||
ClusterIP: "None", |
||||
Ports: []core.ServicePort{ |
||||
{ |
||||
Name: "http", |
||||
Port: manifests.MetricsPort, |
||||
Protocol: "TCP", |
||||
}, |
||||
}, |
||||
Selector: commonLabels, |
||||
}, |
||||
Status: core.ServiceStatus{}, |
||||
}, |
||||
{ |
||||
TypeMeta: metav1.TypeMeta{ |
||||
Kind: "Service", |
||||
APIVersion: apps.SchemeGroupVersion.String(), |
||||
}, |
||||
ObjectMeta: metav1.ObjectMeta{ |
||||
Name: manifests.ServiceNameDistributorGRPC, |
||||
Namespace: namespace, |
||||
Labels: commonLabels, |
||||
}, |
||||
Spec: core.ServiceSpec{ |
||||
ClusterIP: "None", |
||||
Ports: []core.ServicePort{ |
||||
{ |
||||
Name: "grpc", |
||||
Port: manifests.GRPCPort, |
||||
}, |
||||
}, |
||||
Selector: commonLabels, |
||||
}, |
||||
}, |
||||
} |
||||
} |
||||
@ -1 +0,0 @@ |
||||
package manifests |
||||
@ -1,17 +0,0 @@ |
||||
package gossip |
||||
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"github.com/openshift/loki-operator/internal/manifests" |
||||
) |
||||
|
||||
func RingServiceURL(namespace string) string { |
||||
return fmt.Sprintf(`%s.%s.svc.cluster.local:%d`, manifests.ServiceNameGossipRing, namespace, manifests.GossipPort) |
||||
} |
||||
|
||||
func Labels() map[string]string { |
||||
return map[string]string{ |
||||
"loki.grafana.com/gossip": "true", |
||||
} |
||||
} |
||||
@ -1,148 +0,0 @@ |
||||
package ingester |
||||
|
||||
import ( |
||||
"text/template" |
||||
|
||||
"sigs.k8s.io/kustomize/api/filesys" |
||||
"sigs.k8s.io/kustomize/api/krusty" |
||||
) |
||||
|
||||
func Deployment() { |
||||
k := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) |
||||
fs := filesys.MakeEmptyDirInMemory() |
||||
fs.AddFile() |
||||
k.Run(fs, "") |
||||
} |
||||
|
||||
|
||||
var deployment = template.Must(template.New("ingester-deployment.yaml").Parse(deploymentTemplate)) |
||||
|
||||
const deploymentTemplate = ` |
||||
---
|
||||
apiVersion: apps/v1 |
||||
kind: Deployment |
||||
metadata: |
||||
labels: |
||||
app.kubernetes.io/component: ingester |
||||
app.kubernetes.io/name: loki |
||||
name: loki-ingester |
||||
spec: |
||||
replicas: ${{LOKI_INGESTER_REPLICAS}} |
||||
selector: |
||||
matchLabels: |
||||
app.kubernetes.io/component: ingester |
||||
app.kubernetes.io/instance: observatorium |
||||
app.kubernetes.io/name: loki |
||||
app.kubernetes.io/part-of: observatorium |
||||
loki.grafana.com/gossip: "true" |
||||
serviceName: observatorium-loki-ingester-grpc |
||||
template: |
||||
metadata: |
||||
labels: |
||||
app.kubernetes.io/component: ingester |
||||
app.kubernetes.io/instance: observatorium |
||||
app.kubernetes.io/name: loki |
||||
app.kubernetes.io/part-of: observatorium |
||||
app.kubernetes.io/tracing: jaeger-agent |
||||
loki.grafana.com/gossip: "true" |
||||
spec: |
||||
containers: |
||||
- args: |
||||
- -target=ingester |
||||
- -config.file=/etc/loki/config/config.yaml |
||||
- -limits.per-user-override-config=/etc/loki/config/overrides.yaml |
||||
- -log.level=error |
||||
- -distributor.replication-factor=${LOKI_REPLICATION_FACTOR} |
||||
- -querier.worker-parallelism=${LOKI_QUERY_PARALLELISM} |
||||
image: ${LOKI_IMAGE}:${LOKI_IMAGE_TAG} |
||||
livenessProbe: |
||||
failureThreshold: 10 |
||||
httpGet: |
||||
path: /metrics |
||||
port: 3100 |
||||
scheme: HTTP |
||||
periodSeconds: 30 |
||||
name: observatorium-loki-ingester |
||||
ports: |
||||
- containerPort: 3100 |
||||
name: metrics |
||||
- containerPort: 9095 |
||||
name: grpc |
||||
- containerPort: 7946 |
||||
name: gossip-ring |
||||
readinessProbe: |
||||
httpGet: |
||||
path: /ready |
||||
port: 3100 |
||||
scheme: HTTP |
||||
initialDelaySeconds: 15 |
||||
timeoutSeconds: 1 |
||||
resources: |
||||
limits: |
||||
cpu: ${LOKI_INGESTER_CPU_LIMITS} |
||||
memory: ${LOKI_INGESTER_MEMORY_LIMITS} |
||||
requests: |
||||
cpu: ${LOKI_INGESTER_CPU_REQUESTS} |
||||
memory: ${LOKI_INGESTER_MEMORY_REQUESTS} |
||||
volumeMounts: |
||||
- mountPath: /etc/loki/config/ |
||||
name: config |
||||
readOnly: false |
||||
- mountPath: /data |
||||
name: storage |
||||
readOnly: false |
||||
- args: |
||||
- --reporter.grpc.host-port=dns:///jaeger-collector-headless.${JAEGER_COLLECTOR_NAMESPACE}.svc:14250
|
||||
- --reporter.type=grpc |
||||
- --jaeger.tags=pod.namespace=$(NAMESPACE),pod.name=$(POD) |
||||
env: |
||||
- name: NAMESPACE |
||||
valueFrom: |
||||
fieldRef: |
||||
fieldPath: metadata.namespace |
||||
- name: POD |
||||
valueFrom: |
||||
fieldRef: |
||||
fieldPath: metadata.name |
||||
image: ${JAEGER_AGENT_IMAGE}:${JAEGER_AGENT_IMAGE_TAG} |
||||
livenessProbe: |
||||
failureThreshold: 5 |
||||
httpGet: |
||||
path: / |
||||
port: 14271 |
||||
scheme: HTTP |
||||
name: jaeger-agent |
||||
ports: |
||||
- containerPort: 5778 |
||||
name: configs |
||||
- containerPort: 6831 |
||||
name: jaeger-thrift |
||||
- containerPort: 14271 |
||||
name: metrics |
||||
resources: |
||||
limits: |
||||
cpu: 128m |
||||
memory: 128Mi |
||||
requests: |
||||
cpu: 32m |
||||
memory: 64Mi |
||||
serviceAccountName: ${SERVICE_ACCOUNT_NAME} |
||||
volumes: |
||||
- configMap: |
||||
name: observatorium-loki |
||||
name: config |
||||
volumeClaimTemplates: |
||||
- metadata: |
||||
labels: |
||||
app.kubernetes.io/instance: observatorium |
||||
app.kubernetes.io/name: loki |
||||
app.kubernetes.io/part-of: observatorium |
||||
name: storage |
||||
spec: |
||||
accessModes: |
||||
- ReadWriteOnce |
||||
resources: |
||||
requests: |
||||
storage: ${LOKI_PVC_REQUEST} |
||||
storageClassName: ${STORAGE_CLASS} |
||||
` |
||||
@ -1,13 +0,0 @@ |
||||
package tshirt |
||||
|
||||
|
||||
// tshirt sizes to be further determined by the individual components
|
||||
const ( |
||||
XXS = "xxs" |
||||
XS = "s" |
||||
S = "s" |
||||
M = "m" |
||||
L = "l" |
||||
XL = "xl" |
||||
XXL = "xxl" |
||||
) |
||||
Loading…
Reference in new issue