mirror of https://github.com/grafana/loki
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
276 lines
11 KiB
276 lines
11 KiB
package scrapeconfig
|
|
|
|
import (
|
|
"fmt"
|
|
"reflect"
|
|
"time"
|
|
|
|
"github.com/prometheus/common/model"
|
|
"github.com/prometheus/prometheus/discovery"
|
|
"github.com/prometheus/prometheus/discovery/aws"
|
|
"github.com/prometheus/prometheus/discovery/azure"
|
|
"github.com/prometheus/prometheus/discovery/consul"
|
|
"github.com/prometheus/prometheus/discovery/digitalocean"
|
|
"github.com/prometheus/prometheus/discovery/dns"
|
|
"github.com/prometheus/prometheus/discovery/file"
|
|
"github.com/prometheus/prometheus/discovery/gce"
|
|
"github.com/prometheus/prometheus/discovery/kubernetes"
|
|
"github.com/prometheus/prometheus/discovery/marathon"
|
|
"github.com/prometheus/prometheus/discovery/moby"
|
|
"github.com/prometheus/prometheus/discovery/openstack"
|
|
"github.com/prometheus/prometheus/discovery/triton"
|
|
"github.com/prometheus/prometheus/discovery/zookeeper"
|
|
"github.com/prometheus/prometheus/pkg/relabel"
|
|
"github.com/weaveworks/common/server"
|
|
|
|
"github.com/grafana/loki/clients/pkg/logentry/stages"
|
|
"github.com/grafana/loki/clients/pkg/promtail/discovery/consulagent"
|
|
)
|
|
|
|
// Config describes a job to scrape.
|
|
type Config struct {
|
|
JobName string `yaml:"job_name,omitempty"`
|
|
PipelineStages stages.PipelineStages `yaml:"pipeline_stages,omitempty"`
|
|
JournalConfig *JournalTargetConfig `yaml:"journal,omitempty"`
|
|
SyslogConfig *SyslogTargetConfig `yaml:"syslog,omitempty"`
|
|
GcplogConfig *GcplogTargetConfig `yaml:"gcplog,omitempty"`
|
|
PushConfig *PushTargetConfig `yaml:"loki_push_api,omitempty"`
|
|
WindowsConfig *WindowsEventsTargetConfig `yaml:"windows_events,omitempty"`
|
|
RelabelConfigs []*relabel.Config `yaml:"relabel_configs,omitempty"`
|
|
ServiceDiscoveryConfig ServiceDiscoveryConfig `yaml:",inline"`
|
|
}
|
|
|
|
type ServiceDiscoveryConfig struct {
|
|
// List of labeled target groups for this job.
|
|
StaticConfigs discovery.StaticConfig `yaml:"static_configs"`
|
|
// List of DNS service discovery configurations.
|
|
DNSSDConfigs []*dns.SDConfig `yaml:"dns_sd_configs,omitempty"`
|
|
// List of file service discovery configurations.
|
|
FileSDConfigs []*file.SDConfig `yaml:"file_sd_configs,omitempty"`
|
|
// List of Consul service discovery configurations.
|
|
ConsulSDConfigs []*consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
|
|
// List of Consul agent service discovery configurations.
|
|
ConsulAgentSDConfigs []*consulagent.SDConfig `yaml:"consulagent_sd_configs,omitempty"`
|
|
// List of DigitalOcean service discovery configurations.
|
|
DigitalOceanSDConfigs []*digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
|
|
// List of Docker Swarm service discovery configurations.
|
|
DockerSwarmSDConfigs []*moby.DockerSwarmSDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
|
|
// List of Serverset service discovery configurations.
|
|
ServersetSDConfigs []*zookeeper.ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"`
|
|
// NerveSDConfigs is a list of Nerve service discovery configurations.
|
|
NerveSDConfigs []*zookeeper.NerveSDConfig `yaml:"nerve_sd_configs,omitempty"`
|
|
// MarathonSDConfigs is a list of Marathon service discovery configurations.
|
|
MarathonSDConfigs []*marathon.SDConfig `yaml:"marathon_sd_configs,omitempty"`
|
|
// List of Kubernetes service discovery configurations.
|
|
KubernetesSDConfigs []*kubernetes.SDConfig `yaml:"kubernetes_sd_configs,omitempty"`
|
|
// List of GCE service discovery configurations.
|
|
GCESDConfigs []*gce.SDConfig `yaml:"gce_sd_configs,omitempty"`
|
|
// List of EC2 service discovery configurations.
|
|
EC2SDConfigs []*aws.EC2SDConfig `yaml:"ec2_sd_configs,omitempty"`
|
|
// List of OpenStack service discovery configurations.
|
|
OpenstackSDConfigs []*openstack.SDConfig `yaml:"openstack_sd_configs,omitempty"`
|
|
// List of Azure service discovery configurations.
|
|
AzureSDConfigs []*azure.SDConfig `yaml:"azure_sd_configs,omitempty"`
|
|
// List of Triton service discovery configurations.
|
|
TritonSDConfigs []*triton.SDConfig `yaml:"triton_sd_configs,omitempty"`
|
|
}
|
|
|
|
func (cfg ServiceDiscoveryConfig) Configs() (res discovery.Configs) {
|
|
if x := cfg.StaticConfigs; len(x) > 0 {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.DNSSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.FileSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.ConsulSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.ConsulAgentSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.DigitalOceanSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.DockerSwarmSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.ServersetSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.NerveSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.MarathonSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.KubernetesSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.GCESDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.EC2SDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.OpenstackSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.AzureSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
for _, x := range cfg.TritonSDConfigs {
|
|
res = append(res, x)
|
|
}
|
|
return res
|
|
}
|
|
|
|
// JournalTargetConfig describes systemd journal records to scrape.
|
|
type JournalTargetConfig struct {
|
|
// MaxAge determines the oldest relative time from process start that will
|
|
// be read and sent to Loki. Values like 14h means no entry older than
|
|
// 14h will be read. If unspecified, defaults to 7h.
|
|
//
|
|
// A relative time specified here takes precedence over the saved position;
|
|
// if the cursor is older than the MaxAge value, it will not be used.
|
|
MaxAge string `yaml:"max_age"`
|
|
|
|
// JSON forces the output message of entries read from the journal to be
|
|
// JSON. The message will contain all original fields from the source
|
|
// journal entry.
|
|
JSON bool `yaml:"json"`
|
|
|
|
// Labels optionally holds labels to associate with each record coming out
|
|
// of the journal.
|
|
Labels model.LabelSet `yaml:"labels"`
|
|
|
|
// Path to a directory to read journal entries from. Defaults to system path
|
|
// if empty.
|
|
Path string `yaml:"path"`
|
|
}
|
|
|
|
// SyslogTargetConfig describes a scrape config that listens for log lines over syslog.
|
|
type SyslogTargetConfig struct {
|
|
// ListenAddress is the address to listen on for syslog messages.
|
|
ListenAddress string `yaml:"listen_address"`
|
|
|
|
// IdleTimeout is the idle timeout for tcp connections.
|
|
IdleTimeout time.Duration `yaml:"idle_timeout"`
|
|
|
|
// LabelStructuredData sets if the structured data part of a syslog message
|
|
// is translated to a label.
|
|
// [example@99999 test="yes"] => {__syslog_message_sd_example_99999_test="yes"}
|
|
LabelStructuredData bool `yaml:"label_structured_data"`
|
|
|
|
// Labels optionally holds labels to associate with each record read from syslog.
|
|
Labels model.LabelSet `yaml:"labels"`
|
|
|
|
// UseIncomingTimestamp sets the timestamp to the incoming syslog messages
|
|
// timestamp if it's set.
|
|
UseIncomingTimestamp bool `yaml:"use_incoming_timestamp"`
|
|
|
|
// MaxMessageLength sets the maximum limit to the length of syslog messages
|
|
MaxMessageLength int `yaml:"max_message_length"`
|
|
}
|
|
|
|
// WindowsEventsTargetConfig describes a scrape config that listen for windows event logs.
|
|
type WindowsEventsTargetConfig struct {
|
|
|
|
// LCID (Locale ID) for event rendering
|
|
// - 1033 to force English language
|
|
// - 0 to use default Windows locale
|
|
Locale uint32 `yaml:"locale"`
|
|
|
|
// Name of eventlog, used only if xpath_query is empty
|
|
// Example: "Application"
|
|
EventlogName string `yaml:"eventlog_name"`
|
|
|
|
// xpath_query can be in defined short form like "Event/System[EventID=999]"
|
|
// or you can form a XML Query. Refer to the Consuming Events article:
|
|
// https://docs.microsoft.com/en-us/windows/win32/wes/consuming-events
|
|
// XML query is the recommended form, because it is most flexible
|
|
// You can create or debug XML Query by creating Custom View in Windows Event Viewer
|
|
// and then copying resulting XML here
|
|
Query string `yaml:"xpath_query"`
|
|
|
|
// UseIncomingTimestamp sets the timestamp to the incoming windows messages
|
|
// timestamp if it's set.
|
|
UseIncomingTimestamp bool `yaml:"use_incoming_timestamp"`
|
|
|
|
// BookmarkPath sets the bookmark location on the filesystem.
|
|
// The bookmark contains the current position of the target in XML.
|
|
// When restarting or rollingout promtail, the target will continue to scrape events where it left off based on the bookmark position.
|
|
// The position is updated after each entry processed.
|
|
BookmarkPath string `yaml:"bookmark_path"`
|
|
|
|
// PollInterval is the interval at which we're looking if new events are available. By default the target will check every 3seconds.
|
|
PollInterval time.Duration `yaml:"poll_interval"`
|
|
|
|
// ExcludeEventData allows to exclude the xml event data.
|
|
ExcludeEventData bool `yaml:"exclude_event_data"`
|
|
|
|
// ExcludeUserData allows to exclude the user data of each windows event.
|
|
ExcludeUserData bool `yaml:"exclude_user_data"`
|
|
|
|
// Labels optionally holds labels to associate with each log line.
|
|
Labels model.LabelSet `yaml:"labels"`
|
|
}
|
|
|
|
// GcplogTargetConfig describes a scrape config to pull logs from any pubsub topic.
|
|
type GcplogTargetConfig struct {
|
|
// ProjectID is the Cloud project id
|
|
ProjectID string `yaml:"project_id"`
|
|
|
|
// Subscription is the scription name we use to pull logs from a pubsub topic.
|
|
Subscription string `yaml:"subscription"`
|
|
|
|
// Labels are the additional labels to be added to log entry while pushing it to Loki server.
|
|
Labels model.LabelSet `yaml:"labels"`
|
|
|
|
// UseIncomingTimestamp represents whether to keep the timestamp same as actual log entry coming in or replace it with
|
|
// current timestamp at the time of processing.
|
|
// Its default value(`false`) denotes, replace it with current timestamp at the time of processing.
|
|
UseIncomingTimestamp bool `yaml:"use_incoming_timestamp"`
|
|
}
|
|
|
|
// PushTargetConfig describes a scrape config that listens for Loki push messages.
|
|
type PushTargetConfig struct {
|
|
// Server is the weaveworks server config for listening connections
|
|
Server server.Config `yaml:"server"`
|
|
|
|
// Labels optionally holds labels to associate with each record received on the push api.
|
|
Labels model.LabelSet `yaml:"labels"`
|
|
|
|
// If promtail should maintain the incoming log timestamp or replace it with the current time.
|
|
KeepTimestamp bool `yaml:"use_incoming_timestamp"`
|
|
}
|
|
|
|
// DefaultScrapeConfig is the default Config.
|
|
var DefaultScrapeConfig = Config{
|
|
PipelineStages: stages.PipelineStages{},
|
|
}
|
|
|
|
// HasServiceDiscoveryConfig checks to see if the service discovery used for
|
|
// file targets is non-zero.
|
|
func (c *Config) HasServiceDiscoveryConfig() bool {
|
|
return !reflect.DeepEqual(c.ServiceDiscoveryConfig, ServiceDiscoveryConfig{})
|
|
}
|
|
|
|
// UnmarshalYAML implements the yaml.Unmarshaler interface.
|
|
func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
|
|
|
*c = DefaultScrapeConfig
|
|
|
|
type plain Config
|
|
if err := unmarshal((*plain)(c)); err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(c.JobName) == 0 {
|
|
return fmt.Errorf("job_name is empty")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|