diff --git a/MAINTAINERS.md b/MAINTAINERS.md index e3312b3129..8d10a8fbca 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -11,6 +11,7 @@ Maintainers for specific parts of the codebase: * `discovery` * `azure`: Jan-Otto Kröpke ( / @jkroepke) * `k8s`: Frederic Branczyk ( / @brancz) + * `stackit`: Jan-Otto Kröpke ( / @jkroepke) * `documentation` * `prometheus-mixin`: Matthias Loibl ( / @metalmatze) * `model/histogram` and other code related to native histograms: Björn Rabenstein ( / @beorn7), diff --git a/config/config_test.go b/config/config_test.go index ef4f1203f5..3e931e7700 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -50,6 +50,7 @@ import ( "github.com/prometheus/prometheus/discovery/ovhcloud" "github.com/prometheus/prometheus/discovery/puppetdb" "github.com/prometheus/prometheus/discovery/scaleway" + "github.com/prometheus/prometheus/discovery/stackit" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/discovery/triton" "github.com/prometheus/prometheus/discovery/uyuni" @@ -1473,6 +1474,45 @@ var expectedConf = &Config{ }, }, }, + { + JobName: "stackit-servers", + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + EnableCompression: true, + BodySizeLimit: globBodySizeLimit, + SampleLimit: globSampleLimit, + TargetLimit: globTargetLimit, + LabelLimit: globLabelLimit, + LabelNameLengthLimit: globLabelNameLengthLimit, + LabelValueLengthLimit: globLabelValueLengthLimit, + ScrapeProtocols: DefaultGlobalConfig.ScrapeProtocols, + ScrapeFailureLogFile: globScrapeFailureLogFile, + MetricNameValidationScheme: UTF8ValidationConfig, + MetricNameEscapingScheme: model.AllowUTF8, + AlwaysScrapeClassicHistograms: boolPtr(false), + ConvertClassicHistogramsToNHCB: boolPtr(false), + + MetricsPath: DefaultScrapeConfig.MetricsPath, + Scheme: DefaultScrapeConfig.Scheme, + HTTPClientConfig: config.DefaultHTTPClientConfig, + ServiceDiscoveryConfigs: discovery.Configs{ + &stackit.SDConfig{ + Project: "11111111-1111-1111-1111-111111111111", + Region: "eu01", + HTTPClientConfig: config.HTTPClientConfig{ + Authorization: &config.Authorization{ + Type: "Bearer", + Credentials: "abcdef", + }, + FollowRedirects: true, + EnableHTTP2: true, + }, + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + }, + }, + }, { JobName: "uyuni", @@ -1922,7 +1962,7 @@ func TestElideSecrets(t *testing.T) { yamlConfig := string(config) matches := secretRe.FindAllStringIndex(yamlConfig, -1) - require.Len(t, matches, 24, "wrong number of secret matches found") + require.Len(t, matches, 25, "wrong number of secret matches found") require.NotContains(t, yamlConfig, "mysecret", "yaml marshal reveals authentication credentials.") } @@ -2429,6 +2469,10 @@ var expectedErrors = []struct { filename: "scrape_config_utf8_conflicting.bad.yml", errMsg: `utf8 metric names requested but validation scheme is not set to UTF8`, }, + { + filename: "stackit_endpoint.bad.yml", + errMsg: "invalid endpoint", + }, } func TestBadConfigs(t *testing.T) { diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index 2501652d5b..cbe80404bf 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -417,6 +417,12 @@ scrape_configs: - authorization: credentials: abcdef + - job_name: stackit-servers + stackit_sd_configs: + - project: 11111111-1111-1111-1111-111111111111 + authorization: + credentials: abcdef + - job_name: uyuni uyuni_sd_configs: - server: https://localhost:1234 diff --git a/config/testdata/stackit_endpoint.bad.yml b/config/testdata/stackit_endpoint.bad.yml new file mode 100644 index 0000000000..ecb0fefe9e --- /dev/null +++ b/config/testdata/stackit_endpoint.bad.yml @@ -0,0 +1,4 @@ +scrape_configs: + - job_name: stackit + stackit_sd_configs: + - endpoint: "://invalid" diff --git a/discovery/install/install.go b/discovery/install/install.go index f090076b7f..9c397f9d36 100644 --- a/discovery/install/install.go +++ b/discovery/install/install.go @@ -36,6 +36,7 @@ import ( _ "github.com/prometheus/prometheus/discovery/ovhcloud" // register ovhcloud _ "github.com/prometheus/prometheus/discovery/puppetdb" // register puppetdb _ "github.com/prometheus/prometheus/discovery/scaleway" // register scaleway + _ "github.com/prometheus/prometheus/discovery/stackit" // register stackit _ "github.com/prometheus/prometheus/discovery/triton" // register triton _ "github.com/prometheus/prometheus/discovery/uyuni" // register uyuni _ "github.com/prometheus/prometheus/discovery/vultr" // register vultr diff --git a/discovery/stackit/metrics.go b/discovery/stackit/metrics.go new file mode 100644 index 0000000000..4143b144b7 --- /dev/null +++ b/discovery/stackit/metrics.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +import ( + "github.com/prometheus/prometheus/discovery" +) + +var _ discovery.DiscovererMetrics = (*stackitMetrics)(nil) + +type stackitMetrics struct { + refreshMetrics discovery.RefreshMetricsInstantiator +} + +// Register implements discovery.DiscovererMetrics. +func (m *stackitMetrics) Register() error { + return nil +} + +// Unregister implements discovery.DiscovererMetrics. +func (m *stackitMetrics) Unregister() {} diff --git a/discovery/stackit/mock_test.go b/discovery/stackit/mock_test.go new file mode 100644 index 0000000000..59641ce2bc --- /dev/null +++ b/discovery/stackit/mock_test.go @@ -0,0 +1,162 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +// SDMock is the interface for the STACKIT IAAS API mock. +type SDMock struct { + t *testing.T + Server *httptest.Server + Mux *http.ServeMux +} + +// NewSDMock returns a new SDMock. +func NewSDMock(t *testing.T) *SDMock { + return &SDMock{ + t: t, + } +} + +// Endpoint returns the URI to the mock server. +func (m *SDMock) Endpoint() string { + return m.Server.URL + "/" +} + +// Setup creates the mock server. +func (m *SDMock) Setup() { + m.Mux = http.NewServeMux() + m.Server = httptest.NewServer(m.Mux) + m.t.Cleanup(m.Server.Close) +} + +// ShutdownServer creates the mock server. +func (m *SDMock) ShutdownServer() { + m.Server.Close() +} + +const ( + testToken = "LRK9DAWQ1ZAEFSrCNEEzLCUwhYX1U3g7wMg4dTlkkDC96fyDuyJ39nVbVjCKSDfj" + testProjectID = "00000000-0000-0000-0000-000000000000" +) + +// HandleServers mocks the STACKIT IAAS API. +func (m *SDMock) HandleServers() { + // /token endpoint mocks the token endpoint for service account authentication. + // It checks if the request body starts with "assertion=ey" to simulate a valid assertion + // as defined in RFC 7523. + m.Mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) { + reqBody, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = fmt.Fprint(w, err) + return + } + + // Expecting HTTP form encoded body with the field assertion. + // JWT always start with "ey" (base64url encoded). + if !bytes.HasPrefix(reqBody, []byte("assertion=ey")) { + w.WriteHeader(http.StatusUnauthorized) + return + } + + w.Header().Add("content-type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + + _, _ = fmt.Fprintf(w, `{"access_token": "%s"}`, testToken) + }) + + m.Mux.HandleFunc(fmt.Sprintf("/v1/projects/%s/servers", testProjectID), func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", testToken) { + w.WriteHeader(http.StatusUnauthorized) + return + } + + w.Header().Add("content-type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + + _, _ = fmt.Fprint(w, ` +{ + "items": [ + { + "availabilityZone": "eu01-3", + "bootVolume": { + "deleteOnTermination": false, + "id": "1c15e4cc-8474-46be-b875-b473ea9fe80c" + }, + "createdAt": "2025-03-12T14:48:17Z", + "id": "b4176700-596a-4f80-9fc8-5f9c58a606e1", + "labels": { + "provisionSTACKITServerAgent": "true", + "stackit_project_id": "00000000-0000-0000-0000-000000000000" + }, + "launchedAt": "2025-03-12T14:48:52Z", + "machineType": "g1.1", + "name": "runcommandtest", + "nics": [ + { + "ipv4": "10.0.0.153", + "mac": "fa:16:4f:42:1c:d3", + "networkId": "3173494f-2f6c-490d-8c12-4b3c86b4338b", + "networkName": "test", + "publicIp": "192.0.2.1", + "nicId": "b36097c5-e1c5-4e12-ae97-c03e144db127", + "nicSecurity": true, + "securityGroups": [ + "6e60809f-bed3-46c6-a39c-adddd6455674" + ] + } + ], + "powerStatus": "STOPPED", + "serviceAccountMails": [], + "status": "INACTIVE", + "updatedAt": "2025-03-13T07:08:29Z", + "userData": null, + "volumes": [ + "1c15e4cc-8474-46be-b875-b473ea9fe80c" + ] + }, + { + "availabilityZone": "eu01-m", + "bootVolume": { + "deleteOnTermination": false, + "id": "1e3ffe2b-878f-46e5-b39e-372e13a09551" + }, + "createdAt": "2025-04-10T16:45:25Z", + "id": "ee337436-1f15-4647-a03e-154009966179", + "labels": {}, + "launchedAt": "2025-04-10T16:46:00Z", + "machineType": "t1.1", + "name": "server1", + "nics": [], + "powerStatus": "RUNNING", + "serviceAccountMails": [], + "status": "ACTIVE", + "updatedAt": "2025-04-10T16:46:00Z", + "volumes": [ + "1e3ffe2b-878f-46e5-b39e-372e13a09551" + ] + } + ] +}`, + ) + }) +} diff --git a/discovery/stackit/server.go b/discovery/stackit/server.go new file mode 100644 index 0000000000..1be834a689 --- /dev/null +++ b/discovery/stackit/server.go @@ -0,0 +1,222 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/stackitcloud/stackit-sdk-go/core/auth" + stackitconfig "github.com/stackitcloud/stackit-sdk-go/core/config" + + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/util/strutil" +) + +const ( + stackitAPIEndpoint = "https://iaas.api.%s.stackit.cloud" + + stackitLabelPrivateIPv4 = stackitLabelPrefix + "private_ipv4_" + stackitLabelType = stackitLabelPrefix + "type" + stackitLabelLabel = stackitLabelPrefix + "label_" + stackitLabelLabelPresent = stackitLabelPrefix + "labelpresent_" +) + +// Discovery periodically performs STACKIT Cloud requests. +// It implements the Discoverer interface. +type iaasDiscovery struct { + *refresh.Discovery + httpClient *http.Client + logger *slog.Logger + apiEndpoint string + project string + port int +} + +// newServerDiscovery returns a new iaasDiscovery, which periodically refreshes its targets. +func newServerDiscovery(conf *SDConfig, logger *slog.Logger) (*iaasDiscovery, error) { + d := &iaasDiscovery{ + project: conf.Project, + port: conf.Port, + apiEndpoint: conf.Endpoint, + logger: logger, + } + + rt, err := config.NewRoundTripperFromConfig(conf.HTTPClientConfig, "stackit_sd") + if err != nil { + return nil, err + } + + d.apiEndpoint = conf.Endpoint + if d.apiEndpoint == "" { + d.apiEndpoint = fmt.Sprintf(stackitAPIEndpoint, conf.Region) + } + + servers := stackitconfig.ServerConfigurations{stackitconfig.ServerConfiguration{ + URL: d.apiEndpoint, + Description: "STACKIT IAAS API", + }} + + d.httpClient = &http.Client{ + Timeout: time.Duration(conf.RefreshInterval), + Transport: rt, + } + + stackitConfiguration := &stackitconfig.Configuration{ + UserAgent: userAgent, + HTTPClient: d.httpClient, + Servers: servers, + NoAuth: conf.ServiceAccountKey == "" && conf.ServiceAccountKeyPath == "", + + ServiceAccountKey: conf.ServiceAccountKey, + PrivateKey: conf.PrivateKey, + ServiceAccountKeyPath: conf.ServiceAccountKeyPath, + PrivateKeyPath: conf.PrivateKeyPath, + CredentialsFilePath: conf.CredentialsFilePath, + } + + if conf.tokenURL != "" { + stackitConfiguration.TokenCustomUrl = conf.tokenURL + } + + authRoundTripper, err := auth.SetupAuth(stackitConfiguration) + if err != nil { + return nil, fmt.Errorf("setting up authentication: %w", err) + } + + d.httpClient.Transport = authRoundTripper + + return d, nil +} + +func (i *iaasDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + apiURL, err := url.Parse(i.apiEndpoint) + if err != nil { + return nil, fmt.Errorf("invalid API endpoint URL %s: %w", i.apiEndpoint, err) + } + + apiURL.Path, err = url.JoinPath(apiURL.Path, "v1", "projects", i.project, "servers") + if err != nil { + return nil, fmt.Errorf("joining URL path: %w", err) + } + + q := apiURL.Query() + q.Set("details", "true") + apiURL.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL.String(), nil) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + + req.Header.Set("Accept", "application/json") + + res, err := i.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("sending request: %w", err) + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + errorMessage, _ := io.ReadAll(res.Body) + + return nil, fmt.Errorf("unexpected status code %d: %s", res.StatusCode, string(errorMessage)) + } + + var serversResponse *ServerListResponse + + if err := json.NewDecoder(res.Body).Decode(&serversResponse); err != nil { + return nil, fmt.Errorf("decoding response: %w", err) + } + + if serversResponse == nil || serversResponse.Items == nil || len(*serversResponse.Items) == 0 { + return []*targetgroup.Group{{Source: "stackit", Targets: []model.LabelSet{}}}, nil + } + + targets := make([]model.LabelSet, 0, len(*serversResponse.Items)) + for _, server := range *serversResponse.Items { + if server.Nics == nil { + i.logger.Debug("server has no network interfaces. Skipping", slog.String("server_id", server.ID)) + continue + } + + labels := model.LabelSet{ + stackitLabelProject: model.LabelValue(i.project), + stackitLabelID: model.LabelValue(server.ID), + stackitLabelName: model.LabelValue(server.Name), + stackitLabelAvailabilityZone: model.LabelValue(server.AvailabilityZone), + stackitLabelStatus: model.LabelValue(server.Status), + stackitLabelPowerStatus: model.LabelValue(server.PowerStatus), + stackitLabelType: model.LabelValue(server.MachineType), + } + + var ( + addressLabel string + serverPublicIP string + ) + + for _, nic := range server.Nics { + if nic.PublicIP != nil && *nic.PublicIP != "" && serverPublicIP == "" { + serverPublicIP = *nic.PublicIP + addressLabel = serverPublicIP + } + + if nic.IPv4 != nil && *nic.IPv4 != "" { + networkLabel := model.LabelName(stackitLabelPrivateIPv4 + strutil.SanitizeLabelName(nic.NetworkName)) + labels[networkLabel] = model.LabelValue(*nic.IPv4) + if addressLabel == "" { + addressLabel = *nic.IPv4 + } + } + } + + if addressLabel == "" { + // Skip servers without IPs. + continue + } + + // Public IPs for servers are optional. + if serverPublicIP != "" { + labels[stackitLabelPublicIPv4] = model.LabelValue(serverPublicIP) + } + + labels[model.AddressLabel] = model.LabelValue(net.JoinHostPort(addressLabel, strconv.FormatUint(uint64(i.port), 10))) + + for labelKey, labelValue := range server.Labels { + if labelStringValue, ok := labelValue.(string); ok { + presentLabel := model.LabelName(stackitLabelLabelPresent + strutil.SanitizeLabelName(labelKey)) + labels[presentLabel] = "true" + + label := model.LabelName(stackitLabelLabel + strutil.SanitizeLabelName(labelKey)) + labels[label] = model.LabelValue(labelStringValue) + } + } + + targets = append(targets, labels) + } + + return []*targetgroup.Group{{Source: "stackit", Targets: targets}}, nil +} diff --git a/discovery/stackit/server_test.go b/discovery/stackit/server_test.go new file mode 100644 index 0000000000..117fbdd66d --- /dev/null +++ b/discovery/stackit/server_test.go @@ -0,0 +1,131 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "testing" + + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + "github.com/stretchr/testify/require" +) + +type serverSDTestSuite struct { + Mock *SDMock +} + +func (s *serverSDTestSuite) SetupTest(t *testing.T) { + s.Mock = NewSDMock(t) + s.Mock.Setup() + + s.Mock.HandleServers() +} + +func TestServerSDRefresh(t *testing.T) { + for _, tc := range []struct { + name string + cfg SDConfig + }{ + { + name: "default with token", + cfg: func() SDConfig { + cfg := DefaultSDConfig + cfg.HTTPClientConfig.BearerToken = testToken + + return cfg + }(), + }, + { + name: "default with service account key", + cfg: func() SDConfig { + // Generate a new RSA key pair with a size of 2048 bits + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + cfg := DefaultSDConfig + cfg.PrivateKey = string(pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + })) + + cfg.ServiceAccountKey = `{ + "Active": true, + "CreatedAt": "2025-04-05T12:34:56Z", + "Credentials": { + "Aud": "https://stackit-service-account-prod.apps.01.cf.eu01.stackit.cloud", + "Iss": "stackit@sa.stackit.cloud", + "Kid": "123e4567-e89b-12d3-a456-426614174000", + "Sub": "123e4567-e89b-12d3-a456-426614174001" + }, + "ID": "123e4567-e89b-12d3-a456-426614174002", + "KeyAlgorithm": "RSA_2048", + "KeyOrigin": "USER_PROVIDED", + "KeyType": "USER_MANAGED", + "PublicKey": "...", + "ValidUntil": "2025-04-05T13:34:56Z" +}` + + return cfg + }(), + }, + } { + t.Run(tc.name, func(t *testing.T) { + suite := &serverSDTestSuite{} + suite.SetupTest(t) + defer suite.Mock.ShutdownServer() + + tc.cfg.Endpoint = suite.Mock.Endpoint() + tc.cfg.tokenURL = suite.Mock.Endpoint() + "token" + tc.cfg.Project = testProjectID + + d, err := newServerDiscovery(&tc.cfg, promslog.NewNopLogger()) + require.NoError(t, err) + + targetGroups, err := d.refresh(context.Background()) + require.NoError(t, err) + require.Len(t, targetGroups, 1) + + targetGroup := targetGroups[0] + require.NotNil(t, targetGroup, "targetGroup should not be nil") + require.NotNil(t, targetGroup.Targets, "targetGroup.targets should not be nil") + require.Len(t, targetGroup.Targets, 1) + + for i, labelSet := range []model.LabelSet{ + { + "__address__": model.LabelValue("192.0.2.1:80"), + "__meta_stackit_project": model.LabelValue("00000000-0000-0000-0000-000000000000"), + "__meta_stackit_id": model.LabelValue("b4176700-596a-4f80-9fc8-5f9c58a606e1"), + "__meta_stackit_type": model.LabelValue("g1.1"), + "__meta_stackit_private_ipv4_test": model.LabelValue("10.0.0.153"), + "__meta_stackit_public_ipv4": model.LabelValue("192.0.2.1"), + "__meta_stackit_labelpresent_provisionSTACKITServerAgent": model.LabelValue("true"), + "__meta_stackit_label_provisionSTACKITServerAgent": model.LabelValue("true"), + "__meta_stackit_labelpresent_stackit_project_id": model.LabelValue("true"), + "__meta_stackit_name": model.LabelValue("runcommandtest"), + "__meta_stackit_availability_zone": model.LabelValue("eu01-3"), + "__meta_stackit_status": model.LabelValue("INACTIVE"), + "__meta_stackit_power_status": model.LabelValue("STOPPED"), + "__meta_stackit_label_stackit_project_id": model.LabelValue("00000000-0000-0000-0000-000000000000"), + }, + } { + require.Equal(t, labelSet, targetGroup.Targets[i]) + } + }) + } +} diff --git a/discovery/stackit/stackit.go b/discovery/stackit/stackit.go new file mode 100644 index 0000000000..030f2bdb55 --- /dev/null +++ b/discovery/stackit/stackit.go @@ -0,0 +1,153 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/version" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +const ( + stackitLabelPrefix = model.MetaLabelPrefix + "stackit_" + stackitLabelProject = stackitLabelPrefix + "project" + stackitLabelID = stackitLabelPrefix + "id" + stackitLabelName = stackitLabelPrefix + "name" + stackitLabelStatus = stackitLabelPrefix + "status" + stackitLabelPowerStatus = stackitLabelPrefix + "power_status" + stackitLabelAvailabilityZone = stackitLabelPrefix + "availability_zone" + stackitLabelPublicIPv4 = stackitLabelPrefix + "public_ipv4" +) + +var userAgent = version.PrometheusUserAgent() + +// DefaultSDConfig is the default STACKIT SD configuration. +var DefaultSDConfig = SDConfig{ + Region: "eu01", + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + HTTPClientConfig: config.DefaultHTTPClientConfig, +} + +func init() { + discovery.RegisterConfig(&SDConfig{}) +} + +// SDConfig is the configuration for STACKIT based service discovery. +type SDConfig struct { + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` + + Project string `yaml:"project"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port,omitempty"` + Region string `yaml:"region,omitempty"` + Endpoint string `yaml:"endpoint,omitempty"` + ServiceAccountKey string `yaml:"service_account_key,omitempty"` + PrivateKey string `yaml:"private_key,omitempty"` + ServiceAccountKeyPath string `yaml:"service_account_key_path,omitempty"` + PrivateKeyPath string `yaml:"private_key_path,omitempty"` + CredentialsFilePath string `yaml:"credentials_file_path,omitempty"` + + // For testing only + tokenURL string +} + +// NewDiscovererMetrics implements discovery.Config. +func (*SDConfig) NewDiscovererMetrics(_ prometheus.Registerer, rmi discovery.RefreshMetricsInstantiator) discovery.DiscovererMetrics { + return &stackitMetrics{ + refreshMetrics: rmi, + } +} + +// Name returns the name of the Config. +func (*SDConfig) Name() string { return "stackit" } + +// NewDiscoverer returns a Discoverer for the Config. +func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewDiscovery(c, opts.Logger, opts.Metrics) +} + +type refresher interface { + refresh(context.Context) ([]*targetgroup.Group, error) +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + + if c.Endpoint == "" && c.Region == "" { + return errors.New("stackit_sd: endpoint and region missing") + } + + if _, err = url.Parse(c.Endpoint); err != nil { + return fmt.Errorf("stackit_sd: invalid endpoint %q: %w", c.Endpoint, err) + } + + return c.HTTPClientConfig.Validate() +} + +// SetDirectory joins any relative file paths with dir. +func (c *SDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) +} + +// Discovery periodically performs STACKIT API requests. It implements +// the Discoverer interface. +type Discovery struct { + *refresh.Discovery +} + +// NewDiscovery returns a new Discovery which periodically refreshes its targets. +func NewDiscovery(conf *SDConfig, logger *slog.Logger, metrics discovery.DiscovererMetrics) (*refresh.Discovery, error) { + m, ok := metrics.(*stackitMetrics) + if !ok { + return nil, errors.New("invalid discovery metrics type") + } + + r, err := newRefresher(conf, logger) + if err != nil { + return nil, err + } + + return refresh.NewDiscovery( + refresh.Options{ + Logger: logger, + Mech: "stackit", + Interval: time.Duration(conf.RefreshInterval), + RefreshF: r.refresh, + MetricsInstantiator: m.refreshMetrics, + }, + ), nil +} + +func newRefresher(conf *SDConfig, l *slog.Logger) (refresher, error) { + return newServerDiscovery(conf, l) +} diff --git a/discovery/stackit/types.go b/discovery/stackit/types.go new file mode 100644 index 0000000000..66681c3455 --- /dev/null +++ b/discovery/stackit/types.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stackit + +// ServerListResponse Response object for server list request. +// https://docs.api.eu01.stackit.cloud/documentation/iaas/version/v1#tag/Servers/operation/v1ListServersInProject +type ServerListResponse struct { + Items *[]Server `json:"items"` +} + +type Server struct { + AvailabilityZone string `json:"availabilityZone"` + ID string `json:"id"` + Labels map[string]interface{} `json:"labels"` + MachineType string `json:"machineType"` + Name string `json:"name"` + Nics []ServerNetwork `json:"nics"` + PowerStatus string `json:"powerStatus"` + Status string `json:"status"` +} + +// ServerNetwork Describes the object that matches servers to its networks. +type ServerNetwork struct { + NetworkName string `json:"networkName"` + IPv4 *string `json:"ipv4,omitempty"` + PublicIP *string `json:"publicIp,omitempty"` +} diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 21a06748ac..539e9933d3 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -431,6 +431,10 @@ scaleway_sd_configs: serverset_sd_configs: [ - ... ] +# List of STACKIT service discovery configurations. +stackit_sd_configs: + [ - ... ] + # List of Triton service discovery configurations. triton_sd_configs: [ - ... ] @@ -2258,6 +2262,70 @@ paths: Serverset data must be in the JSON format, the Thrift format is not currently supported. +### `` + +[STACKIT](https://www.stackit.de/de/) SD configurations allow retrieving +scrape targets from various APIs. + +The following meta labels are available on targets during [relabeling](#relabel_config): + +* `__meta_stackit_availability_zone`: The availability zone of the server. +* `__meta_stackit_label_`: Each server label, with unsupported characters replaced by underscores. +* `__meta_stackit_labelpresent_`: "true" for each label of the server, with unsupported characters replaced by underscores. +* `__meta_stackit_private_ipv4_`: the private ipv4 address of the server within a given network +* `__meta_stackit_public_ipv4`: the public ipv4 address of the server +* `__meta_stackit_id`: The ID of the target. +* `__meta_stackit_type`: The type or brand of the target. +* `__meta_stackit_name`: The server name. +* `__meta_stackit_status`: The current status of the server. +* `__meta_stackit_power_status`: The power status of the server. + +See below for the configuration options for STACKIT discovery: + +```yaml +# The STACKIT project +project: + +# STACKIT region to use. No automatic discovery of the region is done. +[ region : | default = "eu01" ] + +# Custom API endpoint to be used. Format scheme://host:port +[ endpoint : ] + +# The port to scrape metrics from. +[ port: | default = 80 ] + +# Raw private key string used for authenticating a service account +[ private_key: ] + +# Path to a file containing the raw private key string +[ private_key_path: ] + +# Full JSON-formatted service account key used for authentication +[ service_account_key: ] + +# Path to a file containing the JSON-formatted service account key +[ service_account_key_path: ] + +# Path to a file containing STACKIT credentials. +[ credentials_file_path: ] + +# The time after which the servers are refreshed. +[ refresh_interval: | default = 60s ] + +# HTTP client settings, including authentication methods (such as basic auth and +# authorization), proxy configurations, TLS options, custom HTTP headers, etc. +[ ] +``` + +A Service Account Token can be set through `http_config`. + +```yaml +stackit_sd_config: +- authorization: + credentials: +``` + ### `` [Triton](https://github.com/joyent/triton) SD configurations allow retrieving @@ -2830,6 +2898,10 @@ scaleway_sd_configs: serverset_sd_configs: [ - ... ] +# List of STACKIT service discovery configurations. +stackit_sd_configs: + [ - ... ] + # List of Triton service discovery configurations. triton_sd_configs: [ - ... ] diff --git a/documentation/examples/prometheus-stackit.yml b/documentation/examples/prometheus-stackit.yml new file mode 100644 index 0000000000..623cb231ff --- /dev/null +++ b/documentation/examples/prometheus-stackit.yml @@ -0,0 +1,34 @@ +# A example scrape configuration for running Prometheus with +# STACKIT. + +scrape_configs: + # Make Prometheus scrape itself for metrics. + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + + # Discover Node Exporter instances to scrape. + - job_name: "node" + + stackit_sd_configs: + - project: 11111111-1111-1111-1111-111111111111 + authorization: + credentials: "" + relabel_configs: + # Use the public IPv4 and port 9100 to scrape the target. + - source_labels: [__meta_stackit_public_ipv4] + target_label: __address__ + replacement: "$1:9100" + + # Discover Node Exporter instances to scrape using a STACKIT Subnet called mynet. + - job_name: "node_private" + + stackit_sd_configs: + - project: 11111111-1111-1111-1111-111111111111 + authorization: + credentials: "" + relabel_configs: + # Use the private IPv4 within the STACKIT Subnet and port 9100 to scrape the target. + - source_labels: [__meta_stackit_private_ipv4_mynet] + target_label: __address__ + replacement: "$1:9100" diff --git a/go.mod b/go.mod index 9bf1c26e79..07493eacf4 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,7 @@ require ( github.com/prometheus/sigv4 v0.1.2 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.33 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c + github.com/stackitcloud/stackit-sdk-go/core v0.16.2 github.com/stretchr/testify v1.10.0 github.com/vultr/govultr/v2 v2.17.2 go.opentelemetry.io/collector/component v1.31.0 diff --git a/go.sum b/go.sum index ddb906f7a6..c8ce5bbf67 100644 --- a/go.sum +++ b/go.sum @@ -459,6 +459,8 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stackitcloud/stackit-sdk-go/core v0.16.2 h1:F8A4P/LLlQSbz0S0+G3m8rb3BUOK6EcR/CKx5UQY5jQ= +github.com/stackitcloud/stackit-sdk-go/core v0.16.2/go.mod h1:8KIw3czdNJ9sdil9QQimxjR6vHjeINFrRv0iZ67wfn0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= diff --git a/plugins.yml b/plugins.yml index c7b9d297d0..0541fe4852 100644 --- a/plugins.yml +++ b/plugins.yml @@ -16,6 +16,7 @@ - github.com/prometheus/prometheus/discovery/ovhcloud - github.com/prometheus/prometheus/discovery/puppetdb - github.com/prometheus/prometheus/discovery/scaleway +- github.com/prometheus/prometheus/discovery/stackit - github.com/prometheus/prometheus/discovery/triton - github.com/prometheus/prometheus/discovery/uyuni - github.com/prometheus/prometheus/discovery/vultr diff --git a/plugins/plugins.go b/plugins/plugins.go index 15379d4705..90b1407281 100644 --- a/plugins/plugins.go +++ b/plugins/plugins.go @@ -52,6 +52,8 @@ import ( _ "github.com/prometheus/prometheus/discovery/puppetdb" // Register scaleway plugin. _ "github.com/prometheus/prometheus/discovery/scaleway" + // Register stackit plugin. + _ "github.com/prometheus/prometheus/discovery/stackit" // Register triton plugin. _ "github.com/prometheus/prometheus/discovery/triton" // Register uyuni plugin.