storage: make Azure blobID chunk delimiter configurable (#5777)

* Make Azure chunk delimiter configurable

* Changelog: #5777

* doc update
pull/5818/head
Tat Chiu Leung 4 years ago committed by GitHub
parent 6b74422150
commit e65f26d30f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      CHANGELOG.md
  2. 4
      docs/sources/configuration/_index.md
  3. 4
      pkg/storage/chunk/azure/blob_storage_client.go

@ -1,5 +1,6 @@
## Main
* [5780](https://github.com/grafana/loki/pull/5780) **simonswine**: Update alpine image to 3.15.4.
* [5777](https://github.com/grafana/loki/pull/5777) **tatchiuleung** storage: make Azure blobID chunk delimiter configurable.
* [5715](https://github.com/grafana/loki/pull/5715) **chaudum** Add option to push RFC5424 syslog messages from Promtail in syslog scrape target.
* [5696](https://github.com/grafana/loki/pull/5696) **paullryan** don't block scraping of new logs from cloudflare within promtail if an error is received from cloudflare about too early logs.
* [5685](https://github.com/grafana/loki/pull/5625) **chaudum** Fix bug in push request parser that allowed users to send arbitrary non-string data as "log line".

@ -743,6 +743,10 @@ The `azure_storage_config` configures Azure as a general storage for different d
# CLI flag: -<prefix>.azure.account-key
[account_key: <string> | default = ""]
# Chunk delimiter to build the blobID
# CLI flag: -<prefix>.azure.chunk-delimiter
[chunk_delimiter: <string> | default = "-"]
# Preallocated buffer size for downloads.
# CLI flag: -<prefix>.azure.download-buffer-size
[download_buffer_size: <int> | default = 512000]

@ -85,6 +85,7 @@ type BlobStorageConfig struct {
Environment string `yaml:"environment"`
ContainerName string `yaml:"container_name"`
AccountName string `yaml:"account_name"`
ChunkDelimiter string `yaml:"chunk_delimiter"`
AccountKey flagext.Secret `yaml:"account_key"`
DownloadBufferSize int `yaml:"download_buffer_size"`
UploadBufferSize int `yaml:"upload_buffer_size"`
@ -106,6 +107,7 @@ func (c *BlobStorageConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagS
f.StringVar(&c.Environment, prefix+"azure.environment", azureGlobal, fmt.Sprintf("Azure Cloud environment. Supported values are: %s.", strings.Join(supportedEnvironments, ", ")))
f.StringVar(&c.ContainerName, prefix+"azure.container-name", "cortex", "Name of the blob container used to store chunks. This container must be created before running cortex.")
f.StringVar(&c.AccountName, prefix+"azure.account-name", "", "The Microsoft Azure account name to be used")
f.StringVar(&c.ChunkDelimiter, prefix+"azure.chunk-delimiter", "-", "Chunk delimiter for blob ID to be used")
f.Var(&c.AccountKey, prefix+"azure.account-key", "The Microsoft Azure account key to use.")
f.DurationVar(&c.RequestTimeout, prefix+"azure.request-timeout", 30*time.Second, "Timeout for requests made against azure blob storage.")
f.IntVar(&c.DownloadBufferSize, prefix+"azure.download-buffer-size", 512000, "Preallocated buffer size for downloads.")
@ -251,7 +253,7 @@ func (b *BlobStorage) PutObject(ctx context.Context, objectKey string, object io
}
func (b *BlobStorage) getBlobURL(blobID string, hedging bool) (azblob.BlockBlobURL, error) {
blobID = strings.Replace(blobID, ":", "-", -1)
blobID = strings.Replace(blobID, ":", b.cfg.ChunkDelimiter, -1)
// generate url for new chunk blob
u, err := url.Parse(fmt.Sprintf(b.selectBlobURLFmt(), b.cfg.AccountName, b.cfg.ContainerName, blobID))

Loading…
Cancel
Save