Upgrade to Go 1.19 (#7243)

Upgrade Loki and all related components to Go 1.19.2.
pull/7338/head
Danny Kopping 4 years ago committed by GitHub
parent b795672bba
commit 355175ff89
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      .circleci/config.yml
  2. 34
      .drone/drone.yml
  3. 17
      .golangci.yml
  4. 6
      Makefile
  5. 2
      clients/cmd/docker-driver/Dockerfile
  6. 2
      clients/cmd/fluent-bit/Dockerfile
  7. 2
      clients/cmd/promtail/Dockerfile
  8. 2
      clients/cmd/promtail/Dockerfile.arm32
  9. 4
      clients/cmd/promtail/Dockerfile.cross
  10. 2
      cmd/logcli/Dockerfile
  11. 2
      cmd/logql-analyzer/Dockerfile
  12. 2
      cmd/loki-canary/Dockerfile
  13. 4
      cmd/loki-canary/Dockerfile.cross
  14. 2
      cmd/loki/Dockerfile
  15. 4
      cmd/loki/Dockerfile.cross
  16. 2
      cmd/migrate/Dockerfile
  17. 2
      cmd/querytee/Dockerfile
  18. 4
      cmd/querytee/Dockerfile.cross
  19. 4
      go.mod
  20. 4
      go.sum
  21. 2
      operator/Dockerfile
  22. 4
      operator/Dockerfile.cross
  23. 2
      operator/calculator.Dockerfile
  24. 44
      pkg/storage/chunk/client/grpc/grpc.pb.go
  25. 2
      tools/lambda-promtail/Dockerfile
  26. 2
      vendor/github.com/grafana/regexp/exec.go
  27. 59
      vendor/github.com/grafana/regexp/regexp.go
  28. 185
      vendor/github.com/grafana/regexp/syntax/doc.go
  29. 170
      vendor/github.com/grafana/regexp/syntax/parse.go
  30. 5
      vendor/github.com/grafana/regexp/syntax/prog.go
  31. 2
      vendor/modules.txt

@ -36,7 +36,7 @@ workflows:
# https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/
.defaults: &defaults
docker:
- image: grafana/loki-build-image:0.23.0
- image: grafana/loki-build-image:0.24.1
working_directory: /src/loki
jobs:

@ -53,14 +53,14 @@ steps:
depends_on:
- clone
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: check-drone-drift
- commands:
- make BUILD_IN_CONTAINER=false check-generated-files
depends_on:
- clone
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: check-generated-files
- commands:
- cd ..
@ -69,7 +69,7 @@ steps:
depends_on:
- clone
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: clone-main
- commands:
- make BUILD_IN_CONTAINER=false test
@ -77,7 +77,7 @@ steps:
- clone
- clone-main
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: test
- commands:
- cd ../loki-main
@ -85,7 +85,7 @@ steps:
depends_on:
- clone-main
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: test-main
- commands:
- make BUILD_IN_CONTAINER=false compare-coverage old=../loki-main/test_results.txt
@ -95,7 +95,7 @@ steps:
- test
- test-main
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: compare-coverage
- commands:
- pull=$(echo $CI_COMMIT_REF | awk -F '/' '{print $3}')
@ -108,7 +108,7 @@ steps:
TOKEN:
from_secret: github_token
USER: grafanabot
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: report-coverage
- commands:
- make BUILD_IN_CONTAINER=false lint
@ -116,7 +116,7 @@ steps:
- clone
- check-generated-files
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: lint
- commands:
- make BUILD_IN_CONTAINER=false check-mod
@ -125,7 +125,7 @@ steps:
- test
- lint
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: check-mod
- commands:
- apk add make bash && make lint-scripts
@ -137,21 +137,21 @@ steps:
- clone
- check-generated-files
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: loki
- commands:
- make BUILD_IN_CONTAINER=false validate-example-configs
depends_on:
- loki
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: validate-example-configs
- commands:
- make BUILD_IN_CONTAINER=false check-example-config-doc
depends_on:
- clone
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: check-example-config-doc
trigger:
ref:
@ -178,7 +178,7 @@ steps:
depends_on:
- clone
environment: {}
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: loki-mixin-check
trigger:
ref:
@ -1255,7 +1255,7 @@ steps:
NFPM_SIGNING_KEY:
from_secret: gpg_private_key
NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: write-key
- commands:
- make BUILD_IN_CONTAINER=false packages
@ -1263,7 +1263,7 @@ steps:
NFPM_PASSPHRASE:
from_secret: gpg_passphrase
NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: test packaging
- commands:
- ./tools/packaging/verify-deb-install.sh
@ -1289,7 +1289,7 @@ steps:
NFPM_PASSPHRASE:
from_secret: gpg_passphrase
NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
image: grafana/loki-build-image:0.23.0
image: grafana/loki-build-image:0.24.1
name: publish
when:
event:
@ -1528,6 +1528,6 @@ kind: secret
name: gpg_private_key
---
kind: signature
hmac: 82c2ce679681c7b24901ec66097dd822a33d405663d540cda3572f202d86f6ab
hmac: 455cb80ad1fd753c9d5432aee9ed9242b12a1d31ae33694a014a5387d610f3a2
...

@ -4,7 +4,7 @@
# options for analysis running
run:
# define go version
go: "1.18"
go: "1.19"
# default concurrency is a available CPU number
concurrency: 16
@ -62,23 +62,26 @@ linters:
- deadcode
- errcheck
- goconst
- gofmt
- goimports
- revive
- gosimple
- ineffassign
- staticcheck
- misspell
- structcheck
- unconvert
- unparam
- varcheck
- govet
- unused
- typecheck
- depguard
- exportloopref
# TODO(dannyk): restore these and fix linter errors in separate PR
disable:
- unused
- gofmt
- goimports
- gosimple
- staticcheck
- unparam
issues:
exclude:
- Error return value of .*log\.Logger\)\.Log\x60 is not checked

@ -27,7 +27,7 @@ DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES))
BUILD_IN_CONTAINER ?= true
# ensure you run `make drone` after changing this
BUILD_IMAGE_VERSION := 0.23.0
BUILD_IMAGE_VERSION := 0.24.1
# Docker image info
IMAGE_PREFIX ?= grafana
@ -256,8 +256,10 @@ publish: packages
########
# To run this efficiently on your workstation, run this from the root dir:
# docker run --rm --tty -i -v $(pwd)/.cache:/go/cache -v $(pwd)/.pkg:/go/pkg -v $(pwd):/src/loki grafana/loki-build-image:0.23.0 lint
# docker run --rm --tty -i -v $(pwd)/.cache:/go/cache -v $(pwd)/.pkg:/go/pkg -v $(pwd):/src/loki grafana/loki-build-image:0.24.1 lint
lint:
go version
golangci-lint version
GO111MODULE=on golangci-lint run -v
faillint -paths "sync/atomic=go.uber.org/atomic" ./...

@ -1,4 +1,4 @@
ARG BUILD_IMAGE=grafana/loki-build-image:0.18.0
ARG BUILD_IMAGE=grafana/loki-build-image:0.24.1
# Directories in this file are referenced from the root of the project not this folder
# This file is intended to be called from the root like so:
# docker build -t grafana/loki -f cmd/loki/Dockerfile .

@ -1,4 +1,4 @@
FROM golang:1.18.5 as build
FROM golang:1.19.2 as build
COPY . /src/loki
WORKDIR /src/loki
RUN make clean && make BUILD_IN_CONTAINER=false fluent-bit-plugin

@ -1,4 +1,4 @@
FROM golang:1.18.5-bullseye as build
FROM golang:1.19.2-bullseye as build
COPY . /src/loki
WORKDIR /src/loki

@ -1,4 +1,4 @@
FROM golang:1.18.5-bullseye as build
FROM golang:1.19.2-bullseye as build
COPY . /src/loki
WORKDIR /src/loki

@ -1,8 +1,8 @@
ARG BUILD_IMAGE=grafana/loki-build-image:0.18.0
ARG BUILD_IMAGE=grafana/loki-build-image:0.24.1
# Directories in this file are referenced from the root of the project not this folder
# This file is intended to be called from the root like so:
# docker build -t grafana/promtail -f clients/cmd/promtail/Dockerfile .
FROM golang:1.18.5-alpine as goenv
FROM golang:1.19.2-alpine as goenv
RUN go env GOARCH > /goarch && \
go env GOARM > /goarm

@ -1,4 +1,4 @@
FROM golang:1.18.5 as build
FROM golang:1.19.2 as build
COPY . /src/loki
WORKDIR /src/loki

@ -1,4 +1,4 @@
FROM golang:1.17.9 as build
FROM golang:1.19.2 as build
COPY . /src/loki
WORKDIR /src/loki

@ -1,4 +1,4 @@
FROM golang:1.18.5 as build
FROM golang:1.19.2 as build
COPY . /src/loki
WORKDIR /src/loki

@ -1,8 +1,8 @@
ARG BUILD_IMAGE=grafana/loki-build-image:0.18.0
ARG BUILD_IMAGE=grafana/loki-build-image:0.24.1
# Directories in this file are referenced from the root of the project not this folder
# This file is intended to be called from the root like so:
# docker build -t grafana/promtail -f cmd/promtail/Dockerfile .
FROM golang:1.18.5-alpine as goenv
FROM golang:1.19.2-alpine as goenv
RUN go env GOARCH > /goarch && \
go env GOARM > /goarm

@ -1,4 +1,4 @@
FROM golang:1.18.5 as build
FROM golang:1.19.2 as build
COPY . /src/loki
WORKDIR /src/loki

@ -1,8 +1,8 @@
ARG BUILD_IMAGE=grafana/loki-build-image:0.18.0
ARG BUILD_IMAGE=grafana/loki-build-image:0.24.1
# Directories in this file are referenced from the root of the project not this folder
# This file is intended to be called from the root like so:
# docker build -t grafana/loki -f cmd/loki/Dockerfile .
FROM golang:1.18.5-alpine as goenv
FROM golang:1.19.2-alpine as goenv
RUN go env GOARCH > /goarch && \
go env GOARM > /goarm

@ -1,4 +1,4 @@
FROM golang:1.18.5 as build
FROM golang:1.19.2 as build
COPY . /src/loki
WORKDIR /src/loki
RUN make clean && make BUILD_IN_CONTAINER=false migrate

@ -1,4 +1,4 @@
FROM golang:1.18.5 as build
FROM golang:1.19.2 as build
COPY . /src/loki
WORKDIR /src/loki

@ -1,8 +1,8 @@
ARG BUILD_IMAGE=grafana/loki-build-image:0.18.0
ARG BUILD_IMAGE=grafana/loki-build-image:0.24.1
# Directories in this file are referenced from the root of the project not this folder
# This file is intended to be called from the root like so:
# docker build -t grafana/promtail -f cmd/promtail/Dockerfile .
FROM golang:1.18.5-alpine as goenv
FROM golang:1.19.2-alpine as goenv
RUN go env GOARCH > /goarch && \
go env GOARM > /goarm

@ -1,6 +1,6 @@
module github.com/grafana/loki
go 1.18
go 1.19
require (
cloud.google.com/go/bigtable v1.3.0
@ -51,7 +51,7 @@ require (
github.com/gorilla/websocket v1.4.2
github.com/grafana/dskit v0.0.0-20220928083349-b1b307db4f30
github.com/grafana/go-gelf/v2 v2.0.1
github.com/grafana/regexp v0.0.0-20220304100321-149c8afcd6cb
github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
github.com/hashicorp/consul/api v1.13.0

@ -866,8 +866,8 @@ github.com/grafana/groupcache_exporter v0.0.0-20220629095919-59a8c6428a43/go.mod
github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe h1:yIXAAbLswn7VNWBIvM71O2QsgfgW9fRXZNR0DXe6pDU=
github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
github.com/grafana/regexp v0.0.0-20220304100321-149c8afcd6cb h1:wwzNkyaQwcXCzQuKoWz3lwngetmcyg+EhW0fF5lz73M=
github.com/grafana/regexp v0.0.0-20220304100321-149c8afcd6cb/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 h1:A3dhViTeFDSQcGOXuUi6ukCQSMyDtDISBp2z6OOo2YM=
github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
github.com/grafana/tail v0.0.0-20220426200921-98e8eb28ea4c h1:qIsCzNln5YzuXfXbJgXhpfM+4gY7qi3mED3eYQS4Fls=
github.com/grafana/tail v0.0.0-20220426200921-98e8eb28ea4c/go.mod h1:GIMXMPB/lRAllP5rVDvcGif87ryO2hgD7tCtHMdHrho=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=

@ -1,5 +1,5 @@
# Build the manager binary
FROM golang:1.18.5 as builder
FROM golang:1.19.1 as builder
WORKDIR /workspace
# Copy the Go Modules manifests

@ -1,6 +1,6 @@
ARG BUILD_IMAGE=grafana/loki-build-image:0.18.0
ARG BUILD_IMAGE=grafana/loki-build-image:0.24.1
FROM golang:1.18.5-alpine as goenv
FROM golang:1.19.1-alpine as goenv
RUN go env GOARCH > /goarch && \
go env GOARM > /goarm

@ -1,5 +1,5 @@
# Build the calculator binary
FROM golang:1.18.5 as builder
FROM golang:1.19.1 as builder
WORKDIR /workspace
# Copy the Go Modules manifests

@ -1997,24 +1997,24 @@ const _ = grpc.SupportPackageIsVersion4
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
type GrpcStoreClient interface {
/// WriteIndex writes batch of indexes to the index tables.
// / WriteIndex writes batch of indexes to the index tables.
WriteIndex(ctx context.Context, in *WriteIndexRequest, opts ...grpc.CallOption) (*empty.Empty, error)
/// QueryIndex reads the indexes required for given query & sends back the batch of rows
/// in rpc streams
// / QueryIndex reads the indexes required for given query & sends back the batch of rows
// / in rpc streams
QueryIndex(ctx context.Context, in *QueryIndexRequest, opts ...grpc.CallOption) (GrpcStore_QueryIndexClient, error)
/// DeleteIndex deletes the batch of index entries from the index tables
// / DeleteIndex deletes the batch of index entries from the index tables
DeleteIndex(ctx context.Context, in *DeleteIndexRequest, opts ...grpc.CallOption) (*empty.Empty, error)
/// PutChunks saves the batch of chunks into the chunk tables.
// / PutChunks saves the batch of chunks into the chunk tables.
PutChunks(ctx context.Context, in *PutChunksRequest, opts ...grpc.CallOption) (*empty.Empty, error)
/// GetChunks requests for batch of chunks and the batch of chunks are sent back in rpc streams
/// batching needs to be performed at server level as per requirement instead of sending single chunk per stream.
/// In GetChunks rpc request send buf as nil
// / GetChunks requests for batch of chunks and the batch of chunks are sent back in rpc streams
// / batching needs to be performed at server level as per requirement instead of sending single chunk per stream.
// / In GetChunks rpc request send buf as nil
GetChunks(ctx context.Context, in *GetChunksRequest, opts ...grpc.CallOption) (GrpcStore_GetChunksClient, error)
/// DeleteChunks deletes the chunks based on chunkID.
// / DeleteChunks deletes the chunks based on chunkID.
DeleteChunks(ctx context.Context, in *ChunkID, opts ...grpc.CallOption) (*empty.Empty, error)
/// Lists all the tables that exists in the database.
// / Lists all the tables that exists in the database.
ListTables(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*ListTablesResponse, error)
/// Creates a table with provided name & attributes.
// / Creates a table with provided name & attributes.
CreateTable(ctx context.Context, in *CreateTableRequest, opts ...grpc.CallOption) (*empty.Empty, error)
// Deletes a table using table name provided.
DeleteTable(ctx context.Context, in *DeleteTableRequest, opts ...grpc.CallOption) (*empty.Empty, error)
@ -2179,24 +2179,24 @@ func (c *grpcStoreClient) UpdateTable(ctx context.Context, in *UpdateTableReques
// GrpcStoreServer is the server API for GrpcStore service.
type GrpcStoreServer interface {
/// WriteIndex writes batch of indexes to the index tables.
// / WriteIndex writes batch of indexes to the index tables.
WriteIndex(context.Context, *WriteIndexRequest) (*empty.Empty, error)
/// QueryIndex reads the indexes required for given query & sends back the batch of rows
/// in rpc streams
// / QueryIndex reads the indexes required for given query & sends back the batch of rows
// / in rpc streams
QueryIndex(*QueryIndexRequest, GrpcStore_QueryIndexServer) error
/// DeleteIndex deletes the batch of index entries from the index tables
// / DeleteIndex deletes the batch of index entries from the index tables
DeleteIndex(context.Context, *DeleteIndexRequest) (*empty.Empty, error)
/// PutChunks saves the batch of chunks into the chunk tables.
// / PutChunks saves the batch of chunks into the chunk tables.
PutChunks(context.Context, *PutChunksRequest) (*empty.Empty, error)
/// GetChunks requests for batch of chunks and the batch of chunks are sent back in rpc streams
/// batching needs to be performed at server level as per requirement instead of sending single chunk per stream.
/// In GetChunks rpc request send buf as nil
// / GetChunks requests for batch of chunks and the batch of chunks are sent back in rpc streams
// / batching needs to be performed at server level as per requirement instead of sending single chunk per stream.
// / In GetChunks rpc request send buf as nil
GetChunks(*GetChunksRequest, GrpcStore_GetChunksServer) error
/// DeleteChunks deletes the chunks based on chunkID.
// / DeleteChunks deletes the chunks based on chunkID.
DeleteChunks(context.Context, *ChunkID) (*empty.Empty, error)
/// Lists all the tables that exists in the database.
// / Lists all the tables that exists in the database.
ListTables(context.Context, *empty.Empty) (*ListTablesResponse, error)
/// Creates a table with provided name & attributes.
// / Creates a table with provided name & attributes.
CreateTable(context.Context, *CreateTableRequest) (*empty.Empty, error)
// Deletes a table using table name provided.
DeleteTable(context.Context, *DeleteTableRequest) (*empty.Empty, error)

@ -1,4 +1,4 @@
FROM golang:1.18.5-alpine AS build-image
FROM golang:1.19.2-alpine AS build-image
COPY tools/lambda-promtail /src/lambda-promtail
WORKDIR /src/lambda-promtail

@ -430,6 +430,7 @@ func (re *Regexp) doOnePass(ir io.RuneReader, ib []byte, is string, pos, ncap in
} else {
m.matchcap = m.matchcap[:ncap]
}
for i := range m.matchcap {
m.matchcap[i] = -1
}
@ -440,6 +441,7 @@ func (re *Regexp) doOnePass(ir io.RuneReader, ib []byte, is string, pos, ncap in
} else {
flag = i.context(pos)
}
// If there is a simple literal prefix, skip over it.
if r != endOfText {
r1, width1 = i.step(pos + width)
}

@ -9,17 +9,22 @@
// More precisely, it is the syntax accepted by RE2 and described at
// https://golang.org/s/re2syntax, except for \C.
// For an overview of the syntax, run
// go doc regexp/syntax
//
// go doc regexp/syntax
//
// The regexp implementation provided by this package is
// guaranteed to run in time linear in the size of the input.
// (This is a property not guaranteed by most open source
// implementations of regular expressions.) For more information
// about this property, see
//
// https://swtch.com/~rsc/regexp/regexp1.html
//
// or any book about automata theory.
//
// All characters are UTF-8-encoded code points.
// Following utf8.DecodeRune, each byte of an invalid UTF-8 sequence
// is treated as if it encoded utf8.RuneError (U+FFFD).
//
// There are 16 methods of Regexp that match a regular expression and identify
// the matched text. Their names are matched by this regular expression:
@ -40,11 +45,11 @@
// successive submatches of the expression. Submatches are matches of
// parenthesized subexpressions (also known as capturing groups) within the
// regular expression, numbered from left to right in order of opening
// parenthesis. Submatch 0 is the match of the entire expression, submatch 1
// parenthesis. Submatch 0 is the match of the entire expression, submatch 1 is
// the match of the first parenthesized subexpression, and so on.
//
// If 'Index' is present, matches and submatches are identified by byte index
// pairs within the input string: result[2*n:2*n+1] identifies the indexes of
// pairs within the input string: result[2*n:2*n+2] identifies the indexes of
// the nth submatch. The pair for n==0 identifies the match of the entire
// expression. If 'Index' is not present, the match is identified by the text
// of the match/submatch. If an index is negative or text is nil, it means that
@ -62,7 +67,6 @@
// before returning.
//
// (There are a few other methods that do not match this pattern.)
//
package regexp
import (
@ -278,7 +282,11 @@ func minInputLen(re *syntax.Regexp) int {
case syntax.OpLiteral:
l := 0
for _, r := range re.Rune {
l += utf8.RuneLen(r)
if r == utf8.RuneError {
l++
} else {
l += utf8.RuneLen(r)
}
}
return l
case syntax.OpCapture, syntax.OpPlus:
@ -821,11 +829,12 @@ func (re *Regexp) allMatches(s string, b []byte, n int, deliver func([]int)) {
accept = false
}
var width int
// TODO: use step()
if b == nil {
_, width = utf8.DecodeRuneInString(s[pos:end])
is := inputString{str: s}
_, width = is.step(pos)
} else {
_, width = utf8.DecodeRune(b[pos:end])
ib := inputBytes{str: b}
_, width = ib.step(pos)
}
if width > 0 {
pos += width
@ -956,23 +965,22 @@ func (re *Regexp) ExpandString(dst []byte, template string, src string, match []
func (re *Regexp) expand(dst []byte, template string, bsrc []byte, src string, match []int) []byte {
for len(template) > 0 {
i := strings.Index(template, "$")
if i < 0 {
before, after, ok := strings.Cut(template, "$")
if !ok {
break
}
dst = append(dst, template[:i]...)
template = template[i:]
if len(template) > 1 && template[1] == '$' {
dst = append(dst, before...)
template = after
if template != "" && template[0] == '$' {
// Treat $$ as $.
dst = append(dst, '$')
template = template[2:]
template = template[1:]
continue
}
name, num, rest, ok := extract(template)
if !ok {
// Malformed; treat $ as raw text.
dst = append(dst, '$')
template = template[1:]
continue
}
template = rest
@ -1001,17 +1009,16 @@ func (re *Regexp) expand(dst []byte, template string, bsrc []byte, src string, m
return dst
}
// extract returns the name from a leading "$name" or "${name}" in str.
// extract returns the name from a leading "name" or "{name}" in str.
// (The $ has already been removed by the caller.)
// If it is a number, extract returns num set to that number; otherwise num = -1.
func extract(str string) (name string, num int, rest string, ok bool) {
if len(str) < 2 || str[0] != '$' {
if str == "" {
return
}
brace := false
if str[1] == '{' {
if str[0] == '{' {
brace = true
str = str[2:]
} else {
str = str[1:]
}
i := 0
@ -1268,13 +1275,15 @@ func (re *Regexp) FindAllStringSubmatchIndex(s string, n int) [][]int {
// that contains no metacharacters, it is equivalent to strings.SplitN.
//
// Example:
// s := regexp.MustCompile("a*").Split("abaabaccadaaae", 5)
// // s: ["", "b", "b", "c", "cadaaae"]
//
// s := regexp.MustCompile("a*").Split("abaabaccadaaae", 5)
// // s: ["", "b", "b", "c", "cadaaae"]
//
// The count determines the number of substrings to return:
// n > 0: at most n substrings; the last substring will be the unsplit remainder.
// n == 0: the result is nil (zero substrings)
// n < 0: all substrings
//
// n > 0: at most n substrings; the last substring will be the unsplit remainder.
// n == 0: the result is nil (zero substrings)
// n < 0: all substrings
func (re *Regexp) Split(s string, n int) []string {
if n == 0 {

@ -9,123 +9,132 @@ Package syntax parses regular expressions into parse trees and compiles
parse trees into programs. Most clients of regular expressions will use the
facilities of package regexp (such as Compile and Match) instead of this package.
Syntax
# Syntax
The regular expression syntax understood by this package when parsing with the Perl flag is as follows.
Parts of the syntax can be disabled by passing alternate flags to Parse.
Single characters:
. any character, possibly including newline (flag s=true)
[xyz] character class
[^xyz] negated character class
\d Perl character class
\D negated Perl character class
[[:alpha:]] ASCII character class
[[:^alpha:]] negated ASCII character class
\pN Unicode character class (one-letter name)
\p{Greek} Unicode character class
\PN negated Unicode character class (one-letter name)
\P{Greek} negated Unicode character class
. any character, possibly including newline (flag s=true)
[xyz] character class
[^xyz] negated character class
\d Perl character class
\D negated Perl character class
[[:alpha:]] ASCII character class
[[:^alpha:]] negated ASCII character class
\pN Unicode character class (one-letter name)
\p{Greek} Unicode character class
\PN negated Unicode character class (one-letter name)
\P{Greek} negated Unicode character class
Composites:
xy x followed by y
x|y x or y (prefer x)
xy x followed by y
x|y x or y (prefer x)
Repetitions:
x* zero or more x, prefer more
x+ one or more x, prefer more
x? zero or one x, prefer one
x{n,m} n or n+1 or ... or m x, prefer more
x{n,} n or more x, prefer more
x{n} exactly n x
x*? zero or more x, prefer fewer
x+? one or more x, prefer fewer
x?? zero or one x, prefer zero
x{n,m}? n or n+1 or ... or m x, prefer fewer
x{n,}? n or more x, prefer fewer
x{n}? exactly n x
x* zero or more x, prefer more
x+ one or more x, prefer more
x? zero or one x, prefer one
x{n,m} n or n+1 or ... or m x, prefer more
x{n,} n or more x, prefer more
x{n} exactly n x
x*? zero or more x, prefer fewer
x+? one or more x, prefer fewer
x?? zero or one x, prefer zero
x{n,m}? n or n+1 or ... or m x, prefer fewer
x{n,}? n or more x, prefer fewer
x{n}? exactly n x
Implementation restriction: The counting forms x{n,m}, x{n,}, and x{n}
reject forms that create a minimum or maximum repetition count above 1000.
Unlimited repetitions are not subject to this restriction.
Grouping:
(re) numbered capturing group (submatch)
(?P<name>re) named & numbered capturing group (submatch)
(?:re) non-capturing group
(?flags) set flags within current group; non-capturing
(?flags:re) set flags during re; non-capturing
Flag syntax is xyz (set) or -xyz (clear) or xy-z (set xy, clear z). The flags are:
(re) numbered capturing group (submatch)
(?P<name>re) named & numbered capturing group (submatch)
(?:re) non-capturing group
(?flags) set flags within current group; non-capturing
(?flags:re) set flags during re; non-capturing
Flag syntax is xyz (set) or -xyz (clear) or xy-z (set xy, clear z). The flags are:
i case-insensitive (default false)
m multi-line mode: ^ and $ match begin/end line in addition to begin/end text (default false)
s let . match \n (default false)
U ungreedy: swap meaning of x* and x*?, x+ and x+?, etc (default false)
i case-insensitive (default false)
m multi-line mode: ^ and $ match begin/end line in addition to begin/end text (default false)
s let . match \n (default false)
U ungreedy: swap meaning of x* and x*?, x+ and x+?, etc (default false)
Empty strings:
^ at beginning of text or line (flag m=true)
$ at end of text (like \z not \Z) or line (flag m=true)
\A at beginning of text
\b at ASCII word boundary (\w on one side and \W, \A, or \z on the other)
\B not at ASCII word boundary
\z at end of text
^ at beginning of text or line (flag m=true)
$ at end of text (like \z not \Z) or line (flag m=true)
\A at beginning of text
\b at ASCII word boundary (\w on one side and \W, \A, or \z on the other)
\B not at ASCII word boundary
\z at end of text
Escape sequences:
\a bell (== \007)
\f form feed (== \014)
\t horizontal tab (== \011)
\n newline (== \012)
\r carriage return (== \015)
\v vertical tab character (== \013)
\* literal *, for any punctuation character *
\123 octal character code (up to three digits)
\x7F hex character code (exactly two digits)
\x{10FFFF} hex character code
\Q...\E literal text ... even if ... has punctuation
\a bell (== \007)
\f form feed (== \014)
\t horizontal tab (== \011)
\n newline (== \012)
\r carriage return (== \015)
\v vertical tab character (== \013)
\* literal *, for any punctuation character *
\123 octal character code (up to three digits)
\x7F hex character code (exactly two digits)
\x{10FFFF} hex character code
\Q...\E literal text ... even if ... has punctuation
Character class elements:
x single character
A-Z character range (inclusive)
\d Perl character class
[:foo:] ASCII character class foo
\p{Foo} Unicode character class Foo
\pF Unicode character class F (one-letter name)
x single character
A-Z character range (inclusive)
\d Perl character class
[:foo:] ASCII character class foo
\p{Foo} Unicode character class Foo
\pF Unicode character class F (one-letter name)
Named character classes as character class elements:
[\d] digits (== \d)
[^\d] not digits (== \D)
[\D] not digits (== \D)
[^\D] not not digits (== \d)
[[:name:]] named ASCII class inside character class (== [:name:])
[^[:name:]] named ASCII class inside negated character class (== [:^name:])
[\p{Name}] named Unicode property inside character class (== \p{Name})
[^\p{Name}] named Unicode property inside negated character class (== \P{Name})
[\d] digits (== \d)
[^\d] not digits (== \D)
[\D] not digits (== \D)
[^\D] not not digits (== \d)
[[:name:]] named ASCII class inside character class (== [:name:])
[^[:name:]] named ASCII class inside negated character class (== [:^name:])
[\p{Name}] named Unicode property inside character class (== \p{Name})
[^\p{Name}] named Unicode property inside negated character class (== \P{Name})
Perl character classes (all ASCII-only):
\d digits (== [0-9])
\D not digits (== [^0-9])
\s whitespace (== [\t\n\f\r ])
\S not whitespace (== [^\t\n\f\r ])
\w word characters (== [0-9A-Za-z_])
\W not word characters (== [^0-9A-Za-z_])
\d digits (== [0-9])
\D not digits (== [^0-9])
\s whitespace (== [\t\n\f\r ])
\S not whitespace (== [^\t\n\f\r ])
\w word characters (== [0-9A-Za-z_])
\W not word characters (== [^0-9A-Za-z_])
ASCII character classes:
[[:alnum:]] alphanumeric (== [0-9A-Za-z])
[[:alpha:]] alphabetic (== [A-Za-z])
[[:ascii:]] ASCII (== [\x00-\x7F])
[[:blank:]] blank (== [\t ])
[[:cntrl:]] control (== [\x00-\x1F\x7F])
[[:digit:]] digits (== [0-9])
[[:graph:]] graphical (== [!-~] == [A-Za-z0-9!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~])
[[:lower:]] lower case (== [a-z])
[[:print:]] printable (== [ -~] == [ [:graph:]])
[[:punct:]] punctuation (== [!-/:-@[-`{-~])
[[:space:]] whitespace (== [\t\n\v\f\r ])
[[:upper:]] upper case (== [A-Z])
[[:word:]] word characters (== [0-9A-Za-z_])
[[:xdigit:]] hex digit (== [0-9A-Fa-f])
[[:alnum:]] alphanumeric (== [0-9A-Za-z])
[[:alpha:]] alphabetic (== [A-Za-z])
[[:ascii:]] ASCII (== [\x00-\x7F])
[[:blank:]] blank (== [\t ])
[[:cntrl:]] control (== [\x00-\x1F\x7F])
[[:digit:]] digits (== [0-9])
[[:graph:]] graphical (== [!-~] == [A-Za-z0-9!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~])
[[:lower:]] lower case (== [a-z])
[[:print:]] printable (== [ -~] == [ [:graph:]])
[[:punct:]] punctuation (== [!-/:-@[-`{-~])
[[:space:]] whitespace (== [\t\n\v\f\r ])
[[:upper:]] upper case (== [A-Z])
[[:word:]] word characters (== [0-9A-Za-z_])
[[:xdigit:]] hex digit (== [0-9A-Fa-f])
Unicode character classes are those in unicode.Categories and unicode.Scripts.
*/

@ -43,6 +43,7 @@ const (
ErrMissingRepeatArgument ErrorCode = "missing argument to repetition operator"
ErrTrailingBackslash ErrorCode = "trailing backslash at end of expression"
ErrUnexpectedParen ErrorCode = "unexpected )"
ErrNestingDepth ErrorCode = "expression nests too deeply"
)
func (e ErrorCode) String() string {
@ -90,15 +91,49 @@ const (
// until we've allocated at least maxHeight Regexp structures.
const maxHeight = 1000
// maxSize is the maximum size of a compiled regexp in Insts.
// It too is somewhat arbitrarily chosen, but the idea is to be large enough
// to allow significant regexps while at the same time small enough that
// the compiled form will not take up too much memory.
// 128 MB is enough for a 3.3 million Inst structures, which roughly
// corresponds to a 3.3 MB regexp.
const (
maxSize = 128 << 20 / instSize
instSize = 5 * 8 // byte, 2 uint32, slice is 5 64-bit words
)
// maxRunes is the maximum number of runes allowed in a regexp tree
// counting the runes in all the nodes.
// Ignoring character classes p.numRunes is always less than the length of the regexp.
// Character classes can make it much larger: each \pL adds 1292 runes.
// 128 MB is enough for 32M runes, which is over 26k \pL instances.
// Note that repetitions do not make copies of the rune slices,
// so \pL{1000} is only one rune slice, not 1000.
// We could keep a cache of character classes we've seen,
// so that all the \pL we see use the same rune list,
// but that doesn't remove the problem entirely:
// consider something like [\pL01234][\pL01235][\pL01236]...[\pL^&*()].
// And because the Rune slice is exposed directly in the Regexp,
// there is not an opportunity to change the representation to allow
// partial sharing between different character classes.
// So the limit is the best we can do.
const (
maxRunes = 128 << 20 / runeSize
runeSize = 4 // rune is int32
)
type parser struct {
flags Flags // parse mode flags
stack []*Regexp // stack of parsed expressions
free *Regexp
numCap int // number of capturing groups seen
wholeRegexp string
tmpClass []rune // temporary char class work space
numRegexp int // number of regexps allocated
height map[*Regexp]int // regexp height for height limit check
tmpClass []rune // temporary char class work space
numRegexp int // number of regexps allocated
numRunes int // number of runes in char classes
repeats int64 // product of all repetitions seen
height map[*Regexp]int // regexp height, for height limit check
size map[*Regexp]int64 // regexp compiled size, for size limit check
}
func (p *parser) newRegexp(op Op) *Regexp {
@ -122,6 +157,104 @@ func (p *parser) reuse(re *Regexp) {
p.free = re
}
func (p *parser) checkLimits(re *Regexp) {
if p.numRunes > maxRunes {
panic(ErrInternalError)
}
p.checkSize(re)
p.checkHeight(re)
}
func (p *parser) checkSize(re *Regexp) {
if p.size == nil {
// We haven't started tracking size yet.
// Do a relatively cheap check to see if we need to start.
// Maintain the product of all the repeats we've seen
// and don't track if the total number of regexp nodes
// we've seen times the repeat product is in budget.
if p.repeats == 0 {
p.repeats = 1
}
if re.Op == OpRepeat {
n := re.Max
if n == -1 {
n = re.Min
}
if n <= 0 {
n = 1
}
if int64(n) > maxSize/p.repeats {
p.repeats = maxSize
} else {
p.repeats *= int64(n)
}
}
if int64(p.numRegexp) < maxSize/p.repeats {
return
}
// We need to start tracking size.
// Make the map and belatedly populate it
// with info about everything we've constructed so far.
p.size = make(map[*Regexp]int64)
for _, re := range p.stack {
p.checkSize(re)
}
}
if p.calcSize(re, true) > maxSize {
panic(ErrInternalError)
}
}
func (p *parser) calcSize(re *Regexp, force bool) int64 {
if !force {
if size, ok := p.size[re]; ok {
return size
}
}
var size int64
switch re.Op {
case OpLiteral:
size = int64(len(re.Rune))
case OpCapture, OpStar:
// star can be 1+ or 2+; assume 2 pessimistically
size = 2 + p.calcSize(re.Sub[0], false)
case OpPlus, OpQuest:
size = 1 + p.calcSize(re.Sub[0], false)
case OpConcat:
for _, sub := range re.Sub {
size += p.calcSize(sub, false)
}
case OpAlternate:
for _, sub := range re.Sub {
size += p.calcSize(sub, false)
}
if len(re.Sub) > 1 {
size += int64(len(re.Sub)) - 1
}
case OpRepeat:
sub := p.calcSize(re.Sub[0], false)
if re.Max == -1 {
if re.Min == 0 {
size = 2 + sub // x*
} else {
size = 1 + int64(re.Min)*sub // xxx+
}
break
}
// x{2,5} = xx(x(x(x)?)?)?
size = int64(re.Max)*sub + int64(re.Max-re.Min)
}
if size < 1 {
size = 1
}
p.size[re] = size
return size
}
func (p *parser) checkHeight(re *Regexp) {
if p.numRegexp < maxHeight {
return
@ -133,7 +266,7 @@ func (p *parser) checkHeight(re *Regexp) {
}
}
if p.calcHeight(re, true) > maxHeight {
panic(ErrInternalError)
panic(ErrNestingDepth)
}
}
@ -158,6 +291,7 @@ func (p *parser) calcHeight(re *Regexp, force bool) int {
// push pushes the regexp re onto the parse stack and returns the regexp.
func (p *parser) push(re *Regexp) *Regexp {
p.numRunes += len(re.Rune)
if re.Op == OpCharClass && len(re.Rune) == 2 && re.Rune[0] == re.Rune[1] {
// Single rune.
if p.maybeConcat(re.Rune[0], p.flags&^FoldCase) {
@ -189,7 +323,7 @@ func (p *parser) push(re *Regexp) *Regexp {
}
p.stack = append(p.stack, re)
p.checkHeight(re)
p.checkLimits(re)
return re
}
@ -299,7 +433,7 @@ func (p *parser) repeat(op Op, min, max int, before, after, lastRepeat string) (
re.Sub = re.Sub0[:1]
re.Sub[0] = sub
p.stack[n-1] = re
p.checkHeight(re)
p.checkLimits(re)
if op == OpRepeat && (min >= 2 || max >= 2) && !repeatIsValid(re, 1000) {
return "", &Error{ErrInvalidRepeatSize, before[:len(before)-len(after)]}
@ -444,12 +578,16 @@ func (p *parser) collapse(subs []*Regexp, op Op) *Regexp {
// frees (passes to p.reuse) any removed *Regexps.
//
// For example,
// ABC|ABD|AEF|BCX|BCY
//
// ABC|ABD|AEF|BCX|BCY
//
// simplifies by literal prefix extraction to
// A(B(C|D)|EF)|BC(X|Y)
//
// A(B(C|D)|EF)|BC(X|Y)
//
// which simplifies by character class introduction to
// A(B[CD]|EF)|BC[XY]
//
// A(B[CD]|EF)|BC[XY]
func (p *parser) factor(sub []*Regexp) []*Regexp {
if len(sub) < 2 {
return sub
@ -503,6 +641,7 @@ func (p *parser) factor(sub []*Regexp) []*Regexp {
for j := start; j < i; j++ {
sub[j] = p.removeLeadingString(sub[j], len(str))
p.checkLimits(sub[j])
}
suffix := p.collapse(sub[start:i], OpAlternate) // recurse
@ -560,6 +699,7 @@ func (p *parser) factor(sub []*Regexp) []*Regexp {
for j := start; j < i; j++ {
reuse := j != start // prefix came from sub[start]
sub[j] = p.removeLeadingRegexp(sub[j], reuse)
p.checkLimits(sub[j])
}
suffix := p.collapse(sub[start:i], OpAlternate) // recurse
@ -757,8 +897,10 @@ func parse(s string, flags Flags) (_ *Regexp, err error) {
panic(r)
case nil:
// ok
case ErrInternalError:
case ErrInternalError: // too big
err = &Error{Code: ErrInternalError, Expr: s}
case ErrNestingDepth:
err = &Error{Code: ErrNestingDepth, Expr: s}
}
}()
@ -892,13 +1034,7 @@ func parse(s string, flags Flags) (_ *Regexp, err error) {
case 'Q':
// \Q ... \E: the ... is always literals
var lit string
if i := strings.Index(t, `\E`); i < 0 {
lit = t[2:]
t = ""
} else {
lit = t[2:i]
t = t[i+2:]
}
lit, t, _ = strings.Cut(t[2:], `\E`)
for lit != "" {
c, rest, err := nextRune(lit)
if err != nil {

@ -8,6 +8,7 @@ import (
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
// Compiled program.
@ -101,7 +102,7 @@ func EmptyOpContext(r1, r2 rune) EmptyOp {
return op
}
// IsWordChar reports whether r is consider a ``word character''
// IsWordChar reports whether r is considered a “word character”
// during the evaluation of the \b and \B zero-width assertions.
// These assertions are ASCII-only: the word characters are [A-Za-z0-9_].
func IsWordChar(r rune) bool {
@ -171,7 +172,7 @@ func (p *Prog) PrefixAndCase() (prefix string, complete bool, foldCase bool) {
// Have prefix; gather characters.
var buf strings.Builder
foldCase = (Flags(i.Arg)&FoldCase != 0)
for i.op() == InstRune && len(i.Rune) == 1 && (Flags(i.Arg)&FoldCase != 0) == foldCase {
for i.op() == InstRune && len(i.Rune) == 1 && (Flags(i.Arg)&FoldCase != 0) == foldCase && i.Rune[0] != utf8.RuneError {
buf.WriteRune(i.Rune[0])
i = p.skipNop(i.Out)
}

@ -561,7 +561,7 @@ github.com/grafana/go-gelf/v2/gelf
# github.com/grafana/groupcache_exporter v0.0.0-20220629095919-59a8c6428a43
## explicit; go 1.17
github.com/grafana/groupcache_exporter
# github.com/grafana/regexp v0.0.0-20220304100321-149c8afcd6cb
# github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6
## explicit; go 1.17
github.com/grafana/regexp
github.com/grafana/regexp/syntax

Loading…
Cancel
Save