loki/pkg/chunkenc/testdata/testdata.go

package testdata

import "strings"

// LogString returns a test log line. Returns the same line for the same index.
func LogString(index int64) string {
	if index > int64(len(Logs)-1) {
		index = index % int64(len(Logs))
	}
	return Logs[index]
}

var LogsBytes [][]byte

func init() {
	LogsBytes = make([][]byte, len(Logs))
	for i, l := range Logs {
		LogsBytes[i] = []byte(l)
	}
}

var Logs = strings.Split(`level=info ts=2019-12-12T15:00:08.325Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576130400000 maxt=1576152000000 ulid=01DVX9ZHNM71GRCJS7M34Q0EV7 sources="[01DVWNC6NWY1A60AZV3Z6DGS65 01DVWW7XXX75GHA6ZDTD170CSZ 01DVX33N5W86CWJJVRPAVXJRWJ]" duration=2.897213221s
level=info ts=2019-12-12T15:00:08.296Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576130400000 maxt=1576152000000 ulid=01DVX9ZHQRVN42AF196NYJ9C4C sources="[01DVWNC6NSPJRCSBZ4QD3SXS66 01DVWW7XY69Y4YT09HR0RSR8KY 01DVX33N5SMVPB1TMD9J1M8GGK]" duration=2.800759388s
level=info ts=2019-12-12T15:00:05.285Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1037 last=1039 duration=3.030078405s
level=info ts=2019-12-12T15:00:05.225Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1037 last=1039 duration=3.019791992s
level=info ts=2019-12-12T15:00:02.255Z caller=head.go:596 component=tsdb msg="head GC completed" duration=125.980176ms
level=info ts=2019-12-12T15:00:02.206Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.111334ms
level=info ts=2019-12-12T15:00:01.874Z caller=compact.go:496 component=tsdb msg="write block" mint=1576152000000 maxt=1576159200000 ulid=01DVX9ZCE8WZCTQJWSYDGHVQV8 duration=1.801853505s
level=info ts=2019-12-12T15:00:01.854Z caller=compact.go:496 component=tsdb msg="write block" mint=1576152000000 maxt=1576159200000 ulid=01DVX9ZCDWEBXRYWA7585TN2RV duration=1.794588392s
level=info ts=2019-12-12T13:00:05.461Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1034 last=1036 duration=3.044019343s
level=info ts=2019-12-12T13:00:05.332Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1034 last=1036 duration=3.040243488s
level=info ts=2019-12-12T13:00:02.417Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.883109ms
level=info ts=2019-12-12T13:00:02.291Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.278558ms
level=info ts=2019-12-12T13:00:02.048Z caller=compact.go:496 component=tsdb msg="write block" mint=1576144800000 maxt=1576152000000 ulid=01DVX33N5W86CWJJVRPAVXJRWJ duration=1.987867109s
level=info ts=2019-12-12T13:00:01.914Z caller=compact.go:496 component=tsdb msg="write block" mint=1576144800000 maxt=1576152000000 ulid=01DVX33N5SMVPB1TMD9J1M8GGK duration=1.856432758s
level=info ts=2019-12-12T12:58:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-12T12:52:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-12T11:00:05.320Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1031 last=1033 duration=2.999621843s
level=info ts=2019-12-12T11:00:05.315Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1031 last=1033 duration=2.962560692s
level=info ts=2019-12-12T11:00:02.352Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.600701ms
level=info ts=2019-12-12T11:00:02.321Z caller=head.go:596 component=tsdb msg="head GC completed" duration=134.547131ms
level=info ts=2019-12-12T11:00:01.975Z caller=compact.go:496 component=tsdb msg="write block" mint=1576137600000 maxt=1576144800000 ulid=01DVWW7XY69Y4YT09HR0RSR8KY duration=1.905948839s
level=info ts=2019-12-12T11:00:01.889Z caller=compact.go:496 component=tsdb msg="write block" mint=1576137600000 maxt=1576144800000 ulid=01DVWW7XXX75GHA6ZDTD170CSZ duration=1.828298188s
level=info ts=2019-12-12T10:55:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-12T10:49:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-12T10:33:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-12T10:25:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-12T10:21:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-12T10:14:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-12T09:00:16.465Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576130400000 ulid=01DVWNCFJRNW4RP8C56D4QNRXH sources="[01DVVC60FYTRXZ9457XT10Y7AH 01DVW0S6A5HFTYBYD34SGAZJSR 01DVWNCC9SYJDQP0Y2RXK8XJC9]" duration=7.289011992s
level=info ts=2019-12-12T09:00:15.812Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576130400000 ulid=01DVWNCF9JNSMSKZHW8STXQARA sources="[01DVVC60DBGMXD5DXR6Y5XWNXF 01DVW0S67R7JFBFTFWMNVS8YR3 01DVWNCC599NDRZWRRSZF4XGHF]" duration=6.930550254s
level=info ts=2019-12-12T09:00:08.717Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576108800000 maxt=1576130400000 ulid=01DVWNCC9SYJDQP0Y2RXK8XJC9 sources="[01DVW0S0XW63CVRA3EPRSC8NWQ 01DVW7MR5W18322RVFY6WM9GR2 01DVWEGFDW0C09KSCRQ2F8DGN3]" duration=2.900180235s
level=info ts=2019-12-12T09:00:08.440Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576108800000 maxt=1576130400000 ulid=01DVWNCC599NDRZWRRSZF4XGHF sources="[01DVW0S0XS1SQQQK3CQYCHN9HV 01DVW7MR5ZN3K38ZHBJ243HDZJ 01DVWEGFE0DGKKDG4V9AGAPPBQ]" duration=2.767053211s
level=info ts=2019-12-12T09:00:05.604Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1028 last=1030 duration=2.998418095s
level=info ts=2019-12-12T09:00:05.470Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1028 last=1030 duration=3.008684806s
level=info ts=2019-12-12T09:00:02.606Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.82085ms
level=info ts=2019-12-12T09:00:02.461Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.770206ms
level=info ts=2019-12-12T09:00:01.995Z caller=compact.go:496 component=tsdb msg="write block" mint=1576130400000 maxt=1576137600000 ulid=01DVWNC6NWY1A60AZV3Z6DGS65 duration=1.934602237s
level=info ts=2019-12-12T09:00:01.960Z caller=compact.go:496 component=tsdb msg="write block" mint=1576130400000 maxt=1576137600000 ulid=01DVWNC6NSPJRCSBZ4QD3SXS66 duration=1.902822647s
level=info ts=2019-12-12T08:59:54.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-12T08:54:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-12T08:12:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-12T08:05:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-12T07:00:05.421Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1025 last=1027 duration=3.037037204s
level=info ts=2019-12-12T07:00:05.263Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1025 last=1027 duration=2.984857831s
level=info ts=2019-12-12T07:00:02.383Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.79721ms
level=info ts=2019-12-12T07:00:02.278Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.228064ms
level=info ts=2019-12-12T07:00:02.052Z caller=compact.go:496 component=tsdb msg="write block" mint=1576123200000 maxt=1576130400000 ulid=01DVWEGFE0DGKKDG4V9AGAPPBQ duration=1.987940522s
level=info ts=2019-12-12T07:00:01.927Z caller=compact.go:496 component=tsdb msg="write block" mint=1576123200000 maxt=1576130400000 ulid=01DVWEGFDW0C09KSCRQ2F8DGN3 duration=1.866990386s
level=info ts=2019-12-12T05:00:05.355Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1022 last=1024 duration=3.046145151s
level=info ts=2019-12-12T05:00:05.309Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1022 last=1024 duration=3.019897535s
level=info ts=2019-12-12T05:00:02.309Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.294946ms
level=info ts=2019-12-12T05:00:02.289Z caller=head.go:596 component=tsdb msg="head GC completed" duration=145.150847ms
level=info ts=2019-12-12T05:00:01.939Z caller=compact.go:496 component=tsdb msg="write block" mint=1576116000000 maxt=1576123200000 ulid=01DVW7MR5ZN3K38ZHBJ243HDZJ duration=1.875204968s
level=info ts=2019-12-12T05:00:01.813Z caller=compact.go:496 component=tsdb msg="write block" mint=1576116000000 maxt=1576123200000 ulid=01DVW7MR5W18322RVFY6WM9GR2 duration=1.753345795s
level=info ts=2019-12-12T04:38:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-12T04:33:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-12T04:00:54.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-12T03:56:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-12T03:00:08.433Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576087200000 maxt=1576108800000 ulid=01DVW0S6A5HFTYBYD34SGAZJSR sources="[01DVVC5V5WESMMH77FZVCJ80Q8 01DVVK1JDWNVFGWS4JPY2K4CAS 01DVVSX9NWR5V8SSJAPKQ2TCTH]" duration=2.860812672s
level=info ts=2019-12-12T03:00:08.279Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576087200000 maxt=1576108800000 ulid=01DVW0S67R7JFBFTFWMNVS8YR3 sources="[01DVVC5V6145SMRFE0WR0P3YTQ 01DVVK1JE1SSYY4EKS4HAT4SK3 01DVVSX9NRE3DWK67A2J17BE0T]" duration=2.782760638s
level=info ts=2019-12-12T03:00:05.372Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1019 last=1021 duration=2.990754756s
level=info ts=2019-12-12T03:00:05.289Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1019 last=1021 duration=3.007795347s
level=info ts=2019-12-12T03:00:02.381Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.007667ms
level=info ts=2019-12-12T03:00:02.282Z caller=head.go:596 component=tsdb msg="head GC completed" duration=133.138336ms
level=info ts=2019-12-12T03:00:01.987Z caller=compact.go:496 component=tsdb msg="write block" mint=1576108800000 maxt=1576116000000 ulid=01DVW0S0XW63CVRA3EPRSC8NWQ duration=1.927367458s
level=info ts=2019-12-12T03:00:01.906Z caller=compact.go:496 component=tsdb msg="write block" mint=1576108800000 maxt=1576116000000 ulid=01DVW0S0XS1SQQQK3CQYCHN9HV duration=1.84874308s
level=info ts=2019-12-12T02:39:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-12T02:33:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-12T01:00:05.500Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1016 last=1018 duration=3.027246961s
level=info ts=2019-12-12T01:00:05.265Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1016 last=1018 duration=2.989822576s
level=info ts=2019-12-12T01:00:02.473Z caller=head.go:596 component=tsdb msg="head GC completed" duration=124.134851ms
level=info ts=2019-12-12T01:00:02.275Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.268006ms
level=info ts=2019-12-12T01:00:02.092Z caller=compact.go:496 component=tsdb msg="write block" mint=1576101600000 maxt=1576108800000 ulid=01DVVSX9NRE3DWK67A2J17BE0T duration=2.035218414s
level=info ts=2019-12-12T01:00:01.907Z caller=compact.go:496 component=tsdb msg="write block" mint=1576101600000 maxt=1576108800000 ulid=01DVVSX9NWR5V8SSJAPKQ2TCTH duration=1.847566214s
level=info ts=2019-12-11T23:00:05.552Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1013 last=1015 duration=3.042911717s
level=info ts=2019-12-11T23:00:05.255Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1013 last=1015 duration=3.007686626s
level=info ts=2019-12-11T23:00:02.509Z caller=head.go:596 component=tsdb msg="head GC completed" duration=135.735201ms
level=info ts=2019-12-11T23:00:02.247Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.374582ms
level=info ts=2019-12-11T23:00:02.154Z caller=compact.go:496 component=tsdb msg="write block" mint=1576094400000 maxt=1576101600000 ulid=01DVVK1JE1SSYY4EKS4HAT4SK3 duration=2.088724625s
level=info ts=2019-12-11T23:00:01.873Z caller=compact.go:496 component=tsdb msg="write block" mint=1576094400000 maxt=1576101600000 ulid=01DVVK1JDWNVFGWS4JPY2K4CAS duration=1.813033164s
level=info ts=2019-12-11T21:00:08.427Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576087200000 ulid=01DVVC60FYTRXZ9457XT10Y7AH sources="[01DVTQJNDXSY7N5V60ZX7X1C3J 01DVTYECNW5T3AHHB2EXATYFMJ 01DVV5A3XWVRTNS7G7BBDQ9G2W]" duration=2.925663083s
level=info ts=2019-12-11T21:00:08.281Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576087200000 ulid=01DVVC60DBGMXD5DXR6Y5XWNXF sources="[01DVTQJNDRV9NDCK9H2BCH04R0 01DVTYECNS4AZH3ZMCER87DYWG 01DVV5A3XRVMTB2E7V3MZ6RGCA]" duration=2.862756811s
level=info ts=2019-12-11T21:00:05.288Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1010 last=1012 duration=2.998716456s
level=info ts=2019-12-11T21:00:05.204Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1010 last=1012 duration=3.013679702s
level=info ts=2019-12-11T21:00:02.289Z caller=head.go:596 component=tsdb msg="head GC completed" duration=124.171081ms
level=info ts=2019-12-11T21:00:02.190Z caller=head.go:596 component=tsdb msg="head GC completed" duration=114.925741ms
level=info ts=2019-12-11T21:00:01.942Z caller=compact.go:496 component=tsdb msg="write block" mint=1576087200000 maxt=1576094400000 ulid=01DVVC5V5WESMMH77FZVCJ80Q8 duration=1.881893506s
level=info ts=2019-12-11T21:00:01.837Z caller=compact.go:496 component=tsdb msg="write block" mint=1576087200000 maxt=1576094400000 ulid=01DVVC5V6145SMRFE0WR0P3YTQ duration=1.772164011s
level=info ts=2019-12-11T19:00:05.276Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1007 last=1009 duration=3.031727362s
level=info ts=2019-12-11T19:00:05.222Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1007 last=1009 duration=3.003072336s
level=info ts=2019-12-11T19:00:02.244Z caller=head.go:596 component=tsdb msg="head GC completed" duration=125.675247ms
level=info ts=2019-12-11T19:00:02.219Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.466308ms
level=info ts=2019-12-11T19:00:01.888Z caller=compact.go:496 component=tsdb msg="write block" mint=1576080000000 maxt=1576087200000 ulid=01DVV5A3XRVMTB2E7V3MZ6RGCA duration=1.832443683s
level=info ts=2019-12-11T19:00:01.845Z caller=compact.go:496 component=tsdb msg="write block" mint=1576080000000 maxt=1576087200000 ulid=01DVV5A3XWVRTNS7G7BBDQ9G2W duration=1.784935995s
level=info ts=2019-12-11T18:31:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T18:24:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T17:00:05.233Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1004 last=1006 duration=3.008189996s
level=info ts=2019-12-11T17:00:05.223Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1004 last=1006 duration=2.97892s
level=info ts=2019-12-11T17:00:02.244Z caller=head.go:596 component=tsdb msg="head GC completed" duration=132.385042ms
level=info ts=2019-12-11T17:00:02.225Z caller=head.go:596 component=tsdb msg="head GC completed" duration=125.500534ms
level=info ts=2019-12-11T17:00:01.870Z caller=compact.go:496 component=tsdb msg="write block" mint=1576072800000 maxt=1576080000000 ulid=01DVTYECNW5T3AHHB2EXATYFMJ duration=1.810447322s
level=info ts=2019-12-11T17:00:01.870Z caller=compact.go:496 component=tsdb msg="write block" mint=1576072800000 maxt=1576080000000 ulid=01DVTYECNS4AZH3ZMCER87DYWG duration=1.813347748s
level=info ts=2019-12-11T15:00:16.297Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576065600000 ulid=01DVTQJY4WBY96QVV4XQJTR2JC sources="[01DVSECF6Q4JXFDGMFQB3J1Z9E 01DVT2ZN0DMXXJJDHKS0M8JWMS 01DVTQJTX0GZ1S7J51CN1RJNQX]" duration=7.308935842s
level=info ts=2019-12-11T15:00:15.941Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576065600000 ulid=01DVTQJY58270MEPGVGGDZZRKJ sources="[01DVSECF6J4NZRNHABZ2MSG7V7 01DVT2ZN4RB65KG77XPHPNVSAM 01DVTQJTYGFS18MWME9Z2NFJSW]" duration=6.941637414s
level=info ts=2019-12-11T15:00:08.544Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576044000000 maxt=1576065600000 ulid=01DVTQJTX0GZ1S7J51CN1RJNQX sources="[01DVT2ZFNYB7DEH57ZX4HW2DAV 01DVT9V6XW9ENV15NHKR20T9B4 01DVTGPY5WTBSSEQ37JQ2VPCTQ]" duration=2.880290482s
level=info ts=2019-12-11T15:00:08.541Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576044000000 maxt=1576065600000 ulid=01DVTQJTYGFS18MWME9Z2NFJSW sources="[01DVT2ZFP3R7RB9H6BS3JVAMXJ 01DVT9V6Y21E8YXRKNGA9RPB7D 01DVTGPY5XGARMV8B8VBWQ23W3]" duration=2.829184147s
level=info ts=2019-12-11T15:00:05.505Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1001 last=1003 duration=3.006477625s
level=info ts=2019-12-11T15:00:05.452Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1001 last=1003 duration=2.990895181s
level=info ts=2019-12-11T15:00:02.498Z caller=head.go:596 component=tsdb msg="head GC completed" duration=129.237566ms
level=info ts=2019-12-11T15:00:02.461Z caller=head.go:596 component=tsdb msg="head GC completed" duration=129.961097ms
level=info ts=2019-12-11T15:00:02.022Z caller=compact.go:496 component=tsdb msg="write block" mint=1576065600000 maxt=1576072800000 ulid=01DVTQJNDRV9NDCK9H2BCH04R0 duration=1.96598488s
level=info ts=2019-12-11T15:00:01.933Z caller=compact.go:496 component=tsdb msg="write block" mint=1576065600000 maxt=1576072800000 ulid=01DVTQJNDXSY7N5V60ZX7X1C3J duration=1.871872199s
level=info ts=2019-12-11T14:15:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T14:07:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T13:18:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T13:12:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T13:00:05.395Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=998 last=1000 duration=3.010358861s
level=info ts=2019-12-11T13:00:05.249Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=998 last=1000 duration=3.032196282s
level=info ts=2019-12-11T13:00:02.385Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.568186ms
level=info ts=2019-12-11T13:00:02.217Z caller=head.go:596 component=tsdb msg="head GC completed" duration=136.017788ms
level=info ts=2019-12-11T13:00:02.021Z caller=compact.go:496 component=tsdb msg="write block" mint=1576058400000 maxt=1576065600000 ulid=01DVTGPY5XGARMV8B8VBWQ23W3 duration=1.959903s
level=info ts=2019-12-11T13:00:01.865Z caller=compact.go:496 component=tsdb msg="write block" mint=1576058400000 maxt=1576065600000 ulid=01DVTGPY5WTBSSEQ37JQ2VPCTQ duration=1.805149859s
level=info ts=2019-12-11T11:46:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T11:39:44.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T11:35:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T11:26:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T11:15:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T11:06:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T11:01:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T11:00:05.591Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=995 last=997 duration=3.063684941s
level=info ts=2019-12-11T11:00:05.297Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=995 last=997 duration=3.051047495s
level=info ts=2019-12-11T11:00:02.527Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.530749ms
level=info ts=2019-12-11T11:00:02.246Z caller=head.go:596 component=tsdb msg="head GC completed" duration=123.08975ms
level=info ts=2019-12-11T11:00:02.096Z caller=compact.go:496 component=tsdb msg="write block" mint=1576051200000 maxt=1576058400000 ulid=01DVT9V6Y21E8YXRKNGA9RPB7D duration=2.029825916s
level=info ts=2019-12-11T11:00:01.819Z caller=compact.go:496 component=tsdb msg="write block" mint=1576051200000 maxt=1576058400000 ulid=01DVT9V6XW9ENV15NHKR20T9B4 duration=1.7583013s
level=info ts=2019-12-11T10:54:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T10:46:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T10:39:44.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T10:34:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T10:26:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T10:18:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T10:12:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T09:56:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T09:48:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T09:00:08.553Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576022400000 maxt=1576044000000 ulid=01DVT2ZN4RB65KG77XPHPNVSAM sources="[01DVSEC9XXK1J3B0186KYQECZT 01DVSN815STR0D0B8245RWNF13 01DVSW3RDRKTJVEZWGYE07XBXE]" duration=2.896352595s
level=info ts=2019-12-11T09:00:08.407Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576022400000 maxt=1576044000000 ulid=01DVT2ZN0DMXXJJDHKS0M8JWMS sources="[01DVSEC9XW0W8V42SPRR0YMM0X 01DVSN815W8YTW3DPQTJVRNTS4 01DVSW3RDWE1WHSM8AEW0ARA3S]" duration=2.890101974s
level=info ts=2019-12-11T09:00:05.444Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=992 last=994 duration=3.058184317s
level=info ts=2019-12-11T09:00:05.306Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=992 last=994 duration=2.99204816s
level=info ts=2019-12-11T09:00:02.385Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.295437ms
level=info ts=2019-12-11T09:00:02.313Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.456638ms
level=info ts=2019-12-11T09:00:02.023Z caller=compact.go:496 component=tsdb msg="write block" mint=1576044000000 maxt=1576051200000 ulid=01DVT2ZFP3R7RB9H6BS3JVAMXJ duration=1.955843851s
level=info ts=2019-12-11T09:00:01.935Z caller=compact.go:496 component=tsdb msg="write block" mint=1576044000000 maxt=1576051200000 ulid=01DVT2ZFNYB7DEH57ZX4HW2DAV duration=1.873653026s
level=info ts=2019-12-11T07:00:05.441Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=989 last=991 duration=3.013763908s
level=info ts=2019-12-11T07:00:05.272Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=989 last=991 duration=2.979497994s
level=info ts=2019-12-11T07:00:02.427Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.635643ms
level=info ts=2019-12-11T07:00:02.293Z caller=head.go:596 component=tsdb msg="head GC completed" duration=121.051415ms
level=info ts=2019-12-11T07:00:02.056Z caller=compact.go:496 component=tsdb msg="write block" mint=1576036800000 maxt=1576044000000 ulid=01DVSW3RDWE1WHSM8AEW0ARA3S duration=1.995603695s
level=info ts=2019-12-11T07:00:01.941Z caller=compact.go:496 component=tsdb msg="write block" mint=1576036800000 maxt=1576044000000 ulid=01DVSW3RDRKTJVEZWGYE07XBXE duration=1.885680378s
level=info ts=2019-12-11T06:20:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T06:14:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T05:02:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=5 to=4
level=info ts=2019-12-11T05:01:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=5
level=info ts=2019-12-11T05:00:05.488Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=986 last=988 duration=3.043360624s
level=info ts=2019-12-11T05:00:05.288Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=986 last=988 duration=2.998209654s
level=info ts=2019-12-11T05:00:02.445Z caller=head.go:596 component=tsdb msg="head GC completed" duration=130.642245ms
level=info ts=2019-12-11T05:00:02.290Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.363621ms
level=info ts=2019-12-11T05:00:02.066Z caller=compact.go:496 component=tsdb msg="write block" mint=1576029600000 maxt=1576036800000 ulid=01DVSN815STR0D0B8245RWNF13 duration=2.008689142s
level=info ts=2019-12-11T05:00:01.938Z caller=compact.go:496 component=tsdb msg="write block" mint=1576029600000 maxt=1576036800000 ulid=01DVSN815W8YTW3DPQTJVRNTS4 duration=1.877943808s
level=info ts=2019-12-11T04:55:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T04:35:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T04:28:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T04:15:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T04:07:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T04:03:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T03:57:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T03:52:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T03:43:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T03:32:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T03:24:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T03:19:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T03:12:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T03:00:08.325Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576022400000 ulid=01DVSECF6Q4JXFDGMFQB3J1Z9E sources="[01DVRSS45W7DXE05RGBYGH58PY 01DVS0MVDWGK47AZ3HY5GQEMK4 01DVS7GJNW7BF3R6KK7GW291R0]" duration=2.861556831s
level=info ts=2019-12-11T03:00:08.255Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576022400000 ulid=01DVSECF6J4NZRNHABZ2MSG7V7 sources="[01DVRSS4632MEB6SYC6SB7DTGE 01DVS0MVDR5Z67QJD6T94CXHRA 01DVS7GJNSYRFT48H9CDRP82YV]" duration=2.796902205s
level=info ts=2019-12-11T03:00:05.253Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=983 last=985 duration=3.004398083s
level=info ts=2019-12-11T03:00:05.245Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=983 last=985 duration=3.023743067s
level=info ts=2019-12-11T03:00:02.248Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.893231ms
level=info ts=2019-12-11T03:00:02.221Z caller=head.go:596 component=tsdb msg="head GC completed" duration=132.662929ms
level=info ts=2019-12-11T03:00:01.903Z caller=compact.go:496 component=tsdb msg="write block" mint=1576022400000 maxt=1576029600000 ulid=01DVSEC9XW0W8V42SPRR0YMM0X duration=1.842688968s
level=info ts=2019-12-11T03:00:01.847Z caller=compact.go:496 component=tsdb msg="write block" mint=1576022400000 maxt=1576029600000 ulid=01DVSEC9XXK1J3B0186KYQECZT duration=1.78558499s
level=info ts=2019-12-11T02:18:44.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=5 to=4
level=info ts=2019-12-11T02:18:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=5
level=info ts=2019-12-11T02:11:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T01:59:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-11T01:52:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-11T01:00:05.272Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=980 last=982 duration=3.025045534s
level=info ts=2019-12-11T01:00:05.189Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=980 last=982 duration=2.992866718s
level=info ts=2019-12-11T01:00:02.247Z caller=head.go:596 component=tsdb msg="head GC completed" duration=123.561834ms
level=info ts=2019-12-11T01:00:02.196Z caller=head.go:596 component=tsdb msg="head GC completed" duration=108.589195ms
level=info ts=2019-12-11T01:00:01.905Z caller=compact.go:496 component=tsdb msg="write block" mint=1576015200000 maxt=1576022400000 ulid=01DVS7GJNW7BF3R6KK7GW291R0 duration=1.844635186s
level=info ts=2019-12-11T01:00:01.866Z caller=compact.go:496 component=tsdb msg="write block" mint=1576015200000 maxt=1576022400000 ulid=01DVS7GJNSYRFT48H9CDRP82YV duration=1.809175377s
level=info ts=2019-12-11T00:31:25.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=7 to=5
level=info ts=2019-12-11T00:30:25.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=10 to=7
level=info ts=2019-12-11T00:29:55.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=14 to=10
level=info ts=2019-12-11T00:29:25.064Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=85 to=14
level=info ts=2019-12-11T00:29:15.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=5 to=85
level=info ts=2019-12-10T23:00:05.385Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=977 last=979 duration=3.157877457s
level=info ts=2019-12-10T23:00:05.136Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=977 last=979 duration=3.013728541s
level=info ts=2019-12-10T23:00:02.227Z caller=head.go:596 component=tsdb msg="head GC completed" duration=133.349607ms
level=info ts=2019-12-10T23:00:02.123Z caller=head.go:596 component=tsdb msg="head GC completed" duration=110.615384ms
level=info ts=2019-12-10T23:00:01.832Z caller=compact.go:496 component=tsdb msg="write block" mint=1576008000000 maxt=1576015200000 ulid=01DVS0MVDWGK47AZ3HY5GQEMK4 duration=1.772580137s
level=info ts=2019-12-10T23:00:01.780Z caller=compact.go:496 component=tsdb msg="write block" mint=1576008000000 maxt=1576015200000 ulid=01DVS0MVDR5Z67QJD6T94CXHRA duration=1.724738556s
level=info ts=2019-12-10T21:00:18.426Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575936000000 maxt=1576000800000 ulid=01DVRSSD7JM2Y5MQNCZ7QZRSWK sources="[01DVQGJZNRDFP0P161HP7GJX44 01DVR563YKZY789FPAM3DD8DKX 01DVRSS9Q04WN7F254ZCSQ4YP5]" duration=9.096172888s
level=info ts=2019-12-10T21:00:16.394Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575936000000 maxt=1576000800000 ulid=01DVRSSCTH9ED87EXNSR9J8PE6 sources="[01DVQGJY3RF9X7R93QY6V579W3 01DVR563QG1PZ1AY7RPKSCMKND 01DVRSS9HMN2EC6QQQ2XP7R90D]" duration=7.481260173s
level=info ts=2019-12-10T21:00:08.859Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575979200000 maxt=1576000800000 ulid=01DVRSS9Q04WN7F254ZCSQ4YP5 sources="[01DVR55YDW6Q96ZHGXD1T7HVF4 01DVRC1NNWE3ZMEQ6035ZJTF49 01DVRJXCXW4W35MBB4E9RXX1QD]" duration=3.130772971s
level=info ts=2019-12-10T21:00:08.473Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575979200000 maxt=1576000800000 ulid=01DVRSS9HMN2EC6QQQ2XP7R90D sources="[01DVR55YE3CA9B12S48FTTFSVD 01DVRC1NP4CCWPRMCC7667R1FZ 01DVRJXCY0DY2R6DVWWGNXNPRQ]" duration=2.917254733s
level=info ts=2019-12-10T21:00:05.490Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=974 last=976 duration=3.159908932s
level=info ts=2019-12-10T21:00:05.339Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=974 last=976 duration=3.090229598s
level=info ts=2019-12-10T21:00:02.330Z caller=head.go:596 component=tsdb msg="head GC completed" duration=129.067188ms
level=info ts=2019-12-10T21:00:02.249Z caller=head.go:596 component=tsdb msg="head GC completed" duration=108.272575ms
level=info ts=2019-12-10T21:00:01.804Z caller=compact.go:496 component=tsdb msg="write block" mint=1576000800000 maxt=1576008000000 ulid=01DVRSS45W7DXE05RGBYGH58PY duration=1.743999568s
level=info ts=2019-12-10T21:00:01.774Z caller=compact.go:496 component=tsdb msg="write block" mint=1576000800000 maxt=1576008000000 ulid=01DVRSS4632MEB6SYC6SB7DTGE duration=1.706876662s
level=info ts=2019-12-10T20:35:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
level=info ts=2019-12-10T20:29:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-10T19:00:06.012Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=971 last=973 duration=3.248527735s
level=info ts=2019-12-10T19:00:05.641Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=971 last=973 duration=3.1047498s
level=info ts=2019-12-10T19:00:02.763Z caller=head.go:596 component=tsdb msg="head GC completed" duration=210.397069ms
level=info ts=2019-12-10T19:00:02.536Z caller=head.go:596 component=tsdb msg="head GC completed" duration=171.333573ms
level=info ts=2019-12-10T19:00:02.259Z caller=compact.go:496 component=tsdb msg="write block" mint=1575993600000 maxt=1576000800000 ulid=01DVRJXCXW4W35MBB4E9RXX1QD duration=2.199162459s
level=info ts=2019-12-10T19:00:02.080Z caller=compact.go:496 component=tsdb msg="write block" mint=1575993600000 maxt=1576000800000 ulid=01DVRJXCY0DY2R6DVWWGNXNPRQ duration=2.016321337s
level=info ts=2019-12-10T17:00:05.549Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=968 last=970 duration=3.183706512s
level=info ts=2019-12-10T17:00:05.319Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=968 last=970 duration=3.088304654s
level=info ts=2019-12-10T17:00:02.365Z caller=head.go:596 component=tsdb msg="head GC completed" duration=133.008474ms
level=info ts=2019-12-10T17:00:02.231Z caller=head.go:596 component=tsdb msg="head GC completed" duration=114.89207ms
level=info ts=2019-12-10T17:00:01.942Z caller=compact.go:496 component=tsdb msg="write block" mint=1575986400000 maxt=1575993600000 ulid=01DVRC1NNWE3ZMEQ6035ZJTF49 duration=1.881731957s
level=info ts=2019-12-10T17:00:01.864Z caller=compact.go:496 component=tsdb msg="write block" mint=1575986400000 maxt=1575993600000 ulid=01DVRC1NP4CCWPRMCC7667R1FZ duration=1.795832733s
level=info ts=2019-12-10T15:00:09.507Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575957600000 maxt=1575979200000 ulid=01DVR563YKZY789FPAM3DD8DKX sources="[01DVQGJRNW1DY7K8KW6B2RY4FF 01DVQQEFXWYP18MDFKY58VJCSG 01DVQYA75W7ERXK6FBEMYYEX6S]" duration=3.791514409s
level=info ts=2019-12-10T15:00:08.520Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575957600000 maxt=1575979200000 ulid=01DVR563QG1PZ1AY7RPKSCMKND sources="[01DVQGJRP3CC86KKVR5MZ1YYTK 01DVQQEFY0CEBFC4QE02GW9S4F 01DVQYA75TMRETNZRWPV46G5Y0]" duration=3.032106451s
level=info ts=2019-12-10T15:00:05.484Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=965 last=967 duration=3.124836463s
level=info ts=2019-12-10T15:00:05.277Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=965 last=967 duration=3.055778688s
level=info ts=2019-12-10T15:00:02.359Z caller=head.go:596 component=tsdb msg="head GC completed" duration=130.554076ms
level=info ts=2019-12-10T15:00:02.221Z caller=head.go:596 component=tsdb msg="head GC completed" duration=114.665423ms
level=info ts=2019-12-10T15:00:01.972Z caller=compact.go:496 component=tsdb msg="write block" mint=1575979200000 maxt=1575986400000 ulid=01DVR55YDW6Q96ZHGXD1T7HVF4 duration=1.912209972s
level=info ts=2019-12-10T15:00:01.878Z caller=compact.go:496 component=tsdb msg="write block" mint=1575979200000 maxt=1575986400000 ulid=01DVR55YE3CA9B12S48FTTFSVD duration=1.811316924s
2019-12-10 13:50:13.598594155 +0000 UTC
level=info ts=2019-12-10T13:50:13.596Z caller=main.go:771 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml
level=info ts=2019-12-10T13:50:13.553Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
level=info ts=2019-12-10T13:50:13.552Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
level=info ts=2019-12-10T13:50:13.551Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
level=info ts=2019-12-10T13:50:13.554Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
level=info ts=2019-12-10T13:50:13.546Z caller=main.go:743 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
1 curl -X POST --fail -o - -sS http://localhost:80/prometheus/-/reload
2019/12/10 13:50:13 DEBUG: "/etc/prometheus/..2019_12_05_07_22_08.390693530": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC
2019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC
2019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data/prometheus.yml": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC
2019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data/recording.rules": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC
2019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data/alerts.rules": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC
2019/12/10 13:50:13 DEBUG: Watching /etc/prometheus/..data
2019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data": CREATE at 2019-12-10 13:50:13.341280871 +0000 UTC
2019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data_tmp": RENAME at 2019-12-10 13:50:13.341280871 +0000 UTC
2019/12/10 13:50:13 DEBUG: "/etc/prometheus/..2019_12_10_13_50_13.738754268": CHMOD at 2019-12-10 13:50:13.341280871 +0000 UTC
2019/12/10 13:50:13 DEBUG: Watching /etc/prometheus/..2019_12_10_13_50_13.738754268
2019/12/10 13:50:13 DEBUG: "/etc/prometheus/..2019_12_10_13_50_13.738754268": CREATE at 2019-12-10 13:50:13.341280871 +0000 UTC
2019-12-10 13:49:53.296090488 +0000 UTC
level=info ts=2019-12-10T13:49:53.294Z caller=main.go:771 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml
level=info ts=2019-12-10T13:49:53.254Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
level=info ts=2019-12-10T13:49:53.253Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
level=info ts=2019-12-10T13:49:53.252Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
level=info ts=2019-12-10T13:49:53.251Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
level=info ts=2019-12-10T13:49:53.248Z caller=main.go:743 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
curl -X POST --fail -o - -sS http://localhost:80/prometheus/-/reload
2019/12/10 13:49:53 DEBUG: "/etc/prometheus/..2019_12_05_07_22_36.096562123": REMOVE at 2019-12-10 13:49:53.044039978 +0000 UTC
2019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC
2019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data/prometheus.yml": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC
2019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data/recording.rules": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC
2019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data/alerts.rules": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC
2019/12/10 13:49:53 DEBUG: Watching /etc/prometheus/..data
2019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data": CREATE at 2019-12-10 13:49:53.043039894 +0000 UTC
2019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data_tmp": RENAME at 2019-12-10 13:49:53.044039978 +0000 UTC
2019/12/10 13:49:53 DEBUG: "/etc/prometheus/..2019_12_10_13_49_53.355073198": CHMOD at 2019-12-10 13:49:53.043039894 +0000 UTC
2019/12/10 13:49:53 DEBUG: Watching /etc/prometheus/..2019_12_10_13_49_53.355073198
2019/12/10 13:49:53 DEBUG: "/etc/prometheus/..2019_12_10_13_49_53.355073198": CREATE at 2019-12-10 13:49:53.043039894 +0000 UTC
level=info ts=2019-12-10T13:00:06.007Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=962 last=964 duration=3.145696569s
level=info ts=2019-12-10T13:00:05.601Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=962 last=964 duration=3.062580976s
level=info ts=2019-12-10T13:00:02.861Z caller=head.go:596 component=tsdb msg="head GC completed" duration=169.077152ms
level=info ts=2019-12-10T13:00:02.539Z caller=head.go:596 component=tsdb msg="head GC completed" duration=152.173262ms
level=info ts=2019-12-10T13:00:02.425Z caller=compact.go:496 component=tsdb msg="write block" mint=1575972000000 maxt=1575979200000 ulid=01DVQYA75W7ERXK6FBEMYYEX6S duration=2.364066751s
level=info ts=2019-12-10T13:00:02.150Z caller=compact.go:496 component=tsdb msg="write block" mint=1575972000000 maxt=1575979200000 ulid=01DVQYA75TMRETNZRWPV46G5Y0 duration=2.092629264s
level=info ts=2019-12-10T11:48:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=2 to=1
level=info ts=2019-12-10T11:47:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=3 to=2
level=info ts=2019-12-10T11:47:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=5 to=3
level=info ts=2019-12-10T11:46:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=7 to=5
level=info ts=2019-12-10T11:46:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=10 to=7
level=info ts=2019-12-10T11:46:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=16 to=10
level=info ts=2019-12-10T11:45:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=26 to=16
level=info ts=2019-12-10T11:45:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=39 to=26
level=info ts=2019-12-10T11:45:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=63 to=39
level=info ts=2019-12-10T11:44:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=100 to=63
level=info ts=2019-12-10T11:44:45.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=2 to=1
level=info ts=2019-12-10T11:44:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=156 to=100
level=info ts=2019-12-10T11:44:15.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=3 to=2
level=info ts=2019-12-10T11:44:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=253 to=156
level=info ts=2019-12-10T11:43:55.064Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=4 to=3
level=info ts=2019-12-10T11:43:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=405 to=253
level=info ts=2019-12-10T11:43:45.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=5 to=4
level=info ts=2019-12-10T11:43:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=619 to=405
level=info ts=2019-12-10T11:43:25.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=8 to=5
level=info ts=2019-12-10T11:43:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=1000 to=619
level=info ts=2019-12-10T11:43:05.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=11 to=8
level=info ts=2019-12-10T11:42:45.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=22 to=11
level=info ts=2019-12-10T11:42:25.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=16 to=22
level=error ts=2019-12-10T11:42:11.074Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 400 Bad Request: out of order sample"
level=error ts=2019-12-10T11:42:11.073Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 400 Bad Request: out of order sample"
level=info ts=2019-12-10T11:42:05.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=11 to=16
level=info ts=2019-12-10T11:41:55.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=8 to=11
level=info ts=2019-12-10T11:39:35.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=6 to=8
level=info ts=2019-12-10T11:38:55.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=4 to=6
level=info ts=2019-12-10T11:38:35.064Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=2 to=4
level=error ts=2019-12-10T11:38:12.281Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
level=error ts=2019-12-10T11:38:12.281Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples"
level=error ts=2019-12-10T11:38:12.235Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.236Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.204Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
level=error ts=2019-12-10T11:38:12.183Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples"
level=error ts=2019-12-10T11:38:12.129Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples"
level=error ts=2019-12-10T11:38:12.127Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples"
level=error ts=2019-12-10T11:38:12.128Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.129Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
level=error ts=2019-12-10T11:38:12.139Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
level=error ts=2019-12-10T11:38:12.140Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.141Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.142Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.143Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=56 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 56 samples"
level=error ts=2019-12-10T11:38:12.125Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=52 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 52 samples"
level=error ts=2019-12-10T11:38:12.124Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.123Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
level=error ts=2019-12-10T11:38:12.122Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples"
level=error ts=2019-12-10T11:38:12.120Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.1201Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples"
level=error ts=2019-12-10T11:38:12.1202Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=78 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 78 samples"
level=error ts=2019-12-10T11:38:12.1203Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.1191Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.1192Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.081Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
level=error ts=2019-12-10T11:38:12.0371Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.0372Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.0373Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.024Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.025Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:12.026Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples"
level=error ts=2019-12-10T11:38:11.920Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.917Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.916Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.913Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
level=error ts=2019-12-10T11:38:11.913Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
level=error ts=2019-12-10T11:38:11.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.721Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.176Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.1451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.1452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.1453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples"
level=error ts=2019-12-10T11:38:11.1454Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
level=error ts=2019-12-10T11:38:11.1455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.1441Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.1442Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
level=error ts=2019-12-10T11:38:11.1443Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples"
level=error ts=2019-12-10T11:38:11.1444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
level=error ts=2019-12-10T11:38:11.1445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=46 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 46 samples"
level=error ts=2019-12-10T11:38:11.1431Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.1432Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
level=error ts=2019-12-10T11:38:11.1433Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.1434Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=92 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 92 samples"
level=error ts=2019-12-10T11:38:11.0011Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.0012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:11.0012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
level=error ts=2019-12-10T11:38:11.0013Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.9551Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.9532Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.9421Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
level=error ts=2019-12-10T11:38:10.9422Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.9423Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.941Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=58 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 58 samples"
level=error ts=2019-12-10T11:38:10.942Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.940Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.943Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=23 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 23 samples"
level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=59 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 59 samples"
level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples"
level=error ts=2019-12-10T11:38:10.878Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.879Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.8781Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.8782Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:10.877Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
level=error ts=2019-12-10T11:38:10.877Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
level=error ts=2019-12-10T11:38:10.877Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples"
level=error ts=2019-12-10T11:38:10.039Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.902Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
level=error ts=2019-12-10T11:38:09.901Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.902Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.903Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=92 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 92 samples"
level=error ts=2019-12-10T11:38:09.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8971Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
level=error ts=2019-12-10T11:38:09.8972Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=73 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 73 samples"
level=error ts=2019-12-10T11:38:09.8973Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8974Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=68 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 68 samples"
level=error ts=2019-12-10T11:38:09.8975Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8976Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=35 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 35 samples"
level=error ts=2019-12-10T11:38:09.8977Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8951Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8952Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
level=error ts=2019-12-10T11:38:09.8953Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
level=error ts=2019-12-10T11:38:09.876Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8401Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples"
level=error ts=2019-12-10T11:38:09.8402Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=92 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 92 samples"
level=error ts=2019-12-10T11:38:09.8403Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8404Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
level=error ts=2019-12-10T11:38:09.8406Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8407Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.832Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.825Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8251Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8241Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8242Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8243Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
level=error ts=2019-12-10T11:38:09.8244Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=17 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 17 samples"
level=error ts=2019-12-10T11:38:09.8245Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.8246Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.815Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=94 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 94 samples"
level=error ts=2019-12-10T11:38:09.806Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples"
level=error ts=2019-12-10T11:38:09.736Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.736Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=48 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 48 samples"
level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples"
level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=43 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 43 samples"
level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=19 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 19 samples"
level=error ts=2019-12-10T11:38:09.733Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
level=error ts=2019-12-10T11:38:09.700Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:09.700Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=50 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 50 samples"
level=error ts=2019-12-10T11:38:09.700Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=37 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 37 samples"
level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples"
level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
level=error ts=2019-12-10T11:38:08.795Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples"
level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=94 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 94 samples"
level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples"
level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:08.694Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
level=error ts=2019-12-10T11:38:08.694Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:08.694Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:08.693Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=58 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 58 samples"
level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples"
level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=59 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 59 samples"
level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:08.691Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:08.678Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
level=error ts=2019-12-10T11:38:08.677Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:08.648Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
level=error ts=2019-12-10T11:38:08.647Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
level=error ts=2019-12-10T11:38:08.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples"
level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
level=error ts=2019-12-10T11:38:08.602Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
level=error ts=2019-12-10T11:38:08.596Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
level=error ts=2019-12-10T11:38:08.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
level=error ts=2019-12-10T11:38:08.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
level=error ts=2019-12-10T11:38:08.553Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
level=error ts=2019-12-10T11:38:08.552Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
level=error ts=2019-12-10T11:38:08.552Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=73 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 73 samples"
level=error ts=2019-12-10T11:38:08.552Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=87 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 87 samples"
level=error ts=2019-12-10T11:38:08.541Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
level=error ts=2019-12-10T11:38:08.501Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
level=error ts=2019-12-10T11:38:08.501Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples"
level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples"
level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
level=error ts=2019-12-10T11:38:08.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
level=error ts=2019-12-10T11:38:08.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=65 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 65 samples"
level=error ts=2019-12-10T11:38:08.441Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples"
level=error ts=2019-12-10T11:38:08.433Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:08.431Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples"
level=error ts=2019-12-10T11:38:08.407Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
level=error ts=2019-12-10T11:38:08.393Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:08.3941Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:08.395Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:08.396Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
level=error ts=2019-12-10T11:38:08.393Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
level=error ts=2019-12-10T11:38:08.394Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:08.391Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:08.395Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:07.604Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:07.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:07.602Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=60 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 60 samples"
level=error ts=2019-12-10T11:38:07.602Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples"
level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
level=error ts=2019-12-10T11:38:07.600Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:07.599Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:07.599Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=93 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 93 samples"
level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples"
level=error ts=2019-12-10T11:38:07.599Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:07.596Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
level=error ts=2019-12-10T11:38:07.596Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
level=error ts=2019-12-10T11:38:07.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
level=error ts=2019-12-10T11:38:07.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples"
level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples"
level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples"
level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=73 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 73 samples"
level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples"
level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
level=error ts=2019-12-10T11:38:07.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
level=error ts=2019-12-10T11:38:07.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=53 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 53 samples"
level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=39 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 39 samples"
level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
level=error ts=2019-12-10T11:38:07.587Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
level=error ts=2019-12-10T11:38:07.513Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
level=error ts=2019-12-10T11:38:07.512Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=59 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 59 samples"
level=error ts=2019-12-10T11:38:07.512Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
level=error ts=2019-12-10T11:38:07.510Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
level=error ts=2019-12-10T11:38:07.506Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
level=error ts=2019-12-10T11:38:07.506Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples"
level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=60 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 60 samples"
level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples"
level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples"
level=error ts=2019-12-10T11:38:07.494Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples"
level=error ts=2019-12-10T11:38:07.489Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
level=error ts=2019-12-10T11:38:07.488Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
level=error ts=2019-12-10T11:38:07.484Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
level=error ts=2019-12-10T11:38:07.483Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
level=error ts=2019-12-10T11:38:07.483Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples"
level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples"
level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples"
level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 66 samples"
level=error ts=2019-12-10T11:38:07.478Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=76 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 76 samples"
level=error ts=2019-12-10T11:38:07.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
level=error ts=2019-12-10T11:38:07.455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:07.455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:07.455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=65 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 65 samples"
level=error ts=2019-12-10T11:38:07.454Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:07.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:07.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
level=error ts=2019-12-10T11:38:07.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=37 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 37 samples"
level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=30 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 30 samples"
level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples"
level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples"
level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=63 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 63 samples"
level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:07.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:07.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:07.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
level=error ts=2019-12-10T11:38:07.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:07.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples"
level=error ts=2019-12-10T11:38:07.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:07.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:07.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:07.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:07.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=57 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 57 samples"
level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:07.444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:07.444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:07.444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
level=error ts=2019-12-10T11:38:07.443Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:07.390Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
level=error ts=2019-12-10T11:38:07.388Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
level=error ts=2019-12-10T11:38:07.388Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
level=error ts=2019-12-10T11:38:07.388Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples"
level=error ts=2019-12-10T11:38:07.387Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
level=error ts=2019-12-10T11:38:07.387Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
level=error ts=2019-12-10T11:38:07.387Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=63 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 63 samples"
level=error ts=2019-12-10T11:38:07.376Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
level=error ts=2019-12-10T11:38:07.358Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
level=error ts=2019-12-10T11:38:07.310Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
level=error ts=2019-12-10T11:38:07.259Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
level=error ts=2019-12-10T11:38:07.259Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
level=error ts=2019-12-10T11:38:07.045Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
level=error ts=2019-12-10T11:38:07.044Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
level=error ts=2019-12-10T11:38:07.043Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
level=error ts=2019-12-10T11:38:07.043Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples"
level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
level=error ts=2019-12-10T11:38:07.041Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=49 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 49 samples"
level=error ts=2019-12-10T11:38:06.947Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples"
level=error ts=2019-12-10T11:38:06.947Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
level=error ts=2019-12-10T11:38:06.946Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
level=error ts=2019-12-10T11:38:06.944Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples"
level=error ts=2019-12-10T11:38:06.943Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
level=error ts=2019-12-10T11:38:06.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples"
level=error ts=2019-12-10T11:38:06.938Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
level=error ts=2019-12-10T11:38:06.937Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
level=error ts=2019-12-10T11:38:06.848Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=54 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 54 samples"
level=error ts=2019-12-10T11:38:06.848Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
level=error ts=2019-12-10T11:38:06.842Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
level=error ts=2019-12-10T11:38:06.841Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
level=error ts=2019-12-10T11:38:06.747Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples"
level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples"
level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=65 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 65 samples"
level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:06.715Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:06.646Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
level=error ts=2019-12-10T11:38:06.646Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=37 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 37 samples"
level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:06.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:06.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:06.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:06.639Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:06.638Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:06.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=93 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 93 samples"
level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
level=error ts=2019-12-10T11:38:06.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
level=error ts=2019-12-10T11:38:06.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=56 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 56 samples"
level=error ts=2019-12-10T11:38:06.590Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
level=error ts=2019-12-10T11:38:06.590Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
level=error ts=2019-12-10T11:38:06.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
level=error ts=2019-12-10T11:38:06.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
level=error ts=2019-12-10T11:38:06.588Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
level=error ts=2019-12-10T11:38:06.588Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
level=error ts=2019-12-10T11:38:06.581Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
level=error ts=2019-12-10T11:38:06.581Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
level=error ts=2019-12-10T11:38:06.563Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
level=error ts=2019-12-10T11:38:06.563Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
level=error ts=2019-12-10T11:38:06.563Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
level=error ts=2019-12-10T11:38:06.562Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
level=error ts=2019-12-10T11:38:06.562Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
level=error ts=2019-12-10T11:38:06.561Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
level=error ts=2019-12-10T11:38:06.560Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
level=error ts=2019-12-10T11:38:06.551Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
level=error ts=2019-12-10T11:38:06.550Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
level=error ts=2019-12-10T11:38:06.544Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
level=error ts=2019-12-10T11:38:06.544Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples"
level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples"
level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
level=error ts=2019-12-10T11:38:06.542Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
level=error ts=2019-12-10T11:38:06.542Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
level=error ts=2019-12-10T11:38:06.542Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
level=error ts=2019-12-10T11:38:06.540Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples"
level=error ts=2019-12-10T11:38:06.538Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples"
level=error ts=2019-12-10T11:38:06.538Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 137 samples"
level=error ts=2019-12-10T11:38:06.512Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 138 samples"
level=error ts=2019-12-10T11:38:06.506Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 140 samples"
level=error ts=2019-12-10T11:38:06.505Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 141 samples"
level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples"
level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 142 samples"
level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 143 samples"
level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
level=error ts=2019-12-10T11:38:06.489Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 144 samples"
level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 145 samples"
level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 146 samples"
level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 147 samples"
level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
level=error ts=2019-12-10T11:38:06.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 148 samples"
level=error ts=2019-12-10T11:38:06.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 149 samples"
level=error ts=2019-12-10T11:38:06.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
level=error ts=2019-12-10T11:38:06.459Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 150 samples"
level=error ts=2019-12-10T11:38:06.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 151 samples"
level=error ts=2019-12-10T11:38:06.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 152 samples"
level=error ts=2019-12-10T11:38:06.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 153 samples"
level=error ts=2019-12-10T11:38:06.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 154 samples"
level=error ts=2019-12-10T11:38:06.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
level=error ts=2019-12-10T11:38:06.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 155 samples"
level=error ts=2019-12-10T11:38:06.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 156 samples"
level=error ts=2019-12-10T11:38:06.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 157 samples"
level=error ts=2019-12-10T11:38:06.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
level=error ts=2019-12-10T11:38:06.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:06.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:06.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
level=error ts=2019-12-10T11:38:06.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 66 samples"
level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=33 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 33 samples"
level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples"
level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=62 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 62 samples"
level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=58 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 58 samples"
level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples"
level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:06.353Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=60 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 60 samples"
level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=22 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 22 samples"
level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 155 samples"
level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:06.347Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:06.347Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:06.346Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
level=error ts=2019-12-10T11:38:06.346Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:06.340Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples"
level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
level=error ts=2019-12-10T11:38:06.338Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
level=error ts=2019-12-10T11:38:06.338Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
level=error ts=2019-12-10T11:38:06.337Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
level=error ts=2019-12-10T11:38:06.330Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
level=error ts=2019-12-10T11:38:06.329Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
level=error ts=2019-12-10T11:38:06.329Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
level=error ts=2019-12-10T11:38:06.327Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
level=error ts=2019-12-10T11:38:06.244Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
level=error ts=2019-12-10T11:38:06.231Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
level=error ts=2019-12-10T11:38:06.230Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
level=error ts=2019-12-10T11:38:06.229Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
level=error ts=2019-12-10T11:38:06.229Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
level=error ts=2019-12-10T11:38:06.229Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=94 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 94 samples"
level=error ts=2019-12-10T11:38:06.206Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples"
level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples"
level=error ts=2019-12-10T11:38:06.198Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples"
level=error ts=2019-12-10T11:38:06.091Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples"
level=error ts=2019-12-10T11:38:06.091Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples"
level=error ts=2019-12-10T11:38:06.091Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 139 samples"
level=error ts=2019-12-10T11:38:06.088Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 137 samples"
level=error ts=2019-12-10T11:38:06.087Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 138 samples"
level=error ts=2019-12-10T11:38:05.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:05.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:05.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
level=error ts=2019-12-10T11:38:05.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=info ts=2019-12-10T11:38:05.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=1 to=2
level=error ts=2019-12-10T11:38:04.977Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:04.976Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:04.821Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:04.821Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
level=error ts=2019-12-10T11:38:04.821Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:04.815Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:04.815Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:04.720Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples"
level=error ts=2019-12-10T11:38:04.720Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:04.720Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:04.719Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
level=error ts=2019-12-10T11:38:04.719Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:04.717Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:04.716Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=87 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 87 samples"
level=error ts=2019-12-10T11:38:04.716Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
level=error ts=2019-12-10T11:38:04.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
level=error ts=2019-12-10T11:38:04.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 999 samples"
level=error ts=2019-12-10T11:38:04.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
level=error ts=2019-12-10T11:38:04.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
level=error ts=2019-12-10T11:38:04.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
level=error ts=2019-12-10T11:38:04.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
level=error ts=2019-12-10T11:38:04.709Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 66 samples"
level=error ts=2019-12-10T11:38:04.709Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=64 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 64 samples"
level=error ts=2019-12-10T11:38:04.631Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
level=error ts=2019-12-10T11:38:04.631Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
level=error ts=2019-12-10T11:38:04.631Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
level=error ts=2019-12-10T11:38:04.627Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
level=error ts=2019-12-10T11:38:04.626Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
level=error ts=2019-12-10T11:38:04.626Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
level=error ts=2019-12-10T11:38:04.626Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 1244 samples"
level=error ts=2019-12-10T11:38:04.621Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
level=error ts=2019-12-10T11:38:04.620Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
level=error ts=2019-12-10T11:38:04.620Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
level=error ts=2019-12-10T11:38:04.620Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
level=error ts=2019-12-10T11:38:04.619Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
level=error ts=2019-12-10T11:38:04.136Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
level=error ts=2019-12-10T11:38:04.136Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
level=error ts=2019-12-10T11:38:04.135Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
level=error ts=2019-12-10T11:38:04.018Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
level=error ts=2019-12-10T11:38:04.018Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples"
level=error ts=2019-12-10T11:38:04.017Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 141 samples"
level=error ts=2019-12-10T11:38:04.016Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 142 samples"
level=error ts=2019-12-10T11:38:04.015Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 153 samples"
level=error ts=2019-12-10T11:38:04.015Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 1223 samples"
level=error ts=2019-12-10T11:38:04.013Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 164 samples"
level=error ts=2019-12-10T11:38:04.013Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 161 samples"
level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 162 samples"
level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 163 samples"
level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 169 samples"
level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 170 samples"
level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
level=error ts=2019-12-10T11:38:04.010Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:04.009Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:04.009Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:04.008Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:04.007Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples"
level=error ts=2019-12-10T11:38:04.007Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:04.007Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:04.005Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
level=error ts=2019-12-10T11:38:04.005Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
level=error ts=2019-12-10T11:38:04.004Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
level=error ts=2019-12-10T11:38:04.003Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
level=error ts=2019-12-10T11:38:04.002Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
level=error ts=2019-12-10T11:38:04.002Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
level=error ts=2019-12-10T11:38:04.001Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
level=error ts=2019-12-10T11:38:03.923Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
level=error ts=2019-12-10T11:38:03.923Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
level=error ts=2019-12-10T11:38:03.922Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
level=error ts=2019-12-10T11:38:03.908Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
level=error ts=2019-12-10T11:38:03.908Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
level=error ts=2019-12-10T11:38:03.907Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
level=error ts=2019-12-10T11:38:03.907Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
level=error ts=2019-12-10T11:38:03.906Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples"
level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
level=error ts=2019-12-10T11:38:03.902Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
level=error ts=2019-12-10T11:38:03.901Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples"
level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples"
level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples"
level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples"
level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 137 samples"
level=error ts=2019-12-10T11:38:03.897Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 138 samples"
level=error ts=2019-12-10T11:38:03.897Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 139 samples"
level=error ts=2019-12-10T11:38:03.723Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
level=error ts=2019-12-10T11:38:03.723Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 140 samples"
level=error ts=2019-12-10T11:38:03.723Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 141 samples"
level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 142 samples"
level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 143 samples"
level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 144 samples"
level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 145 samples"
level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 146 samples"
level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=76 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 76 samples"
level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:03.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:03.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:03.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
level=error ts=2019-12-10T11:38:03.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
level=error ts=2019-12-10T11:38:03.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
level=error ts=2019-12-10T11:38:03.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples"
level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=78 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 78 samples"
level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 155 samples"
`, "\n")