Loki: Add a `prepare-shutdown` endpoint (#8786)

**What this PR does / why we need it**:
Add a new `/ingester/prepare_shutdown` endpoint. This new endpoint
configures the ingester to release resources on the next SIGTERM.

---------

Co-authored-by: J Stickler <julie.stickler@grafana.com>

CHANGELOG.md

@@ -8,6 +8,7 @@
 
 ##### Enhancements
 
+* [8786](https://github.com/grafana/loki/pull/8786) **DylanGuedes**: Ingester: add new /ingester/prepare_shutdown endpoint.
 * [8744](https://github.com/grafana/loki/pull/8744) **dannykopping**: Ruler: remote rule evaluation.
 * [8727](https://github.com/grafana/loki/pull/8727) **cstyan** **jeschkies**: Propagate per-request limit header to querier.
 * [8682](https://github.com/grafana/loki/pull/8682) **dannykopping**: Add fetched chunk size distribution metric `loki_chunk_fetcher_fetched_size_bytes`.

docs/sources/api/_index.md

@@ -621,6 +621,17 @@ backing store. Mainly used for local testing.
 
 In microservices mode, the `/flush` endpoint is exposed by the ingester.
 
+### Tell ingester to release all resources on next SIGTERM
+
+```
+POST /ingester/prepare_shutdown
+```
+
+`/ingester/prepare_shutdown` will prepare the ingester to release resources on the next SIGTERM signal,
+where releasing resources means flushing data and unregistering from the ingester ring.
+This endpoint supersedes any YAML configurations and isn't necessary if the ingester is already
+configured to unregister from the ring or to flush on shutdown.
+
 ## Flush in-memory chunks and shut down
 
 ```
@@ -1401,4 +1412,4 @@ $ curl -H "Content-Type: application/json" -XPOST -s "https://localhost:3100/api
 This is helpful for scaling down WAL-enabled ingesters where we want to ensure old WAL directories are not orphaned,
 but instead flushed to our chunk backend.
 
-In microservices mode, the `/ingester/flush_shutdown` endpoint is exposed by the ingester.
+In microservices mode, the `/ingester/flush_shutdown` endpoint is exposed by the ingester.

pkg/ingester/ingester.go

@@ -181,6 +181,7 @@ type Interface interface {
 	// deprecated
 	LegacyShutdownHandler(w http.ResponseWriter, r *http.Request)
 	ShutdownHandler(w http.ResponseWriter, r *http.Request)
+	PrepareShutdown(w http.ResponseWriter, r *http.Request)
 }
 
 // Ingester builds chunks for incoming log streams.
@@ -502,7 +503,8 @@ func (i *Ingester) running(ctx context.Context) error {
 	return serviceError
 }
 
-// Called after running exits, when Ingester transitions to Stopping state.
+// stopping is called when Ingester transitions to Stopping state.
+//
 // At this point, loop no longer runs, but flushers are still running.
 func (i *Ingester) stopping(_ error) error {
 	i.stopIncomingRequests()
@@ -523,8 +525,8 @@ func (i *Ingester) stopping(_ error) error {
 
 	i.streamRateCalculator.Stop()
 
-	// In case the flag to terminate on shutdown is set we need to mark the
-	// ingester service as "failed", so Loki will shut down entirely.
+	// In case the flag to terminate on shutdown is set or this instance is marked to release its resources,
+	// we need to mark the ingester service as "failed", so Loki will shut down entirely.
 	// The module manager logs the failure `modules.ErrStopProcess` in a special way.
 	if i.terminateOnShutdown && errs.Err() == nil {
 		return modules.ErrStopProcess
@@ -568,6 +570,19 @@ func (i *Ingester) LegacyShutdownHandler(w http.ResponseWriter, r *http.Request)
 	w.WriteHeader(http.StatusNoContent)
 }
 
+// PrepareShutdown will handle the /ingester/prepare_shutdown endpoint.
+//
+// Internally, when triggered, this handler will configure the ingester service to release their resources whenever a SIGTERM is received.
+// Releasing resources meaning flushing data, deleting tokens, and removing itself from the ring.
+func (i *Ingester) PrepareShutdown(w http.ResponseWriter, r *http.Request) {
+	level.Info(util_log.Logger).Log("msg", "preparing full ingester shutdown, resources will be released on SIGTERM")
+	i.lifecycler.SetFlushOnShutdown(true)
+	i.lifecycler.SetUnregisterOnShutdown(true)
+	i.terminateOnShutdown = true
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+}
+
 // ShutdownHandler handles a graceful shutdown of the ingester service and
 // termination of the Loki process.
 func (i *Ingester) ShutdownHandler(w http.ResponseWriter, r *http.Request) {

pkg/loki/loki.go

@@ -372,6 +372,8 @@ type Loki struct {
 	deleteClientMetrics *deletion.DeleteRequestClientMetrics
 
 	HTTPAuthMiddleware middleware.Interface
+
+	ingesterRelease *atomic.Bool
 }
 
 // New makes a new Loki.

pkg/loki/modules.go

@@ -471,6 +471,9 @@ func (t *Loki) initIngester() (_ services.Service, err error) {
 	t.Server.HTTP.Methods("POST").Path("/ingester/flush_shutdown").Handler(
 		httpMiddleware.Wrap(http.HandlerFunc(t.Ingester.LegacyShutdownHandler)),
 	)
+	t.Server.HTTP.Methods("POST").Path("/ingester/prepare_shutdown").Handler(
+		httpMiddleware.Wrap(http.HandlerFunc(t.Ingester.PrepareShutdown)),
+	)
 	t.Server.HTTP.Methods("POST").Path("/ingester/shutdown").Handler(
 		httpMiddleware.Wrap(http.HandlerFunc(t.Ingester.ShutdownHandler)),
 	)