chore: add flag to disable pod constraints to querier configs (#14875)

Co-authored-by: Paul Rogers <129207811+paul1r@users.noreply.github.com>

production/ksonnet/loki/config.libsonnet

@@ -52,6 +52,10 @@
       // cores and will result in scheduling delays.
       concurrency: 4,
 
+      // use_no_constraints is false by default allowing either TopologySpreadConstraints or pod antiAffinity to be configured.
+      // If no_schedule_constraints is set to true, neither of the pod constraints will be applied.
+      no_schedule_constraints: false,
+
       // If use_topology_spread is true, queriers can run on nodes already running queriers but will be
       // spread through the available nodes using a TopologySpreadConstraints with a max skew
       // of topology_spread_max_skew.

production/ksonnet/loki/querier.libsonnet

@@ -29,6 +29,7 @@ local k = import 'ksonnet-util/kausal.libsonnet';
   local topologySpreadConstraints = k.core.v1.topologySpreadConstraint,
 
   querier_deployment: if !$._config.stateful_queriers then
+    assert !($._config.querier.no_schedule_constraints && $._config.querier.use_topology_spread) : 'Must configure either no_schedule_constraints or TopologySpreadConstraints, but not both';
     deployment.new('querier', 3, [$.querier_container]) +
     $.config_hash_mixin +
     k.util.configVolumeMount('loki', '/etc/loki/config') +
@@ -38,7 +39,8 @@ local k = import 'ksonnet-util/kausal.libsonnet';
     ) +
     deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge('15%') +
     deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable('15%') +
-    if $._config.querier.use_topology_spread then
+    if $._config.querier.no_schedule_constraints then {}
+    else if $._config.querier.use_topology_spread then
       deployment.spec.template.spec.withTopologySpreadConstraints(
         // Evenly spread queriers among available nodes.
         topologySpreadConstraints.labelSelector.withMatchLabels({ name: 'querier' }) +
@@ -50,6 +52,7 @@ local k = import 'ksonnet-util/kausal.libsonnet';
       k.util.antiAffinity
   else {},
 
+
   // PVC for queriers when running as statefulsets
   querier_data_pvc:: if $._config.stateful_queriers then
     pvc.new('querier-data') +