diff --git a/production/docker/config/loki.yaml b/production/docker/config/loki.yaml
index ac05ff954e..1430494a8c 100644
--- a/production/docker/config/loki.yaml
+++ b/production/docker/config/loki.yaml
@@ -18,6 +18,7 @@ common:
       secret_access_key: supersecret
       s3forcepathstyle: true
   compactor_address: http://loki-write:3100
+  replication_factor: 3
 
 memberlist:
   join_members: ["loki-read", "loki-write"]
diff --git a/production/docker/docker-compose.yaml b/production/docker/docker-compose.yaml
index 8c64c672a5..c654f00cd8 100644
--- a/production/docker/docker-compose.yaml
+++ b/production/docker/docker-compose.yaml
@@ -42,7 +42,7 @@ services:
   prometheus:
     image: prom/prometheus:v2.27.0
     ports:
-      - 9090:9090
+      - 9090
     volumes:
       - ./config/prometheus.yaml:/etc/prometheus/prometheus.yml
       - prometheus:/prometheus
@@ -76,7 +76,7 @@ services:
       - ./loki/:/var/log/
       - ./config:/etc/promtail/
     ports:
-      - "9080:9080"
+      - 9080
     command: -config.file=/etc/promtail/promtail.yaml
     networks:
       - loki
@@ -171,7 +171,7 @@ services:
       - "7946"
       # uncomment to use interactive debugging
       # - "60000-60002:40000" #  makes the replicas available on ports 60000, 60001, 60002
-    command: "-config.file=/etc/loki/loki.yaml -target=backend"
+    command: "-config.file=/etc/loki/loki.yaml -target=backend -legacy-read-mode=false"
     networks:
       - loki
     restart: always
diff --git a/tools/dev/k3d/chartfile.yaml b/tools/dev/k3d/chartfile.yaml
index 9d1a542f2b..9fea6f241b 100644
--- a/tools/dev/k3d/chartfile.yaml
+++ b/tools/dev/k3d/chartfile.yaml
@@ -12,11 +12,11 @@ requires:
 - chart: grafana/grafana
   version: 6.20.1
 - chart: prometheus-community/prometheus
-  version: 15.0.1
+  version: 24.0.0
 - chart: minio/minio
   version: 4.0.12
 - chart: grafana/loki-distributed
   version: 0.42.0
 - chart: prometheus-community/kube-prometheus-stack
-  version: 36.0.1
+  version: 49.0.0
 version: 1
diff --git a/tools/dev/k3d/environments/helm-cluster/main.jsonnet b/tools/dev/k3d/environments/helm-cluster/main.jsonnet
index 8d49959647..5a607a0513 100644
--- a/tools/dev/k3d/environments/helm-cluster/main.jsonnet
+++ b/tools/dev/k3d/environments/helm-cluster/main.jsonnet
@@ -41,6 +41,12 @@ local tenant = 'loki';
         enabled: false,
       },
 
+      // the CRDs are failing to be created due to an annotation being too many characters
+      // they shouldn't be needed as they are already installed in the create_cluster.sh script
+      crds+: {
+        enabled: false,
+      },
+
       kubelet: {
         serviceMonitor: {
           cAdvisorRelabelings: [
diff --git a/tools/dev/k3d/environments/helm-cluster/spec.json b/tools/dev/k3d/environments/helm-cluster/spec.json
index 4dab6b75e4..8da04edb48 100644
--- a/tools/dev/k3d/environments/helm-cluster/spec.json
+++ b/tools/dev/k3d/environments/helm-cluster/spec.json
@@ -6,7 +6,7 @@
     "namespace": "environments/helm-cluster/main.jsonnet"
   },
   "spec": {
-    "apiServer": "https://0.0.0.0:45123",
+    "apiServer": "https://0.0.0.0:38311",
     "namespace": "k3d-helm-cluster",
     "resourceDefaults": {},
     "expectVersions": {}
diff --git a/tools/dev/k3d/jsonnetfile.lock.json b/tools/dev/k3d/jsonnetfile.lock.json
index 4b043ce233..655532ca30 100644
--- a/tools/dev/k3d/jsonnetfile.lock.json
+++ b/tools/dev/k3d/jsonnetfile.lock.json
@@ -8,7 +8,7 @@
           "subdir": "consul"
         }
       },
-      "version": "af3ca2c3fae4096002b0c0c921f18ca7da8d361f",
+      "version": "c0abc546c782a095a22c277d36f871bb94ffc944",
       "sum": "Po3c1Ic96ngrJCtOazic/7OsLkoILOKZWXWyZWl+od8="
     },
     {
@@ -18,7 +18,7 @@
           "subdir": "enterprise-metrics"
         }
       },
-      "version": "af3ca2c3fae4096002b0c0c921f18ca7da8d361f",
+      "version": "c0abc546c782a095a22c277d36f871bb94ffc944",
       "sum": "hi2ZpHKl7qWXmSZ46sAycjWEQK6oGsoECuDKQT1dA+k="
     },
     {
@@ -28,7 +28,7 @@
           "subdir": "etcd-operator"
         }
       },
-      "version": "af3ca2c3fae4096002b0c0c921f18ca7da8d361f",
+      "version": "c0abc546c782a095a22c277d36f871bb94ffc944",
       "sum": "duHm6wmUju5KHQurOe6dnXoKgl5gTUsfGplgbmAOsHw="
     },
     {
@@ -38,7 +38,7 @@
           "subdir": "grafana"
         }
       },
-      "version": "af3ca2c3fae4096002b0c0c921f18ca7da8d361f",
+      "version": "c0abc546c782a095a22c277d36f871bb94ffc944",
       "sum": "Y5nheroSOIwmE+djEVPq4OvvTxKenzdHhpEwaR3Ebjs="
     },
     {
@@ -48,8 +48,8 @@
           "subdir": "jaeger-agent-mixin"
         }
       },
-      "version": "af3ca2c3fae4096002b0c0c921f18ca7da8d361f",
-      "sum": "nsukyr2SS8h97I2mxvBazXZp2fxu1i6eg+rKq3/NRwY="
+      "version": "c0abc546c782a095a22c277d36f871bb94ffc944",
+      "sum": "NyRKfJyqLhB9oHLpr+b47b5yiB3BuBB9ZmRcVk0IVEk="
     },
     {
       "source": {
@@ -58,7 +58,7 @@
           "subdir": "ksonnet-util"
         }
       },
-      "version": "af3ca2c3fae4096002b0c0c921f18ca7da8d361f",
+      "version": "c0abc546c782a095a22c277d36f871bb94ffc944",
       "sum": "0y3AFX9LQSpfWTxWKSwoLgbt0Wc9nnCwhMH2szKzHv0="
     },
     {
@@ -78,8 +78,8 @@
           "subdir": "memcached"
         }
       },
-      "version": "af3ca2c3fae4096002b0c0c921f18ca7da8d361f",
-      "sum": "SWywAq4U0MRPMbASU0Ez8O9ArRNeoZzb75sEuReueow="
+      "version": "c0abc546c782a095a22c277d36f871bb94ffc944",
+      "sum": "Cc715Y3rgTuimgDFIw+FaKzXSJGRYwt1pFTMbdrNBD8="
     },
     {
       "source": {
@@ -88,7 +88,7 @@
           "subdir": "tanka-util"
         }
       },
-      "version": "af3ca2c3fae4096002b0c0c921f18ca7da8d361f",
+      "version": "c0abc546c782a095a22c277d36f871bb94ffc944",
       "sum": "ShSIissXdvCy1izTCDZX6tY7qxCoepE5L+WJ52Hw7ZQ="
     },
     {
@@ -108,8 +108,8 @@
           "subdir": "doc-util"
         }
       },
-      "version": "2eae33a828320269c42acf38e808479a33e416db",
-      "sum": "lppHbNARpG3YTpuSv94X9TyIE9TfV3CyTVceIHSRxpc="
+      "version": "7c865ec0606f2b68c0f6b2721f101e6a99cd2593",
+      "sum": "zjjufxN4yAIevldYEERiZEp27vK0BJKj1VvZcVtWiOo="
     },
     {
       "source": {
@@ -118,7 +118,7 @@
           "subdir": "1.20"
         }
       },
-      "version": "9e5b48eee32913938d3cac30f183b49ecd9fe13a",
+      "version": "44a9f3d21c089a01f62b22e25bdf553f488a74e8",
       "sum": "KXx5RVXiqTJQo2GVfrD8DIvlm292s0TxfTKT8I591+c="
     }
   ],
diff --git a/tools/dev/k3d/scripts/create_cluster.sh b/tools/dev/k3d/scripts/create_cluster.sh
index 8f83d0d1f0..ede41dc89f 100755
--- a/tools/dev/k3d/scripts/create_cluster.sh
+++ b/tools/dev/k3d/scripts/create_cluster.sh
@@ -37,7 +37,7 @@ fi
 # The following two sections pre-emptively apply the CRDs we need to the cluster.
 
 # Apply CRDs needed for prometheus operator
-prometheus_crd_base_url="https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd"
+prometheus_crd_base_url="https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.67.1/example/prometheus-operator-crd"
 for file in monitoring.coreos.com_alertmanagerconfigs.yaml \
 	monitoring.coreos.com_alertmanagers.yaml \
 	monitoring.coreos.com_podmonitors.yaml \
@@ -54,10 +54,7 @@ done
 
 # Apply CRDs needed for grafana agent
 agent_crd_base_url="https://raw.githubusercontent.com/grafana/agent/main/production/operator/crds"
-for file in monitoring.coreos.com_podmonitors.yaml \
-	monitoring.coreos.com_probes.yaml \
-	monitoring.coreos.com_servicemonitors.yaml \
-	monitoring.grafana.com_grafanaagents.yaml \
+for file in monitoring.grafana.com_grafanaagents.yaml \
 	monitoring.grafana.com_integrations.yaml \
 	monitoring.grafana.com_logsinstances.yaml \
 	monitoring.grafana.com_metricsinstances.yaml \