fix(monitoring): upgrade legacy templating queries for Grafana 11 and force Grafana recreation on deploy to load the Memento dashboard

monitoring/grafana-dashboards/node-exporter.json

@@ -7205,7 +7205,11 @@
         "multi": false,
         "name": "node",
         "options": [],
-        "query": "label_values(node_boot_time{job=\"node-exporter\"}, instance)",
+        "query": {
+          "qryType": 0,
+          "query": "label_values(node_boot_time{job=\"node-exporter\"}, instance)",
+          "refId": "PrometheusVariableQuery"
+        },
         "refresh": 1,
         "regex": "/([^:]+):.*/",
         "sort": 0,
@@ -7213,7 +7217,8 @@
         "tags": [],
         "tagsQuery": "",
         "type": "query",
-        "useTags": false
+        "useTags": false,
+        "definition": "label_values(node_boot_time{job=\"node-exporter\"}, instance)"
       }
     ]
   },

monitoring/grafana-dashboards/postgres.json

@@ -2937,7 +2937,11 @@
         "multi": false,
         "name": "namespace",
         "options": [],
-        "query": "query_result(pg_exporter_last_scrape_duration_seconds)",
+        "query": {
+          "qryType": 0,
+          "query": "query_result(pg_exporter_last_scrape_duration_seconds)",
+          "refId": "PrometheusVariableQuery"
+        },
         "refresh": 2,
         "regex": "/.*kubernetes_namespace=\"([^\"]+).*/",
         "sort": 1,
@@ -2945,7 +2949,8 @@
         "tags": [],
         "tagsQuery": "",
         "type": "query",
-        "useTags": false
+        "useTags": false,
+        "definition": "query_result(pg_exporter_last_scrape_duration_seconds)"
       },
       {
         "allValue": null,
@@ -2957,7 +2962,11 @@
         "multi": false,
         "name": "release",
         "options": [],
-        "query": "query_result(pg_exporter_last_scrape_duration_seconds{kubernetes_namespace=\"$namespace\"})",
+        "query": {
+          "qryType": 0,
+          "query": "query_result(pg_exporter_last_scrape_duration_seconds{kubernetes_namespace=\"$namespace\"})",
+          "refId": "PrometheusVariableQuery"
+        },
         "refresh": 2,
         "regex": "/.*release=\"([^\"]+)/",
         "sort": 1,
@@ -2965,7 +2974,8 @@
         "tags": [],
         "tagsQuery": "",
         "type": "query",
-        "useTags": false
+        "useTags": false,
+        "definition": "query_result(pg_exporter_last_scrape_duration_seconds{kubernetes_namespace=\"$namespace\"})"
       },
       {
         "allValue": null,
@@ -2977,7 +2987,11 @@
         "multi": false,
         "name": "instance",
         "options": [],
-        "query": "query_result(up{release=\"$release\"})",
+        "query": {
+          "qryType": 0,
+          "query": "query_result(up{release=\"$release\"})",
+          "refId": "PrometheusVariableQuery"
+        },
         "refresh": 1,
         "regex": "/.*instance=\"([^\"]+).*/",
         "sort": 1,
@@ -2985,7 +2999,8 @@
         "tags": [],
         "tagsQuery": "",
         "type": "query",
-        "useTags": false
+        "useTags": false,
+        "definition": "query_result(up{release=\"$release\"})"
       },
       {
         "allValue": null,
@@ -2997,7 +3012,11 @@
         "multi": true,
         "name": "datname",
         "options": [],
-        "query": "label_values(datname)",
+        "query": {
+          "qryType": 0,
+          "query": "label_values(datname)",
+          "refId": "PrometheusVariableQuery"
+        },
         "refresh": 1,
         "regex": "",
         "sort": 1,
@@ -3005,7 +3024,8 @@
         "tags": [],
         "tagsQuery": "",
         "type": "query",
-        "useTags": false
+        "useTags": false,
+        "definition": "label_values(datname)"
       },
       {
         "allValue": null,
@@ -3017,7 +3037,11 @@
         "multi": true,
         "name": "mode",
         "options": [],
-        "query": "label_values({mode=~\"accessexclusivelock|accesssharelock|exclusivelock|rowexclusivelock|rowsharelock|sharelock|sharerowexclusivelock|shareupdateexclusivelock\"}, mode)",
+        "query": {
+          "qryType": 0,
+          "query": "label_values({mode=~\"accessexclusivelock|accesssharelock|exclusivelock|rowexclusivelock|rowsharelock|sharelock|sharerowexclusivelock|shareupdateexclusivelock\"}, mode)",
+          "refId": "PrometheusVariableQuery"
+        },
         "refresh": 1,
         "regex": "",
         "sort": 0,
@@ -3025,7 +3049,8 @@
         "tags": [],
         "tagsQuery": "",
         "type": "query",
-        "useTags": false
+        "useTags": false,
+        "definition": "label_values({mode=~\"accessexclusivelock|accesssharelock|exclusivelock|rowexclusivelock|rowsharelock|sharelock|sharerowexclusivelock|shareupdateexclusivelock\"}, mode)"
       }
     ]
   },

monitoring/grafana-dashboards/redis.json

@@ -1052,10 +1052,15 @@
         "multi": false,
         "name": "addr",
         "options": [],
-        "query": "label_values(redis_connected_clients, addr)",
+        "query": {
+          "qryType": 0,
+          "query": "label_values(redis_connected_clients, addr)",
+          "refId": "PrometheusVariableQuery"
+        },
         "refresh": 1,
         "regex": "",
-        "type": "query"
+        "type": "query",
+        "definition": "label_values(redis_connected_clients, addr)"
       }
     ]
   },

scripts/deploy-prod.sh

@@ -158,6 +158,10 @@ for i in $(seq 1 "$HEALTH_CHECK_MAX_ITERATIONS"); do
       docker compose -f monitoring/docker-compose.monitoring.yml up -d --remove-orphans 2>&1 || echo "WARN: Failed to bring up monitoring stack"
     fi
 
+    # Force recreation of Grafana to reload physical JSON dashboards mounted from the host
+    echo "=== Recreating Grafana to load updated dashboards ==="
+    docker compose -f monitoring/docker-compose.monitoring.yml up -d --force-recreate grafana 2>&1 || true
+
     if docker ps --format '{{.Names}}' | grep -q "^memento-prometheus$"; then
       echo "=== Reloading Prometheus configuration ==="
       docker compose -f monitoring/docker-compose.monitoring.yml exec -T prometheus kill -SIGHUP 1 2>/dev/null || true