diff --git a/grafana/postgres/v12/index-overview.json b/grafana/postgres/v12/index-overview.json index 6323a53f8d..dcf1768d54 100644 --- a/grafana/postgres/v12/index-overview.json +++ b/grafana/postgres/v12/index-overview.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 7, + "id": 15, "links": [], "panels": [ { @@ -26,7 +26,6 @@ "type": "grafana-postgresql-datasource", "uid": "pgwatch-metrics" }, - "description": "Adding indexes can have unexpected performance effects, so ensure that you test them first to verify that the recommendation has the desired effect.\n\nRequires `reco_add_index` metric. \n\nColumn Guide:\n- **Affected Queries**: number of queries affected by index creation.\n- **Heaviest Query**: the query with the biggest total execution time among all affected queries.\n- **Recommendation**: the recommended index ddl.", "fieldConfig": { "defaults": { "color": { @@ -40,37 +39,18 @@ "footer": { "reducers": [] }, - "inspect": false, - "wrapText": false + "inspect": false }, + "decimals": 2, + "displayName": "", "mappings": [], "thresholds": { "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - } + "steps": [] + }, + "unit": "short" }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Category" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - }, { "matcher": { "id": "byName", @@ -78,26 +58,31 @@ }, "properties": [ { - "id": "custom.width", - "value": 243 + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 }, { "id": "links", "value": [ { - "title": "Opens 'Table details' dashboard for this table", - "url": "d/table-details/5-table-details?orgId=1&var-dbname=${__data.fields[\"Source Name\"]}&var-table_full_name=${__value.raw}" + "targetBlank": true, + "title": "Opens 'Table details' dashboard", + "url": "/d/table-details?var-dbname=${__data.fields[\"dbname\"]}&var-table_full_name=${__data.fields[\"table_full_name\"]}" } ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Recommendation" - }, - "properties": [ + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "custom.width", + "value": 480 + }, { "id": "custom.inspect", "value": true @@ -107,57 +92,60 @@ { "matcher": { "id": "byName", - "options": "Source Name" + "options": "Index Size" }, "properties": [ { - "id": "custom.width", - "value": 116 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Object" - }, - "properties": [ + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.align", + "value": "left" + }, { "id": "custom.width", - "value": 163 + "value": 301 } ] }, { "matcher": { "id": "byName", - "options": "Heaviest Query" + "options": "Scans" }, "properties": [ { - "id": "custom.width", - "value": 187 + "id": "unit", + "value": "short" }, { - "id": "links", - "value": [ - { - "title": "Opens 'Single query details' dashboard for this query id", - "url": "/d/single-query-details?orgId=1&var-dbname=${__data.fields[\"Source Name\"]}&var-queryid=${__value.raw}&from=${__from}&to=${__to}" - } - ] + "id": "decimals", + "value": 0 + }, + { + "id": "custom.align", + "value": "left" } ] }, { "matcher": { "id": "byName", - "options": "Affected Queries" + "options": "Index Name" }, "properties": [ { "id": "custom.width", - "value": 144 + "value": 392 + }, + { + "id": "custom.inspect", + "value": true } ] } @@ -169,10 +157,9 @@ "x": 0, "y": 0 }, - "id": 12, + "id": 9, "options": { - "cellHeight": "md", - "enablePagination": false, + "cellHeight": "sm", "showHeader": true }, "pluginVersion": "12.3.1", @@ -184,17 +171,38 @@ }, "format": "table", "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", "rawQuery": true, - "rawSql": "SELECT\n dbname AS \"Source Name\",\n tag_data->>'object_name' AS \"Table Name\",\n data->>'affected_query_count' AS \"Affected Queries\",\n data->>'query_id' AS \"Heaviest Query\",\n data->>'recommendation' AS \"Recommendation\"\nFROM recommendations \nWHERE \n time in (\n select max(time) \n from recommendations \n where $__timeFilter(time) AND dbname in ($dbname) and tag_data->>'reco_topic' = 'create_index' \n group by dbname\n )\n AND dbname IN ($dbname) AND tag_data->>'reco_topic' = 'create_index' \nORDER BY (data->>'total_exec_time')::float8 DESC", + "rawSql": "WITH index_scan_changes AS (\n SELECT\n time,\n dbname,\n tag_data ->> 'index_full_name' AS index_full_name,\n tag_data ->> 'table_full_name' AS table_full_name,\n (data ->> 'idx_scan')::int8 - LEAD((data ->> 'idx_scan')::int8) OVER w AS idx_scans,\n (data ->> 'index_size_b')::int8 AS index_size_b,\n ROW_NUMBER() OVER w AS rn\n FROM\n index_stats\n WHERE\n dbname IN ($dbname)\n WINDOW w AS (\n PARTITION BY dbname,\n tag_data ->> 'table_full_name',\n tag_data ->> 'index_full_name'\n ORDER BY time DESC\n )\n),\ntotal_scans AS (\n SELECT\n dbname,\n table_full_name,\n index_full_name,\n SUM(COALESCE(idx_scans, 0)) AS total_idx_scans,\n MAX(index_size_b) FILTER (WHERE rn = 1) AS latest_index_size\n FROM\n index_scan_changes\n WHERE\n $__timeFilter(time)\n GROUP BY\n dbname, table_full_name, index_full_name\n)\nSELECT\n dbname AS \"Source Name\",\n index_full_name AS \"Index Name\",\n table_full_name AS \"Table Name\",\n latest_index_size AS \"Index Size\",\n total_idx_scans AS \"Scans\"\nFROM\n total_scans\nORDER BY\n latest_index_size DESC\nLIMIT $top", "refId": "A", + "resultFormat": "table", "select": [ [ { "params": [ "value" ], - "type": "column" + "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -215,6 +223,7 @@ ], "limit": 50 }, + "tags": [], "timeColumn": "time", "where": [ { @@ -225,7 +234,7 @@ ] } ], - "title": "Missing Indexes", + "title": "Top $top biggest indexes", "transformations": [ { "id": "merge", @@ -241,24 +250,46 @@ "type": "grafana-postgresql-datasource", "uid": "pgwatch-metrics" }, - "description": "Requires `reco_drop_index` metric.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false }, - "footer": { - "reducers": [] + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "inspect": false + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "decimals": 2, - "displayName": "", "mappings": [], "thresholds": { "mode": "absolute", @@ -266,213 +297,602 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Index def" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "with index_stats_changes as (\n select\n time,\n dbname,\n tag_data->>'index_full_name' as index_full_name,\n coalesce((data->>'idx_blks_read')::int8 - lag((data->>'idx_blks_read')::int8) over w, 0) as blks_read\n from index_stats \n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname, tag_data->>'index_full_name' order by time)\n), top_indexes as (\n select\n dbname,\n index_full_name,\n sum(blks_read) as total_blks_read\n from index_stats_changes\n group by dbname, index_full_name\n order by total_blks_read desc\n limit $top\n)\nselect \n $__timeGroup(time, $agg_interval),\n sum(blks_read) as blks_read,\n dbname || ' ' || index_full_name as metric\nfrom index_stats_changes\nwhere dbname in (select dbname from top_indexes)\nand index_full_name in (select index_full_name from top_indexes)\ngroup by 1, dbname, index_full_name\norder by 1, dbname, index_full_name", + "refId": "A", + "sql": { + "columns": [ { - "id": "custom.align", - "value": "left" - }, + "parameters": [], + "type": "function" + } + ], + "groupBy": [ { - "id": "custom.inspect", - "value": true + "property": { + "type": "string" + }, + "type": "groupBy" } - ] + ], + "limit": 50 + } + } + ], + "title": "Top $top indexes by disk reads (blocks)", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byName", - "options": "Max Index size" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false }, - "properties": [ - { - "id": "unit", - "value": "bytes" - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "custom.align", - "value": "left" - }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "custom.width", - "value": 150 + "color": "green", + "value": 0 } ] }, - { - "matcher": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "with index_stats_changes as (\n select\n time,\n dbname,\n tag_data->>'index_full_name' as index_full_name,\n coalesce((data->>'idx_blks_hit')::int8 - lag((data->>'idx_blks_hit')::int8) over w, 0) as blks_hit\n from index_stats \n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname, tag_data->>'index_full_name' order by time)\n), top_indexes as (\n select\n dbname,\n index_full_name,\n sum(blks_hit) as total_blks_hit\n from index_stats_changes\n group by dbname, index_full_name\n order by total_blks_hit desc\n limit $top\n)\nselect \n $__timeGroup(time, $agg_interval),\n sum(blks_hit) as blks_hit,\n dbname || ' ' || index_full_name as metric\nfrom index_stats_changes\nwhere dbname in (select dbname from top_indexes)\nand index_full_name in (select index_full_name from top_indexes)\ngroup by 1, dbname, index_full_name\norder by 1, dbname, index_full_name", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Top $top indexes by cache hits (blocks)", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "with index_stats_changes as (\n select\n time,\n dbname,\n tag_data->>'index_full_name' as index_full_name,\n coalesce((data->>'idx_tup_read')::int8 - lag((data->>'idx_tup_read')::int8) over w, 0) as tuples_read\n from index_stats \n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname, tag_data->>'index_full_name' order by time)\n), top_indexes as (\n select\n dbname,\n index_full_name,\n sum(tuples_read) as total_tuples_read\n from index_stats_changes\n group by dbname, index_full_name\n order by total_tuples_read desc\n limit $top\n)\nselect \n $__timeGroup(time, $agg_interval),\n sum(tuples_read) as tuples_read,\n dbname || ' ' || index_full_name as metric\nfrom index_stats_changes\nwhere dbname in (select dbname from top_indexes)\nand index_full_name in (select index_full_name from top_indexes)\ngroup by 1, dbname, index_full_name\norder by 1, dbname, index_full_name", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "description": "Tuples Read = Number of index entries returned by scans on this index", + "title": "Top $top indexes by number of tuples read", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "with index_stats_changes as (\n select\n time,\n dbname,\n tag_data->>'index_full_name' as index_full_name,\n coalesce((data->>'idx_tup_fetch')::int8 - lag((data->>'idx_tup_fetch')::int8) over w, 0) as tuples_fetched\n from index_stats \n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname, tag_data->>'index_full_name' order by time)\n), top_indexes as (\n select\n dbname,\n index_full_name,\n sum(tuples_fetched) as total_tuples_fetched\n from index_stats_changes\n group by dbname, index_full_name\n order by total_tuples_fetched desc\n limit $top\n)\nselect \n $__timeGroup(time, $agg_interval),\n sum(tuples_fetched) as tuples_fetched,\n dbname || ' ' || index_full_name as metric\nfrom index_stats_changes\nwhere dbname in (select dbname from top_indexes)\nand index_full_name in (select index_full_name from top_indexes)\ngroup by 1, dbname, index_full_name\norder by 1, dbname, index_full_name", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "description": "Tuples Fetched = Number of live rows fetched from the table by simple index scans using this index", + "title": "Top $top indexes by number of tuples fetched", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "description": "Adding indexes can have unexpected performance effects, so ensure that you test them first to verify that the recommendation has the desired effect.\n\nRequires `reco_add_index` metric. \n\nColumn Guide:\n- **Affected Queries**: number of queries affected by index creation.\n- **Heaviest Query**: the query with the biggest total execution time among all affected queries.\n- **Recommendation**: the recommended index ddl.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false, + "wrapText": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { "id": "byName", - "options": "Table Name" + "options": "Category" }, "properties": [ { - "id": "unit", - "value": "short" - }, + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Table Name" + }, + "properties": [ { - "id": "decimals", - "value": 2 + "id": "custom.width", + "value": 243 }, { "id": "links", "value": [ { - "targetBlank": true, - "title": "Opens 'Table details' dashboard", - "url": "/d/table-details?var-dbname=${__data.fields[\"dbname\"]}&var-table_full_name=${__data.fields[\"table_full_name\"]}" + "title": "Opens 'Table details' dashboard for this table", + "url": "d/table-details/5-table-details?orgId=1&var-dbname=${__data.fields[\"Source Name\"]}&var-table_full_name=${__value.raw}" } ] - }, - { - "id": "custom.align", - "value": "left" - }, - { - "id": "custom.width", - "value": 236 } ] }, { "matcher": { "id": "byName", - "options": "Count" + "options": "Recommendation" }, "properties": [ { - "id": "unit", - "value": "short" - }, - { - "id": "custom.align", - "value": "left" - }, - { - "id": "decimals", - "value": 0 - }, + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Object" + }, + "properties": [ { "id": "custom.width", - "value": 106 + "value": 163 } ] }, { "matcher": { "id": "byName", - "options": "Source Name" + "options": "Heaviest Query" }, "properties": [ { "id": "custom.width", - "value": 121 + "value": 187 + }, + { + "id": "links", + "value": [ + { + "title": "Opens 'Single query details' dashboard for this query id", + "url": "/d/single-query-details?orgId=1&var-dbname=${__data.fields[\"Source Name\"]}&var-queryid=${__value.raw}&from=${__from}&to=${__to}" + } + ] } ] }, { "matcher": { "id": "byName", - "options": "Num of duplicates" + "options": "Affected Queries" }, "properties": [ { "id": "custom.width", - "value": 154 + "value": 144 } ] }, { "matcher": { "id": "byName", - "options": "OIDs" + "options": "Source Name" }, "properties": [ { "id": "custom.width", - "value": 312 + "value": 151 } ] } ] }, "gridPos": { - "h": 5, + "h": 8, "w": 24, "x": 0, - "y": 8 + "y": 24 }, - "id": 8, + "id": 12, "options": { - "cellHeight": "sm", - "showHeader": true - }, - "pluginVersion": "12.3.1", - "targets": [ - { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "pgwatch-metrics" - }, - "editorMode": "code", - "format": "table", - "group": [], - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], + "cellHeight": "md", + "enablePagination": false, + "showHeader": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "format": "table", + "group": [], "metricColumn": "none", - "orderByTime": "ASC", - "policy": "default", "rawQuery": true, - "rawSql": "SELECT\n dbname AS \"Source Name\",\n tag_data->>'table_full_name' as \"Table Name\",\n (data->>'index_size_b')::int8 as \"Max Index size\",\n data->>'duplicate_count' as \"Num of duplicates\",\n data->>'oids' as \"OIDs\",\n data->>'index_def' as \"Index def\"\nFROM\n recommendations \nWHERE \n time in (\n select max(time) \n from recommendations \n where $__timeFilter(time) AND dbname in ($dbname) \n and tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'duplicate' \n group by dbname\n )\n AND dbname IN ($dbname) AND tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'duplicate'\n\n", + "rawSql": "SELECT\n dbname as \"Source Name\",\n tag_data->>'object_name' AS \"Table Name\",\n data->>'affected_query_count' AS \"Affected Queries\",\n data->>'query_id' AS \"Heaviest Query\",\n data->>'recommendation' AS \"Recommendation\"\nFROM recommendations \nWHERE \n time in (\n select max(time) \n from recommendations \n where $__timeFilter(time) AND dbname in ($dbname) and tag_data->>'reco_topic' = 'create_index' \n group by dbname\n )\n AND dbname IN ($dbname) AND tag_data->>'reco_topic' = 'create_index' \nORDER BY (data->>'total_exec_time')::float8 DESC", "refId": "A", - "resultFormat": "table", "select": [ [ { "params": [ "value" ], - "type": "field" - }, - { - "params": [], - "type": "mean" + "type": "column" } ] ], @@ -493,7 +913,6 @@ ], "limit": 50 }, - "tags": [], "timeColumn": "time", "where": [ { @@ -504,7 +923,7 @@ ] } ], - "title": "Duplicate Indexes", + "title": "Missing Indexes", "transformations": [ { "id": "merge", @@ -520,7 +939,7 @@ "type": "grafana-postgresql-datasource", "uid": "pgwatch-metrics" }, - "description": "Before dropping indexes make sure they are not used on replicas, if having any. FYI - one could also adjust the default 0 threshold so that \"little used\" indexes are shown.\n\nRequires `reco_drop_index` metric.", + "description": "Requires `reco_drop_index` metric.", "fieldConfig": { "defaults": { "color": { @@ -572,12 +991,15 @@ ] }, { - "id": "custom.align", - "value": "left" + "id": "custom.align" }, { "id": "custom.width", - "value": 385 + "value": 245 + }, + { + "id": "custom.inspect", + "value": true } ] }, @@ -601,18 +1023,6 @@ } ] }, - { - "matcher": { - "id": "byName", - "options": "Source Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 218 - } - ] - }, { "matcher": { "id": "byName", @@ -621,38 +1031,27 @@ "properties": [ { "id": "custom.width", - "value": 448 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Scans" - }, - "properties": [ - { - "id": "decimals", - "value": 0 + "value": 203 }, { - "id": "custom.align", - "value": "left" + "id": "custom.inspect", + "value": true } ] } ] }, "gridPos": { - "h": 5, - "w": 24, + "h": 7, + "w": 12, "x": 0, - "y": 13 + "y": 32 }, - "id": 10, + "id": 11, "options": { "cellHeight": "sm", - "showHeader": true + "showHeader": true, + "sortBy": [] }, "pluginVersion": "12.3.1", "targets": [ @@ -681,7 +1080,7 @@ "orderByTime": "ASC", "policy": "default", "rawQuery": true, - "rawSql": "SELECT\n dbname AS \"Source Name\",\n tag_data->>'table_full_name' as \"Table Name\",\n tag_data->>'object_name' as \"Index Name\",\n (data->>'index_size_b')::int8 as \"Index size\",\n 0::int8 as \"Scans\"\nFROM\n recommendations \nWHERE \n time in (\n select max(time) \n from recommendations \n where $__timeFilter(time) AND dbname in ($dbname) \n and tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'unused' \n group by dbname\n )\n AND dbname IN ($dbname) AND tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'unused'\n", + "rawSql": "SELECT\n dbname as \"Source Name\",\n tag_data->>'table_full_name' as \"Table Name\",\n tag_data->>'object_name' as \"Index Name\",\n (data->>'index_size_b')::int8 as \"Index size\"\nFROM\n recommendations \nWHERE \n time in (\n select max(time) \n from recommendations \n where $__timeFilter(time) AND dbname in ($dbname) \n and tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'invalid' \n group by dbname\n )\n AND dbname IN ($dbname) AND tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'invalid'\n", "refId": "A", "resultFormat": "table", "select": [ @@ -698,6 +1097,23 @@ } ] ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, "tags": [], "timeColumn": "time", "where": [ @@ -709,7 +1125,7 @@ ] } ], - "title": "Unused indexes (Non-PK)", + "title": "Invalid indexes", "transformations": [ { "id": "merge", @@ -725,7 +1141,7 @@ "type": "grafana-postgresql-datasource", "uid": "pgwatch-metrics" }, - "description": "Requires `reco_drop_index` metric.", + "description": "Before dropping indexes make sure they are not used on replicas, if having any. FYI - one could also adjust the default 0 threshold so that \"little used\" indexes are shown.\n\nRequires `reco_drop_index` metric.", "fieldConfig": { "defaults": { "color": { @@ -777,11 +1193,16 @@ ] }, { - "id": "custom.align" + "id": "custom.align", + "value": "left" }, { "id": "custom.width", - "value": 385 + "value": 173 + }, + { + "id": "custom.inspect", + "value": true } ] }, @@ -802,45 +1223,70 @@ { "id": "custom.align", "value": "left" + }, + { + "id": "custom.width", + "value": 126 } ] }, { "matcher": { "id": "byName", - "options": "Source Name" + "options": "Index Name" }, "properties": [ { "id": "custom.width", - "value": 221 + "value": 179 + }, + { + "id": "custom.inspect", + "value": true } ] }, { "matcher": { "id": "byName", - "options": "Index Name" + "options": "Scans" + }, + "properties": [ + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Source Name" }, "properties": [ { "id": "custom.width", - "value": 449 + "value": 137 } ] } ] }, "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 18 + "h": 7, + "w": 12, + "x": 12, + "y": 32 }, - "id": 11, + "id": 10, "options": { "cellHeight": "sm", - "showHeader": true + "showHeader": true, + "sortBy": [] }, "pluginVersion": "12.3.1", "targets": [ @@ -869,7 +1315,7 @@ "orderByTime": "ASC", "policy": "default", "rawQuery": true, - "rawSql": "SELECT\n dbname AS \"Source Name\",\n tag_data->>'table_full_name' as \"Table Name\",\n tag_data->>'object_name' as \"Index Name\",\n (data->>'index_size_b')::int8 as \"Index size\"\nFROM\n recommendations \nWHERE \n time in (\n select max(time) \n from recommendations \n where $__timeFilter(time) AND dbname in ($dbname) \n and tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'invalid' \n group by dbname\n )\n AND dbname IN ($dbname) AND tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'invalid'\n", + "rawSql": "SELECT\n dbname as \"Source Name\",\n tag_data->>'table_full_name' as \"Table Name\",\n tag_data->>'object_name' as \"Index Name\",\n (data->>'index_size_b')::int8 as \"Index size\",\n 0::int8 as \"Scans\"\nFROM\n recommendations \nWHERE \n time in (\n select max(time) \n from recommendations \n where $__timeFilter(time) AND dbname in ($dbname) \n and tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'unused' \n group by dbname\n )\n AND dbname IN ($dbname) AND tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'unused'\n", "refId": "A", "resultFormat": "table", "select": [ @@ -886,6 +1332,23 @@ } ] ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, "tags": [], "timeColumn": "time", "where": [ @@ -897,7 +1360,7 @@ ] } ], - "title": "Invalid indexes", + "title": "Unused indexes (Non-PK)", "transformations": [ { "id": "merge", @@ -913,6 +1376,7 @@ "type": "grafana-postgresql-datasource", "uid": "pgwatch-metrics" }, + "description": "Requires `reco_drop_index` metric.", "fieldConfig": { "defaults": { "color": { @@ -950,7 +1414,7 @@ { "matcher": { "id": "byName", - "options": "Table Name" + "options": "Index def" }, "properties": [ { @@ -961,30 +1425,20 @@ "id": "decimals", "value": 2 }, - { - "id": "links", - "value": [ - { - "targetBlank": true, - "title": "Opens 'Table details' dashboard", - "url": "/d/table-details?var-dbname=${__data.fields[\"dbname\"]}&var-table_full_name=${__data.fields[\"table_full_name\"]}" - } - ] - }, { "id": "custom.align", "value": "left" }, { - "id": "custom.width", - "value": 432 + "id": "custom.inspect", + "value": true } ] }, { "matcher": { "id": "byName", - "options": "Index Size" + "options": "Max Index size" }, "properties": [ { @@ -1001,14 +1455,14 @@ }, { "id": "custom.width", - "value": 136 + "value": 150 } ] }, { "matcher": { "id": "byName", - "options": "Scans (from last reset)" + "options": "Table Name" }, "properties": [ { @@ -1017,7 +1471,17 @@ }, { "id": "decimals", - "value": 0 + "value": 2 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Opens 'Table details' dashboard", + "url": "/d/table-details?var-dbname=${__data.fields[\"dbname\"]}&var-table_full_name=${__data.fields[\"table_full_name\"]}" + } + ] }, { "id": "custom.align", @@ -1025,43 +1489,63 @@ }, { "id": "custom.width", - "value": 202 + "value": 222 + }, + { + "id": "custom.inspect", + "value": true } ] }, { "matcher": { "id": "byName", - "options": "Source Name" + "options": "Num of duplicates" }, "properties": [ { "id": "custom.width", - "value": 137 + "value": 154 } ] }, { "matcher": { "id": "byName", - "options": "Index Name" + "options": "OIDs" + }, + "properties": [ + { + "id": "custom.width", + "value": 244 + }, + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Source Name" }, "properties": [ { "id": "custom.width", - "value": 537 + "value": 146 } ] } ] }, "gridPos": { - "h": 8, + "h": 6, "w": 24, "x": 0, - "y": 22 + "y": 39 }, - "id": 9, + "id": 8, "options": { "cellHeight": "sm", "showHeader": true @@ -1073,7 +1557,6 @@ "type": "grafana-postgresql-datasource", "uid": "pgwatch-metrics" }, - "editorMode": "code", "format": "table", "group": [], "groupBy": [ @@ -1094,7 +1577,7 @@ "orderByTime": "ASC", "policy": "default", "rawQuery": true, - "rawSql": "SELECT\n dbname as \"Source Name\",\n tag_data->>'index_full_name' as \"Index Name\",\n tag_data->>'table_full_name' as \"Table Name\",\n max((data->>'index_size_b')::int8) as \"Index Size\",\n max((data->>'idx_scan')::int8) as \"Scans (from last reset)\"\nFROM\n index_stats\nWHERE\n time in (select max(time) from index_stats where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\nGROUP BY\n 1, 2, 3\nORDER BY\n 4 DESC\nLIMIT\n 20", + "rawSql": "SELECT\n dbname as \"Source Name\",\n tag_data->>'table_full_name' as \"Table Name\",\n (data->>'index_size_b')::int8 as \"Max Index size\",\n data->>'duplicate_count' as \"Num of duplicates\",\n data->>'oids' as \"OIDs\",\n data->>'index_def' as \"Index def\"\nFROM\n recommendations \nWHERE \n time in (\n select max(time) \n from recommendations \n where $__timeFilter(time) AND dbname in ($dbname) \n and tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'duplicate' \n group by dbname\n )\n AND dbname IN ($dbname) AND tag_data->>'reco_topic' = 'drop_index' and tag_data->>'issue_type' = 'duplicate'\n\n", "refId": "A", "resultFormat": "table", "select": [ @@ -1111,23 +1594,6 @@ } ] ], - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, "tags": [], "timeColumn": "time", "where": [ @@ -1139,7 +1605,7 @@ ] } ], - "title": "Top 20 biggest indexes", + "title": "Duplicate Indexes", "transformations": [ { "id": "merge", @@ -1156,10 +1622,10 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 7, "w": 24, "x": 0, - "y": 30 + "y": 45 }, "id": 6, "options": { @@ -1206,16 +1672,109 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "10", + "value": "10" + }, + "label": "Top", + "name": "top", + "options": [ + { + "selected": true, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + }, + { + "selected": false, + "text": "30", + "value": "30" + }, + { + "selected": false, + "text": "50", + "value": "50" + }, + { + "selected": false, + "text": "100", + "value": "100" + } + ], + "query": "10,15,30,50,100", + "type": "custom" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "label": "Aggregate Interval", + "name": "agg_interval", + "options": [ + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "type": "interval" } ] }, "time": { - "from": "now-1h", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Index overview", "uid": "index-overview", - "version": 22 + "version": 70 } \ No newline at end of file diff --git a/internal/metrics/metrics.yaml b/internal/metrics/metrics.yaml index 835da41446..66bc503c49 100644 --- a/internal/metrics/metrics.yaml +++ b/internal/metrics/metrics.yaml @@ -703,49 +703,30 @@ metrics: /* biggest */ select * from ( - select indexrelid - from q_index_details - where idx_scan > 1 - order by index_size_b desc - limit 200 - ) x + select indexrelid + from q_index_details + where idx_scan > 1 + order by index_size_b desc + limit 100 + ) x union /* most block traffic */ select * from ( - select indexrelid - from q_index_details - order by coalesce(idx_blks_read, 0) + coalesce(idx_blks_hit, 0) desc - limit 200 - ) y + select indexrelid + from q_index_details + order by coalesce(idx_blks_read, 0) + coalesce(idx_blks_hit, 0) desc + limit 100 + ) y union /* most scans */ select * from ( - select indexrelid - from q_index_details - order by idx_scan desc nulls last - limit 200 - ) z - union - /* biggest unused non-constraint */ - select * - from ( - select q.indexrelid - from q_index_details q - where idx_scan = 0 - and not (indisprimary or indisunique or indisexclusion) - order by index_size_b desc - limit 200 - ) z - union - /* all invalid */ - select * - from ( - select q.indexrelid - from q_index_details q - where not indisvalid - ) zz + select indexrelid + from q_index_details + order by idx_scan desc nulls last + limit 100 + ) z ) select /* pgwatch_generated */ (extract(epoch from now()) * 1e9)::int8 as epoch_ns, @@ -757,6 +738,8 @@ metrics: coalesce(idx_scan, 0) as idx_scan, coalesce(idx_tup_read, 0) as idx_tup_read, coalesce(idx_tup_fetch, 0) as idx_tup_fetch, + coalesce(idx_blks_read, 0) as idx_blks_read, + coalesce(idx_blks_hit, 0) as idx_blks_hit, coalesce(index_size_b, 0) as index_size_b, quote_ident(schemaname)||'.'||quote_ident(indexrelname) as index_full_name_val, md5(regexp_replace(regexp_replace(pg_get_indexdef(indexrelid),indexrelname,'X'), '^CREATE UNIQUE','CREATE')) as tag_index_def_hash, @@ -806,30 +789,30 @@ metrics: /* biggest */ select * from ( - select indexrelid - from q_index_details - where idx_scan > 1 - order by index_size_b desc - limit 200 - ) x + select indexrelid + from q_index_details + where idx_scan > 1 + order by index_size_b desc + limit 100 + ) x union /* most block traffic */ select * from ( - select indexrelid - from q_index_details - order by coalesce(idx_blks_read, 0) + coalesce(idx_blks_hit, 0) desc - limit 200 - ) y + select indexrelid + from q_index_details + order by coalesce(idx_blks_read, 0) + coalesce(idx_blks_hit, 0) desc + limit 100 + ) y union /* most scans */ select * from ( - select indexrelid - from q_index_details - order by idx_scan desc nulls last - limit 200 - ) z + select indexrelid + from q_index_details + order by idx_scan desc nulls last + limit 100 + ) z ) select /* pgwatch_generated */ (extract(epoch from now()) * 1e9)::int8 as epoch_ns, @@ -841,6 +824,8 @@ metrics: coalesce(idx_scan, 0) as idx_scan, coalesce(idx_tup_read, 0) as idx_tup_read, coalesce(idx_tup_fetch, 0) as idx_tup_fetch, + coalesce(idx_blks_read, 0) as idx_blks_read, + coalesce(idx_blks_hit, 0) as idx_blks_hit, coalesce(index_size_b, 0) as index_size_b, quote_ident(schemaname)||'.'||quote_ident(indexrelname) as index_full_name_val, md5(regexp_replace(regexp_replace(pg_get_indexdef(indexrelid),indexrelname,'X'), '^CREATE UNIQUE','CREATE')) as tag_index_def_hash,