Skip to content

Update mixin to latest changes from grafana/postgres_exporter #1179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions postgres_mixin/.lint
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
---
exclusions:
panel-units-rule:
reason: Ignoring so far, need to address this in future
panel-title-description-rule:
reason: Ignoring so far, need to address this in future
panel-datasource-rule:
reason: "Loki datasource variable is being named as loki_datasource now while linter expects 'datasource'"
template-datasource-rule:
reason: "Based on new convention we are using variable names prometheus_datasource and loki_datasource where as linter expects 'datasource'"
alert-name-camelcase:
reason: QPS is a common acronym (Queries Per Second) and should be allowed
entries:
- alert: PostgreSQLQPS

189 changes: 163 additions & 26 deletions postgres_mixin/alerts/postgres.libsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@
{
alert: 'PostgreSQLMaxConnectionsReached',
annotations: {
description: '{{ $labels.instance }} is exceeding the currently configured maximum Postgres connection limit (current value: {{ $value }}s). Services may be degraded - please take immediate action (you probably need to increase max_connections in the Docker image and re-deploy.',
summary: '{{ $labels.instance }} has maxed out Postgres connections.',
description: '{{ $labels.instance }} is exceeding the currently configured maximum Postgres connection limit (current value: {{ $value }}s). Services may be degraded - please take immediate action (you probably need to increase max_connections in the Docker image and re-deploy).',
summary: 'Postgres connections count is over the maximum amount.',
},
expr: |||
sum by (instance) (pg_stat_activity_count{%(postgresExporterSelector)s})
sum by (%(agg)s) (pg_stat_activity_count{%(postgresExporterSelector)s})
>=
sum by (instance) (pg_settings_max_connections{%(postgresExporterSelector)s})
sum by (%(agg)s) (pg_settings_max_connections{%(postgresExporterSelector)s})
-
sum by (instance) (pg_settings_superuser_reserved_connections{%(postgresExporterSelector)s})
||| % $._config,
sum by (%(agg)s) (pg_settings_superuser_reserved_connections{%(postgresExporterSelector)s})
||| % $._config { agg: std.join(', ', $._config.groupLabels + $._config.instanceLabels) },
'for': '1m',
labels: {
severity: 'warning',
Expand All @@ -26,17 +26,17 @@
alert: 'PostgreSQLHighConnections',
annotations: {
description: '{{ $labels.instance }} is exceeding 80% of the currently configured maximum Postgres connection limit (current value: {{ $value }}s). Please check utilization graphs and confirm if this is normal service growth, abuse or an otherwise temporary condition or if new resources need to be provisioned (or the limits increased, which is mostly likely).',
summary: '{{ $labels.instance }} is over 80% of max Postgres connections.',
summary: 'Postgres connections count is over 80% of maximum amount.',
},
expr: |||
sum by (instance) (pg_stat_activity_count{%(postgresExporterSelector)s})
sum by (%(agg)s) (pg_stat_activity_count{%(postgresExporterSelector)s})
>
(
sum by (instance) (pg_settings_max_connections{%(postgresExporterSelector)s})
sum by (%(agg)s) (pg_settings_max_connections{%(postgresExporterSelector)s})
-
sum by (instance) (pg_settings_superuser_reserved_connections{%(postgresExporterSelector)s})
sum by (%(agg)s) (pg_settings_superuser_reserved_connections{%(postgresExporterSelector)s})
) * 0.8
||| % $._config,
||| % $._config { agg: std.join(', ', $._config.groupLabels + $._config.instanceLabels) },
'for': '10m',
labels: {
severity: 'warning',
Expand All @@ -46,7 +46,7 @@
alert: 'PostgreSQLDown',
annotations: {
description: '{{ $labels.instance }} is rejecting query requests from the exporter, and thus probably not allowing DNS requests to work either. User services should not be effected provided at least 1 node is still alive.',
summary: 'PostgreSQL is not processing queries: {{ $labels.instance }}',
summary: 'PostgreSQL is not processing queries.',
},
expr: 'pg_up{%(postgresExporterSelector)s} != 1' % $._config,
'for': '1m',
Expand All @@ -58,15 +58,15 @@
alert: 'PostgreSQLSlowQueries',
annotations: {
description: 'PostgreSQL high number of slow queries {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }} ',
summary: 'PostgreSQL high number of slow on {{ $labels.cluster }} for database {{ $labels.datname }} ',
summary: 'PostgreSQL high number of slow queries.',
},
expr: |||
avg by (datname) (
avg by (datname, %(agg)s) (
rate (
pg_stat_activity_max_tx_duration{datname!~"template.*",%(postgresExporterSelector)s}[2m]
pg_stat_activity_max_tx_duration{%(dbNameFilter)s, %(postgresExporterSelector)s}[2m]
)
) > 2 * 60
||| % $._config,
||| % $._config { agg: std.join(', ', $._config.groupLabels + $._config.instanceLabels) },
'for': '2m',
labels: {
severity: 'warning',
Expand All @@ -76,19 +76,19 @@
alert: 'PostgreSQLQPS',
annotations: {
description: 'PostgreSQL high number of queries per second on {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }}',
summary: 'PostgreSQL high number of queries per second {{ $labels.cluster }} for database {{ $labels.datname }}',
summary: 'PostgreSQL high number of queries per second.',
},
expr: |||
avg by (datname) (
avg by (datname, %(agg)s) (
irate(
pg_stat_database_xact_commit{datname!~"template.*",%(postgresExporterSelector)s}[5m]
pg_stat_database_xact_commit{%(dbNameFilter)s, %(postgresExporterSelector)s}[5m]
)
+
irate(
pg_stat_database_xact_rollback{datname!~"template.*",%(postgresExporterSelector)s}[5m]
pg_stat_database_xact_rollback{%(dbNameFilter)s, %(postgresExporterSelector)s}[5m]
)
) > 10000
||| % $._config,
||| % $._config { agg: std.join(', ', $._config.groupLabels + $._config.instanceLabels) },
'for': '5m',
labels: {
severity: 'warning',
Expand All @@ -98,28 +98,165 @@
alert: 'PostgreSQLCacheHitRatio',
annotations: {
description: 'PostgreSQL low on cache hit rate on {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }}',
summary: 'PostgreSQL low cache hit rate on {{ $labels.cluster }} for database {{ $labels.datname }}',
summary: 'PostgreSQL low cache hit rate.',
},
expr: |||
avg by (datname) (
rate(pg_stat_database_blks_hit{datname!~"template.*",%(postgresExporterSelector)s}[5m])
avg by (datname, %(agg)s) (
rate(pg_stat_database_blks_hit{%(dbNameFilter)s, %(postgresExporterSelector)s}[5m])
/
(
rate(
pg_stat_database_blks_hit{datname!~"template.*",%(postgresExporterSelector)s}[5m]
pg_stat_database_blks_hit{%(dbNameFilter)s, %(postgresExporterSelector)s}[5m]
)
+
rate(
pg_stat_database_blks_read{datname!~"template.*",%(postgresExporterSelector)s}[5m]
pg_stat_database_blks_read{%(dbNameFilter)s, %(postgresExporterSelector)s}[5m]
)
)
) < 0.98
||| % $._config { agg: std.join(', ', $._config.groupLabels + $._config.instanceLabels) },
'for': '5m',
labels: {
severity: 'warning',
},
},
{
alert: 'PostgresHasTooManyRollbacks',
annotations: {
description: 'PostgreSQL has too many rollbacks on {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }}',
summary: 'PostgreSQL has too many rollbacks.',
},
expr: |||
avg without(pod, instance)
(rate(pg_stat_database_xact_rollback{%(dbNameFilter)s}[5m]) /
(rate(pg_stat_database_xact_commit{%(dbNameFilter)s}[5m]) + rate(pg_stat_database_xact_rollback{%(dbNameFilter)s}[5m]))) > 0.10
||| % $._config,
'for': '5m',
labels: {
severity: 'warning',
},
},
{
alert: 'PostgresHasHighDeadLocks',
annotations: {
description: 'PostgreSQL has too high deadlocks on {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }}',
summary: 'PostgreSQL has high number of deadlocks.',
},
expr: |||
max without(pod, instance) (rate(pg_stat_database_deadlocks{%(dbNameFilter)s}[5m]) * 60) > 5
||| % $._config,
'for': '5m',
labels: {
severity: 'warning',
},
},
{
alert: 'PostgresAcquiredTooManyLocks',
annotations: {
description: 'PostgreSQL has acquired too many locks on {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }}',
summary: 'PostgreSQL has high number of acquired locks.',
},
expr: |||
max by(datname, %(agg)s) (
(pg_locks_count{%(dbNameFilter)s})
/
on(%(aggWithoutServer)s) group_left(server) (
pg_settings_max_locks_per_transaction{} * pg_settings_max_connections{}
)
) > 0.20
||| % $._config { agg: std.join(',', $._config.groupLabels + $._config.instanceLabels), aggWithoutServer: std.join(',', std.filter(function(x) x != "server", $._config.groupLabels + $._config.instanceLabels)) },
'for': '5m',
labels: {
severity: 'warning',
},
},
{
alert: 'PostgresReplicationLaggingMore1Hour',
annotations: {
description: '{{ $labels.instance }} replication lag exceeds 1 hour. Check for network issues or load imbalances.',
summary: 'PostgreSQL replication lagging more than 1 hour.',
},
expr: |||
(pg_replication_lag{} > 3600) and on (%(agg)s) (pg_replication_is_replica{} == 1)
||| % $._config { agg: std.join(', ', $._config.groupLabels + $._config.instanceLabels) },
'for': '5m',
labels: {
severity: 'warning',
},
},
{
alert: 'PostgresHasReplicationSlotUsed',
annotations: {
description: '{{ $labels.instance }} has replication slots that are not used, which might lead to replication lag or data inconsistency.',
summary: 'PostgreSQL has unused replication slots.',
},
expr: 'pg_replication_slots_active{} == 0',
'for': '30m',
labels: {
severity: 'critical',
},
},
{
alert: 'PostgresReplicationRoleChanged',
annotations: {
description: '{{ $labels.instance }} replication role has changed. Verify if this is expected or if it indicates a failover.',
summary: 'PostgreSQL replication role change detected.',
},
expr: 'pg_replication_is_replica{} and changes(pg_replication_is_replica{}[1m]) > 0',
labels: {
severity: 'warning',
},
},
{
alert: 'PostgresHasExporterErrors',
annotations: {
description: '{{ $labels.instance }} exporter is experiencing errors. Verify exporter health and configuration.',
summary: 'PostgreSQL exporter errors detected.',
},
expr: 'pg_exporter_last_scrape_error{} > 0',
'for': '30m',
labels: {
severity: 'critical',
},
},
{
alert: 'PostgresTablesNotVaccumed',
annotations: {
description: '{{ $labels.instance }} tables have not been vacuumed recently within the last hour, which may lead to performance degradation.',
summary: 'PostgreSQL tables not vacuumed.',
},
expr: |||
group without(pod, instance)(
timestamp(
pg_stat_user_tables_n_dead_tup{} >
pg_stat_user_tables_n_live_tup{}
* on(%(agg)s) group_left pg_settings_autovacuum_vacuum_scale_factor{}
+ on(%(agg)s) group_left pg_settings_autovacuum_vacuum_threshold{}
)
< time() - 36000
)
||| % $._config { agg: std.join(', ', $._config.groupLabels + $._config.instanceLabels) },
'for': '30m',
labels: {
severity: 'critical',
},
},
{
alert: 'PostgresTooManyCheckpointsRequested',
annotations: {
description: '{{ $labels.instance }} is requesting too many checkpoints, which may lead to performance degradation.',
summary: 'PostgreSQL too many checkpoints requested.',
},
expr: |||
rate(pg_stat_bgwriter_checkpoints_timed_total{}[5m]) /
(rate(pg_stat_bgwriter_checkpoints_timed_total{}[5m]) + rate(pg_stat_bgwriter_checkpoints_req_total{}[5m]))
< 0.5
|||,
'for': '5m',
labels: {
severity: 'warning',
},
},
],
},
],
Expand Down
4 changes: 4 additions & 0 deletions postgres_mixin/config.libsonnet
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
{
_config+:: {
dbNameFilter: 'datname!~"template.*"',
postgresExporterSelector: '',
groupLabels: if self.enableMultiCluster then ['job', 'cluster'] else ['job'],
instanceLabels: ['instance', 'server'],
enableMultiCluster: false,
},
}
Loading