2866 lines
69 KiB
JSON
2866 lines
69 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": "-- Grafana --",
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"type": "dashboard"
|
|
},
|
|
{
|
|
"datasource": "-- Grafana --",
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "#73BF69",
|
|
"limit": 100,
|
|
"name": "PromEx service start",
|
|
"showIn": 0,
|
|
"tags": ["prom_ex", "pinchflat", "start"],
|
|
"type": "tags"
|
|
},
|
|
{
|
|
"datasource": "-- Grafana --",
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "#FF9830",
|
|
"limit": 100,
|
|
"name": "PromEx service stop",
|
|
"showIn": 0,
|
|
"tags": ["prom_ex", "pinchflat", "stop"],
|
|
"type": "tags"
|
|
}
|
|
]
|
|
},
|
|
"description": "All the data that is presented here is captured by the PromEx Oban plugin (https://github.com/akoutmos/prom_ex/blob/master/lib/prom_ex/plugins/oban.ex)",
|
|
"editable": false,
|
|
"gnetId": null,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [
|
|
{
|
|
"asDropdown": false,
|
|
"icon": "bolt",
|
|
"includeVars": false,
|
|
"keepTime": false,
|
|
"tags": [],
|
|
"targetBlank": true,
|
|
"title": "Sponsor PromEx",
|
|
"tooltip": "",
|
|
"type": "link",
|
|
"url": "https://github.com/sponsors/akoutmos"
|
|
},
|
|
{
|
|
"asDropdown": false,
|
|
"icon": "doc",
|
|
"includeVars": false,
|
|
"keepTime": false,
|
|
"tags": [],
|
|
"targetBlank": true,
|
|
"title": "Oban Plugin Docs",
|
|
"tooltip": "",
|
|
"type": "link",
|
|
"url": "https://hexdocs.pm/prom_ex/PromEx.Plugins.Oban.html"
|
|
}
|
|
],
|
|
"panels": [
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 19,
|
|
"panels": [],
|
|
"title": "Overview",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "The configuration of the selected Oban instance",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": null,
|
|
"displayMode": "auto"
|
|
},
|
|
"decimals": 1,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "ms"
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Config Setting"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "mappings",
|
|
"value": [
|
|
{
|
|
"from": "",
|
|
"id": 1,
|
|
"text": "Cooldown",
|
|
"to": "",
|
|
"type": 1,
|
|
"value": "pinchflat_prom_ex_oban_init_dispatch_cooldown_milliseconds"
|
|
},
|
|
{
|
|
"from": "",
|
|
"id": 2,
|
|
"text": "Global Poll Interval",
|
|
"to": "",
|
|
"type": 1,
|
|
"value": "pinchflat_prom_ex_oban_init_poll_interval_milliseconds"
|
|
},
|
|
{
|
|
"from": "",
|
|
"id": 3,
|
|
"text": "Shutdown Grace Period",
|
|
"to": "",
|
|
"type": 1,
|
|
"value": "pinchflat_prom_ex_oban_init_shutdown_grace_period_milliseconds"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 5,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"id": 50,
|
|
"options": {
|
|
"showHeader": true
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "{__name__=~\"pinchflat_prom_ex_oban_init_shutdown_grace_period_milliseconds|pinchflat_prom_ex_oban_init_poll_interval_milliseconds|pinchflat_prom_ex_oban_init_dispatch_cooldown_milliseconds\", job=\"$job\", instance=\"$instance\", name=\"$oban\"}",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Oban Time Limit Settings",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"Time": true,
|
|
"__name__": false,
|
|
"instance": true,
|
|
"job": true,
|
|
"name": true
|
|
},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"Value": "Time Value",
|
|
"__name__": "Config Setting"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "table"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "The number of jobs in each queue that are in the available state.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {},
|
|
"mappings": [],
|
|
"max": 500,
|
|
"noValue": "NA",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "#EAB839",
|
|
"value": 400
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 500
|
|
}
|
|
]
|
|
},
|
|
"unit": "locale"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 12,
|
|
"w": 7,
|
|
"x": 5,
|
|
"y": 1
|
|
},
|
|
"id": 60,
|
|
"options": {
|
|
"displayMode": "lcd",
|
|
"orientation": "horizontal",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"showUnfilled": true
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "pinchflat_prom_ex_oban_queue_length_count{job=\"$job\", instance=\"$instance\", name=\"$oban\", state=\"available\"}",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "{{ queue }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Available Jobs in Queue",
|
|
"type": "bargauge"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "The configuration of the selected Oban instance",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": null,
|
|
"displayMode": "auto"
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"id": 49,
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": []
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "{__name__=\"pinchflat_prom_ex_oban_init_status_info\", job=\"$job\", instance=\"$instance\", name=\"$oban\"}",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Oban Configuration",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"Time": true,
|
|
"Value": true,
|
|
"__name__": true,
|
|
"instance": true,
|
|
"job": true
|
|
},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"name": "Oban Instance",
|
|
"node": "Node",
|
|
"plugins": "Configured Plugins",
|
|
"prefix": "Postgres Schema",
|
|
"queues": "Configured Queues",
|
|
"repo": "Ecto Repo"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "table"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "The configured concurrency limits for each of the queues under the Oban supervisor.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": null
|
|
},
|
|
"decimals": 1,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "locale"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 5,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"id": 36,
|
|
"options": {
|
|
"showHeader": true
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "pinchflat_prom_ex_oban_init_queue_concurrency_limit{instance=\"$instance\", job=\"$job\", name=\"$oban\"}",
|
|
"format": "table",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Queue Concurrency Limits",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"Time": true,
|
|
"__name__": true,
|
|
"instance": true,
|
|
"job": true,
|
|
"name": true
|
|
},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"Value": "Concurrent Workers",
|
|
"queue": "Queue"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "table"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "The number of jobs processed across all queues and states.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {},
|
|
"decimals": 1,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "locale"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 12,
|
|
"y": 7
|
|
},
|
|
"id": 45,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "round(sum(increase(pinchflat_prom_ex_oban_job_processing_duration_milliseconds_count{instance=\"$instance\", job=\"$job\", name=\"$oban\"}[$interval])))",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A",
|
|
"format": "table"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Processed Jobs (by $interval interval)",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "The number of jobs that resulted in an error across all queues.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {},
|
|
"decimals": 1,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "locale"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 16,
|
|
"y": 7
|
|
},
|
|
"id": 46,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "round(sum(increase(pinchflat_prom_ex_oban_job_exception_duration_milliseconds_count{instance=\"$instance\", job=\"$job\", name=\"$oban\"}[$interval])))",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A",
|
|
"format": "table"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Job Errors (by $interval interval)",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "The number of jobs that have been enqueued.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {},
|
|
"decimals": 1,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "locale"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 20,
|
|
"y": 7
|
|
},
|
|
"id": 47,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "round(sum(increase(pinchflat_prom_ex_oban_producer_dispatched_count_count{instance=\"$instance\", job=\"$job\", name=\"$oban\"}[$interval])))",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A",
|
|
"format": "table"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Jobs Enqueued (by $interval interval)",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"id": 2,
|
|
"panels": [],
|
|
"title": "Successful Job Processing Details",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"cards": {
|
|
"cardPadding": null,
|
|
"cardRound": null
|
|
},
|
|
"color": {
|
|
"cardColor": "#b4ff00",
|
|
"colorScale": "sqrt",
|
|
"colorScheme": "interpolateOranges",
|
|
"exponent": 0.5,
|
|
"mode": "spectrum"
|
|
},
|
|
"dataFormat": "tsbuckets",
|
|
"datasource": "prometheus",
|
|
"description": "A heatmap showing the time it took to process jobs across all queues.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"heatmap": {},
|
|
"hideZeroBuckets": true,
|
|
"highlightCards": true,
|
|
"id": 6,
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"reverseYBuckets": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(irate(pinchflat_prom_ex_oban_job_processing_duration_milliseconds_bucket{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])) by (le)",
|
|
"format": "heatmap",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Job Execution Time",
|
|
"tooltip": {
|
|
"show": true,
|
|
"showHistogram": true
|
|
},
|
|
"type": "heatmap",
|
|
"xAxis": {
|
|
"show": true
|
|
},
|
|
"xBucketNumber": null,
|
|
"xBucketSize": null,
|
|
"yAxis": {
|
|
"decimals": null,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true,
|
|
"splitFactor": null
|
|
},
|
|
"yBucketBound": "auto",
|
|
"yBucketNumber": null,
|
|
"yBucketSize": null
|
|
},
|
|
{
|
|
"cards": {
|
|
"cardPadding": null,
|
|
"cardRound": null
|
|
},
|
|
"color": {
|
|
"cardColor": "#b4ff00",
|
|
"colorScale": "sqrt",
|
|
"colorScheme": "interpolateOranges",
|
|
"exponent": 0.5,
|
|
"mode": "spectrum"
|
|
},
|
|
"dataFormat": "tsbuckets",
|
|
"datasource": "prometheus",
|
|
"description": "A heatmap showing how long jobs are waiting in queue for processing.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 14
|
|
},
|
|
"heatmap": {},
|
|
"hideZeroBuckets": true,
|
|
"highlightCards": true,
|
|
"id": 12,
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"reverseYBuckets": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(irate(pinchflat_prom_ex_oban_job_queue_time_milliseconds_bucket{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])) by (le)",
|
|
"format": "heatmap",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Job Queue Time",
|
|
"tooltip": {
|
|
"show": true,
|
|
"showHistogram": true
|
|
},
|
|
"type": "heatmap",
|
|
"xAxis": {
|
|
"show": true
|
|
},
|
|
"xBucketNumber": null,
|
|
"xBucketSize": null,
|
|
"yAxis": {
|
|
"decimals": null,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true,
|
|
"splitFactor": null
|
|
},
|
|
"yBucketBound": "auto",
|
|
"yBucketNumber": null,
|
|
"yBucketSize": null
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The average time it took to process jobs per queue. This will include jobs that ended with a successful, snoozed, or discarded state.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 27
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 11,
|
|
"legend": {
|
|
"alignAsTable": true,
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": false,
|
|
"hideZero": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": true,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null as zero",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "irate(pinchflat_prom_ex_oban_job_processing_duration_milliseconds_sum{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval]) / irate(pinchflat_prom_ex_oban_job_processing_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "({{ state }}) {{ name }} :: {{ worker }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Average Job Execution Time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": "Execution Time",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The average time a job waited in queue for processing.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 27
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 44,
|
|
"legend": {
|
|
"alignAsTable": true,
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": false,
|
|
"hideZero": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": true,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null as zero",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "irate(pinchflat_prom_ex_oban_job_queue_time_milliseconds_sum{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval]) / irate(pinchflat_prom_ex_oban_job_queue_time_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "({{ state }}) {{ name }} :: {{ worker }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Average Job Queue Time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": "In Queue Time",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The average time it took to process jobs per queue. This will include jobs that ended with a successful, snoozed, or discarded state.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 40
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 51,
|
|
"legend": {
|
|
"alignAsTable": true,
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": false,
|
|
"hideZero": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": true,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null as zero",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "irate(pinchflat_prom_ex_oban_job_complete_attempts_sum{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval]) / irate(pinchflat_prom_ex_oban_job_complete_attempts_count{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "({{ state }}) {{ name }} :: {{ worker }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Average Job Attempts",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "locale",
|
|
"label": "Job Attempts",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 53
|
|
},
|
|
"id": 54,
|
|
"panels": [],
|
|
"title": "Job Error Details",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"cards": {
|
|
"cardPadding": null,
|
|
"cardRound": null
|
|
},
|
|
"color": {
|
|
"cardColor": "#b4ff00",
|
|
"colorScale": "sqrt",
|
|
"colorScheme": "interpolateOranges",
|
|
"exponent": 0.5,
|
|
"mode": "spectrum"
|
|
},
|
|
"dataFormat": "tsbuckets",
|
|
"datasource": "prometheus",
|
|
"description": "A heatmap showing the time it took to process jobs across all queues that resulted in an error.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 54
|
|
},
|
|
"heatmap": {},
|
|
"hideZeroBuckets": true,
|
|
"highlightCards": true,
|
|
"id": 55,
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"reverseYBuckets": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(irate(pinchflat_prom_ex_oban_job_exception_duration_milliseconds_bucket{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])) by (le)",
|
|
"format": "heatmap",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Job Execution Time",
|
|
"tooltip": {
|
|
"show": true,
|
|
"showHistogram": true
|
|
},
|
|
"type": "heatmap",
|
|
"xAxis": {
|
|
"show": true
|
|
},
|
|
"xBucketNumber": null,
|
|
"xBucketSize": null,
|
|
"yAxis": {
|
|
"decimals": null,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true,
|
|
"splitFactor": null
|
|
},
|
|
"yBucketBound": "auto",
|
|
"yBucketNumber": null,
|
|
"yBucketSize": null
|
|
},
|
|
{
|
|
"cards": {
|
|
"cardPadding": null,
|
|
"cardRound": null
|
|
},
|
|
"color": {
|
|
"cardColor": "#b4ff00",
|
|
"colorScale": "sqrt",
|
|
"colorScheme": "interpolateOranges",
|
|
"exponent": 0.5,
|
|
"mode": "spectrum"
|
|
},
|
|
"dataFormat": "tsbuckets",
|
|
"datasource": "prometheus",
|
|
"description": "A heatmap showing how long jobs are waiting in queue for processing that resulted in an error.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 54
|
|
},
|
|
"heatmap": {},
|
|
"hideZeroBuckets": true,
|
|
"highlightCards": true,
|
|
"id": 56,
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"reverseYBuckets": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(irate(pinchflat_prom_ex_oban_job_exception_queue_time_milliseconds_bucket{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])) by (le)",
|
|
"format": "heatmap",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Job Queue Time",
|
|
"tooltip": {
|
|
"show": true,
|
|
"showHistogram": true
|
|
},
|
|
"type": "heatmap",
|
|
"xAxis": {
|
|
"show": true
|
|
},
|
|
"xBucketNumber": null,
|
|
"xBucketSize": null,
|
|
"yAxis": {
|
|
"decimals": null,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true,
|
|
"splitFactor": null
|
|
},
|
|
"yBucketBound": "auto",
|
|
"yBucketNumber": null,
|
|
"yBucketSize": null
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The average time it took to process jobs per queue that resulted in an error.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 67
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 57,
|
|
"legend": {
|
|
"alignAsTable": true,
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": false,
|
|
"hideZero": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": true,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null as zero",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "irate(pinchflat_prom_ex_oban_job_exception_duration_milliseconds_sum{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval]) / irate(pinchflat_prom_ex_oban_job_exception_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "({{ error }}) {{ name }} :: {{ worker }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Average Job Execution Time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": "Execution Time",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The average time a job waited in queue for processing prior to resulting in an error.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 67
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 58,
|
|
"legend": {
|
|
"alignAsTable": true,
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": false,
|
|
"hideZero": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": true,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null as zero",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "irate(pinchflat_prom_ex_oban_job_exception_queue_time_milliseconds_sum{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval]) / irate(pinchflat_prom_ex_oban_job_exception_queue_time_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "({{ error }}) {{ name }} :: {{ worker }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Average Job Queue Time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": "In Queue Time",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The average time it took to process jobs per queue that resulted in an error.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 80
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 52,
|
|
"legend": {
|
|
"alignAsTable": true,
|
|
"avg": false,
|
|
"current": false,
|
|
"hideEmpty": false,
|
|
"hideZero": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": true,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null as zero",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "irate(pinchflat_prom_ex_oban_job_exception_attempts_sum{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval]) / irate(pinchflat_prom_ex_oban_job_exception_attempts_count{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "({{ state }}) {{ name }} :: {{ worker }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Average Job Failure Attempts",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "locale",
|
|
"label": "Job Attempts",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 93
|
|
},
|
|
"id": 4,
|
|
"panels": [],
|
|
"title": "Queue Details",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The number of jobs marked as available in each queue.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 94
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 15,
|
|
"legend": {
|
|
"alignAsTable": false,
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null as zero",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "pinchflat_prom_ex_oban_queue_length_count{job=\"$job\", instance=\"$instance\", name=\"$oban\", state=\"available\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{ queue }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Available Jobs per Queue",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": "Number of Jobs",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The number of jobs marked as completed in each queue.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 94
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 63,
|
|
"legend": {
|
|
"alignAsTable": false,
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null as zero",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "pinchflat_prom_ex_oban_queue_length_count{job=\"$job\", instance=\"$instance\", name=\"$oban\", state=\"completed\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{ queue }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Completed Jobs per Queue",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": "Number of Jobs",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The number of jobs marked as executing in each queue.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 107
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 62,
|
|
"legend": {
|
|
"alignAsTable": false,
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null as zero",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "pinchflat_prom_ex_oban_queue_length_count{job=\"$job\", instance=\"$instance\", name=\"$oban\", state=\"executing\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{ queue }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Executing Jobs per Queue",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": "Number of Jobs",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The number of jobs marked as retryable in each queue.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 107
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 61,
|
|
"legend": {
|
|
"alignAsTable": false,
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"rightSide": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null as zero",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "pinchflat_prom_ex_oban_queue_length_count{job=\"$job\", instance=\"$instance\", name=\"$oban\", state=\"retryable\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{ queue }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Retryable Jobs per Queue",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": "Number of Jobs",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 120
|
|
},
|
|
"id": 30,
|
|
"panels": [],
|
|
"title": "Producer Details",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"cards": {
|
|
"cardPadding": null,
|
|
"cardRound": null
|
|
},
|
|
"color": {
|
|
"cardColor": "#b4ff00",
|
|
"colorScale": "sqrt",
|
|
"colorScheme": "interpolateOranges",
|
|
"exponent": 0.5,
|
|
"mode": "spectrum"
|
|
},
|
|
"dataFormat": "tsbuckets",
|
|
"datasource": "prometheus",
|
|
"description": "A heatmap showing how long jobs to be dispatched.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 121
|
|
},
|
|
"heatmap": {},
|
|
"hideZeroBuckets": true,
|
|
"highlightCards": true,
|
|
"id": 73,
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"reverseYBuckets": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(irate(pinchflat_prom_ex_oban_producer_duration_milliseconds_bucket{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])) by (le)",
|
|
"format": "heatmap",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Producer Dispatch Time",
|
|
"tooltip": {
|
|
"show": true,
|
|
"showHistogram": true
|
|
},
|
|
"type": "heatmap",
|
|
"xAxis": {
|
|
"show": true
|
|
},
|
|
"xBucketNumber": null,
|
|
"xBucketSize": null,
|
|
"yAxis": {
|
|
"decimals": null,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true,
|
|
"splitFactor": null
|
|
},
|
|
"yBucketBound": "auto",
|
|
"yBucketNumber": null,
|
|
"yBucketSize": null
|
|
},
|
|
{
|
|
"cards": {
|
|
"cardPadding": null,
|
|
"cardRound": null
|
|
},
|
|
"color": {
|
|
"cardColor": "#b4ff00",
|
|
"colorScale": "sqrt",
|
|
"colorScheme": "interpolateOranges",
|
|
"exponent": 0.5,
|
|
"mode": "spectrum"
|
|
},
|
|
"dataFormat": "tsbuckets",
|
|
"datasource": "prometheus",
|
|
"description": "A heatmap showing the number of jobs that were dispatched.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 121
|
|
},
|
|
"heatmap": {},
|
|
"hideZeroBuckets": true,
|
|
"highlightCards": true,
|
|
"id": 74,
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"reverseYBuckets": false,
|
|
"targets": [
|
|
{
|
|
"expr": "sum(irate(pinchflat_prom_ex_oban_producer_dispatched_count_bucket{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])) by (le)",
|
|
"format": "heatmap",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Producer Dispatch Count",
|
|
"tooltip": {
|
|
"show": true,
|
|
"showHistogram": true
|
|
},
|
|
"type": "heatmap",
|
|
"xAxis": {
|
|
"show": true
|
|
},
|
|
"xBucketNumber": null,
|
|
"xBucketSize": null,
|
|
"yAxis": {
|
|
"decimals": null,
|
|
"format": "locale",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true,
|
|
"splitFactor": null
|
|
},
|
|
"yBucketBound": "auto",
|
|
"yBucketNumber": null,
|
|
"yBucketSize": null
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The average amount of time it took to dispatch jobs to each queue.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 134
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 67,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "irate(pinchflat_prom_ex_oban_producer_duration_milliseconds_sum{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval]) / irate(pinchflat_prom_ex_oban_producer_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "{{ queue }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Average Dispatch Time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": "Number of Jobs",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The average number of jobs dispatched to each queue.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 134
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 75,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "irate(pinchflat_prom_ex_oban_producer_dispatched_count_sum{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval]) / irate(pinchflat_prom_ex_oban_producer_dispatched_count_count{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "{{ queue }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Average Dispatch Count",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": "Number of Jobs",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The average amount of time it took to encounter an encounter an error.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 147
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 76,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "irate(pinchflat_prom_ex_oban_producer_exception_duration_milliseconds_sum{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval]) / irate(pinchflat_prom_ex_oban_producer_exception_duration_milliseconds_count{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "{{ queue }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Average Exception Duration",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": "Number of Jobs",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"datasource": null,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 160
|
|
},
|
|
"id": 65,
|
|
"panels": [],
|
|
"title": "Circuit Breaker Details",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "This is the total number of trip events that have been encountered by the selected application and Oban instances.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {},
|
|
"noValue": "0",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 161
|
|
},
|
|
"id": 78,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "increase(pinchflat_prom_ex_oban_circuit_trip_total{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[1h])",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Circuit Trip Events (last 1 hour)",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "This is the total number of trip events that have been encountered by the selected application and Oban instances.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {},
|
|
"noValue": "0",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 161
|
|
},
|
|
"id": 80,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "pinchflat_prom_ex_oban_circuit_trip_total{job=\"$job\", instance=\"$instance\", name=\"$oban\"}",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Total Circuit Trip Events",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "This is the total number of open events that have been encountered by the selected application and Oban instances.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {},
|
|
"noValue": "0",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 161
|
|
},
|
|
"id": 81,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "increase(pinchflat_prom_ex_oban_circuit_open_total{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[1h])",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Circuit Open Events (last 1 hour)",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": "prometheus",
|
|
"description": "This is the total number of open events that have been encountered by the selected application and Oban instances.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {},
|
|
"noValue": "0",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 161
|
|
},
|
|
"id": 79,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": ["lastNotNull"],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "7.1.3",
|
|
"targets": [
|
|
{
|
|
"expr": "pinchflat_prom_ex_oban_circuit_open_total{job=\"$job\", instance=\"$instance\", name=\"$oban\"}",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"timeFrom": null,
|
|
"timeShift": null,
|
|
"title": "Total Circuit Open Events",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The number of circuit breaker trip events that occurred along with what component tripped the breaker.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 166
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 70,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "increase(pinchflat_prom_ex_oban_circuit_trip_total{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "{{ circuit_breaker }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Circuit Breaker Trip Events",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": "Number of Circuit Trips",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": "prometheus",
|
|
"description": "The number of circuit breaker open events that occurred along with what component tripped the breaker.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 13,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 166
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 72,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"percentage": false,
|
|
"pluginVersion": "7.1.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"expr": "increase(pinchflat_prom_ex_oban_circuit_open_total{job=\"$job\", instance=\"$instance\", name=\"$oban\"}[$interval])",
|
|
"interval": "",
|
|
"legendFormat": "{{ circuit_breaker }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeFrom": null,
|
|
"timeRegions": [],
|
|
"timeShift": null,
|
|
"title": "Circuit Breaker Open Events",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 2,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"buckets": null,
|
|
"mode": "time",
|
|
"name": null,
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": "Number of Circuit Opens",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false,
|
|
"alignLevel": null
|
|
}
|
|
}
|
|
],
|
|
"refresh": "5s",
|
|
"schemaVersion": 26,
|
|
"style": "dark",
|
|
"tags": ["PromEx", "Oban", "pinchflat"],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"allValue": null,
|
|
"current": {
|
|
"selected": false,
|
|
"text": "elixir_app",
|
|
"value": "elixir_app"
|
|
},
|
|
"datasource": "prometheus",
|
|
"definition": "label_values(pinchflat_prom_ex_prom_ex_status_info, job)",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Prometheus Job",
|
|
"multi": false,
|
|
"name": "job",
|
|
"options": [],
|
|
"query": "label_values(pinchflat_prom_ex_prom_ex_status_info, job)",
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 6,
|
|
"tagValuesQuery": "",
|
|
"tags": [],
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"allValue": null,
|
|
"current": {
|
|
"selected": false,
|
|
"text": "elixir_app_one:4000",
|
|
"value": "elixir_app_one:4000"
|
|
},
|
|
"datasource": "prometheus",
|
|
"definition": "label_values(pinchflat_prom_ex_prom_ex_status_info, instance)",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Application Instance",
|
|
"multi": false,
|
|
"name": "instance",
|
|
"options": [],
|
|
"query": "label_values(pinchflat_prom_ex_prom_ex_status_info{job=\"$job\"}, instance)",
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"tagValuesQuery": "",
|
|
"tags": [],
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"allValue": null,
|
|
"datasource": "prometheus",
|
|
"definition": "label_values(pinchflat_prom_ex_oban_init_status_info, name)",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Oban Instance",
|
|
"multi": false,
|
|
"name": "oban",
|
|
"options": [],
|
|
"query": "label_values(pinchflat_prom_ex_oban_init_status_info, name)",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"tagValuesQuery": "",
|
|
"tags": [],
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"auto": false,
|
|
"auto_count": 30,
|
|
"auto_min": "10s",
|
|
"current": {
|
|
"selected": true,
|
|
"text": "30s",
|
|
"value": "30s"
|
|
},
|
|
"hide": 0,
|
|
"label": "Interval",
|
|
"name": "interval",
|
|
"options": [
|
|
{
|
|
"selected": false,
|
|
"text": "15s",
|
|
"value": "15s"
|
|
},
|
|
{
|
|
"selected": true,
|
|
"text": "30s",
|
|
"value": "30s"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1m",
|
|
"value": "1m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "5m",
|
|
"value": "5m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "15m",
|
|
"value": "15m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "30m",
|
|
"value": "30m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1h",
|
|
"value": "1h"
|
|
}
|
|
],
|
|
"query": "15s, 30s, 1m, 5m, 15m, 30m, 1h",
|
|
"queryValue": "",
|
|
"refresh": 2,
|
|
"skipUrlSync": false,
|
|
"type": "interval"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {
|
|
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m"]
|
|
},
|
|
"timezone": "",
|
|
"title": "Pinchflat - PromEx Oban Dashboard",
|
|
"uid": "AF9058A0496CA7FF8D4FA747EEDC7AF3",
|
|
"version": 1
|
|
}
|