From 561acc05f1ec320db691376ac5d56d9aafb0d0b2 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Sun, 1 Nov 2026 01:27:10 -0400 Subject: [PATCH 1/5] services/nomad/monitoring/prometheus: convert to nomad services --- services/nomad/monitoring/prometheus.nomad | 14 +- services/nomad/monitoring/prometheus.yml | 259 +++++---------------- 2 files changed, 69 insertions(+), 204 deletions(-) diff --git a/services/nomad/monitoring/prometheus.nomad b/services/nomad/monitoring/prometheus.nomad index 1b865626..e5b0e9d4 100644 --- a/services/nomad/monitoring/prometheus.nomad +++ b/services/nomad/monitoring/prometheus.nomad @@ -21,12 +21,14 @@ job "prometheus" { } service { - name = "prometheus" - port = "http" - meta { - nginx_enable = "true" - nginx_names = "prometheus.s.voidlinux.org prometheus.voidlinux.org" - } + provider = "nomad" + name = "prometheus" + port = "http" + + tags = [ + "nginx_enable", + "nginx_names=prometheus.s.voidlinux.org prometheus.voidlinux.org", + ] check { type = "http" diff --git a/services/nomad/monitoring/prometheus.yml b/services/nomad/monitoring/prometheus.yml index 68c6e26c..33211c1a 100644 --- a/services/nomad/monitoring/prometheus.yml +++ b/services/nomad/monitoring/prometheus.yml @@ -1,4 +1,5 @@ --- +# {{ $allocID := env "NOMAD_ALLOC_ID" }} global: scrape_interval: 30s evaluation_interval: 30s @@ -6,10 +7,11 @@ rule_files: - /local/alerts/*.yml alerting: alertmanagers: - - consul_sd_configs: - - server: {{ env "NOMAD_HOST_IP_http" }}:8500 - datacenter: void - services: ['alertmanager'] + - static_configs: + - targets: + # {{ range nomadService "alertmanager?ns=monitoring" }} + - '{{ .Address }}:{{ .Port }}' + # {{ end }} scrape_configs: - job_name: prometheus static_configs: @@ -17,12 +19,13 @@ scrape_configs: - localhost:9090 - job_name: node consul_sd_configs: - - server: {{ env "NOMAD_HOST_IP_http" }}:8500 + - server: '{{ env "NOMAD_HOST_IP_http" }}:8500' datacenter: void services: ['node-exporter'] relabel_configs: - source_labels: ['__meta_consul_node'] target_label: instance + # {{ range nomadService 1 $allocID "ssl-exporter?ns=monitoring" }} - job_name: 'ssl' metrics_path: /probe scrape_interval: 2m @@ -46,7 +49,7 @@ scrape_configs: - source_labels: [__param_target] target_label: instance - target_label: __address__ - replacement: ssl-exporter.service.consul:9219 + replacement: '{{ .Address }}:{{ .Port }}' - job_name: 'ssl-self-signed' metrics_path: /probe scrape_interval: 2m @@ -61,89 +64,27 @@ scrape_configs: - source_labels: [__param_target] target_label: instance - target_label: __address__ - replacement: ssl-exporter.service.consul:9219 - - job_name: alertmanager - consul_sd_configs: - - server: {{ env "NOMAD_HOST_IP_http" }}:8500 - datacenter: void - services: ['alertmanager'] - relabel_configs: - - source_labels: ['__meta_consul_node'] - target_label: instance - - job_name: loki - consul_sd_configs: - - server: {{ env "NOMAD_HOST_IP_http" }}:8500 - datacenter: void - services: ['loki'] - - job_name: 'sync-x86_64-glibc' - metrics_path: /probe - scrape_interval: 2m - params: - arch: [x86_64] - static_configs: - - targets: - - root-pkgs-internal.service.consul - - repo-default.voidlinux.org/current - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_arch] - target_label: arch - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: repo-exporter-exporter.service.consul:9213 - - target_label: zone - replacement: internal - - job_name: 'sync-i686-glibc' - metrics_path: /probe - scrape_interval: 2m - params: - arch: [i686] - static_configs: - - targets: - - root-pkgs-internal.service.consul - - repo-default.voidlinux.org/current - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_arch] - target_label: arch - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: repo-exporter-exporter.service.consul:9213 - - target_label: zone - replacement: internal - - job_name: 'sync-armv7l-glibc' - metrics_path: /probe - scrape_interval: 2m - params: - arch: [armv7l] - static_configs: - - targets: - - root-pkgs-internal.service.consul - - repo-default.voidlinux.org/current - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_arch] - target_label: arch - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: repo-exporter-exporter.service.consul:9213 - - target_label: zone - replacement: internal - - job_name: 'sync-armv6l-glibc' + replacement: '{{ .Address }}:{{ .Port }}' + # {{ end }} + # {{ range $svc := nomadService 1 $allocID "repo-exporter?ns=monitoring" }} + # {{ range $arch := "x86_64,x86_64-musl,i686,aarch64,aarch64-musl,armv7l,armv7l-musl,armv6l,armv6l-musl" | split "," }} + # {{ $path := "" }} + # {{ if $arch | regexMatch "^aarch64.*$" }} + # {{ $path = "/aarch64" }} + # {{ else if $arch | regexMatch "^.*-musl$" }} + # {{ $path = "/musl" }} + # {{ end }} + - job_name: 'sync-{{ $arch }}' metrics_path: /probe scrape_interval: 2m params: - arch: [armv6l] + arch: ['{{ $arch }}'] static_configs: - targets: - - root-pkgs-internal.service.consul - - repo-default.voidlinux.org/current + # {{ range nomadService 1 $allocID "root-pkgs-internal?ns=build" }} + - '{{ .Address }}:{{ .Port }}/{{ $path }}' + # {{ end }} + - 'repo-default.voidlinux.org/current{{ $path }}' relabel_configs: - source_labels: [__address__] target_label: __param_target @@ -152,109 +93,10 @@ scrape_configs: - source_labels: [__param_target] target_label: instance - target_label: __address__ - replacement: repo-exporter-exporter.service.consul:9213 - - target_label: zone - replacement: internal - - job_name: 'sync-x86_64-musl' - metrics_path: /probe - scrape_interval: 2m - params: - arch: [x86_64-musl] - static_configs: - - targets: - - root-pkgs-internal.service.consul/musl - - repo-default.voidlinux.org/current/musl - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_arch] - target_label: arch - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: repo-exporter-exporter.service.consul:9213 - - target_label: zone - replacement: internal - - job_name: 'sync-armv7l-musl' - metrics_path: /probe - scrape_interval: 2m - params: - arch: [armv7l-musl] - static_configs: - - targets: - - root-pkgs-internal.service.consul/musl - - repo-default.voidlinux.org/current/musl - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_arch] - target_label: arch - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: repo-exporter-exporter.service.consul:9213 - - target_label: zone - replacement: internal - - job_name: 'sync-armv6l-musl' - metrics_path: /probe - scrape_interval: 2m - params: - arch: [armv6l-musl] - static_configs: - - targets: - - root-pkgs-internal.service.consul/musl - - repo-default.voidlinux.org/current/musl - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_arch] - target_label: arch - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: repo-exporter-exporter.service.consul:9213 - - target_label: zone - replacement: internal - - job_name: 'sync-aarch64-glibc' - metrics_path: /probe - scrape_interval: 2m - params: - arch: [aarch64] - static_configs: - - targets: - - root-pkgs-internal.service.consul/aarch64 - - repo-default.voidlinux.org/current/aarch64 - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_arch] - target_label: arch - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: repo-exporter-exporter.service.consul:9213 - - target_label: zone - replacement: internal - - job_name: 'sync-aarch64-musl' - metrics_path: /probe - scrape_interval: 2m - params: - arch: [aarch64-musl] - static_configs: - - targets: - - root-pkgs-internal.service.consul/aarch64 - - repo-default.voidlinux.org/current/aarch64 - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_arch] - target_label: arch - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: repo-exporter-exporter.service.consul:9213 + replacement: '{{ $svc.Address }}:{{ $svc.Port }}' - target_label: zone replacement: internal + # {{ end }} - job_name: 'mirror-sync' metrics_path: /probe scrape_interval: 2m @@ -270,19 +112,40 @@ scrape_configs: - source_labels: [__param_target] target_label: instance - target_label: __address__ - replacement: repo-exporter-exporter.service.consul:9213 + replacement: "{{ $svc.Address }}:{{ $svc.Port }}" - target_label: zone replacement: external - - job_name: ccache - consul_sd_configs: - - server: {{ env "NOMAD_HOST_IP_http" }}:8500 - datacenter: void - services: ['ccache-exporter'] + # {{ end }} + # discover metrics from nomad services with the 'prometheus_enable' tag + # use the 'prometheus_path=...' tag to change the url path to scrape metrics from (default: '/metrics') + # use the 'prometheus_job=...' tag to change the name of the job in prometheus (default: service name) + - job_name: nomad_sd + nomad_sd_configs: + # all namespaces + # {{ range "apps,apps-restricted,build,infrastructure,mirror,monitoring" | split "," }} + - server: 'http://{{ env "attr.unique.network.ip-address" }}:4646' + namespace: '{{ . }}' + # {{ end }} relabel_configs: - - source_labels: ['__meta_consul_node'] - target_label: instance - - job_name: buildbot - consul_sd_configs: - - server: {{ env "NOMAD_HOST_IP_http" }}:8500 - datacenter: void - services: ['buildbot-metrics'] + # keep only services with the 'prometheus_enable=true' tag + - source_labels: ['__meta_nomad_tags'] + regex: '.*,prometheus_enable,.*' + action: keep + # change the metrics url path + - source_labels: ['__meta_nomad_tags'] + regex: '.*,prometheus_path=(.*),.*' + replacement: '$1' + target_label: '__metrics_path__' + # set default job name to nomad service name + - source_labels: ['__meta_nomad_service'] + target_label: job + # override job name via tag + - source_labels: ['__meta_nomad_tags'] + regex: '.*,prometheus_job=(.*),.*' + target_label: job + # get namespace from nomad + - source_labels: ['__meta_nomad_namespace'] + target_label: namespace + # get datacenter from nomad + - source_labels: ['__meta_nomad_dc'] + target_label: datacenter From 87b73b6d7925ba18eb79200140cc6808bf073f40 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Sun, 1 Nov 2026 01:26:36 -0400 Subject: [PATCH 2/5] services/nomad/build/buildbot: tag buildbot-metrics service --- services/nomad/build/buildbot.nomad | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/nomad/build/buildbot.nomad b/services/nomad/build/buildbot.nomad index d10b7a8b..2605b6bb 100644 --- a/services/nomad/build/buildbot.nomad +++ b/services/nomad/build/buildbot.nomad @@ -57,6 +57,8 @@ job "buildbot" { provider = "nomad" name = "buildbot-metrics" port = "metrics" + + tags = ["prometheus_enable", "prometheus_job=buildbot"] } task "buildbot" { From ed94f884c75ae86c0ed303ddc3591bc8909eb9fb Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Sun, 1 Nov 2026 01:26:06 -0400 Subject: [PATCH 3/5] services/nomad/monitoring/*: use nomad services --- services/nomad/monitoring/alertmanager.nomad | 20 ++++++++++++------- services/nomad/monitoring/alertrelay.nomad | 5 +++-- services/nomad/monitoring/ccache.nomad | 7 +++++-- services/nomad/monitoring/grafana.nomad | 14 +++++++------ services/nomad/monitoring/promtail.nomad | 12 ++++++----- services/nomad/monitoring/repo_exporter.nomad | 5 ++++- services/nomad/monitoring/ssl_exporter.nomad | 5 +++-- services/nomad/monitoring/vector.nomad | 12 ++++++----- services/nomad/monitoring/vmlogs.nomad | 14 +++++++------ 9 files changed, 58 insertions(+), 36 deletions(-) diff --git a/services/nomad/monitoring/alertmanager.nomad b/services/nomad/monitoring/alertmanager.nomad index bc2f7f24..996bc480 100644 --- a/services/nomad/monitoring/alertmanager.nomad +++ b/services/nomad/monitoring/alertmanager.nomad @@ -12,12 +12,15 @@ job "alertmanager" { } service { - name = "alertmanager" - port = "http" - meta { - nginx_enable = "true" - nginx_names = "alertmanager.s.voidlinux.org alertmanager.voidlinux.org" - } + provider = "nomad" + name = "alertmanager" + port = "http" + + tags = [ + "nginx_enable", + "nginx_names=alertmanager.s.voidlinux.org alertmanager.voidlinux.org", + "prometheus_enable", + ] check { type = "http" @@ -46,6 +49,7 @@ job "alertmanager" { template { data = < Date: Sun, 1 Nov 2026 03:10:25 -0500 Subject: [PATCH 4/5] services/nomad/apps/popcorn: use nomad services --- services/nomad/apps/popcorn.nomad | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/services/nomad/apps/popcorn.nomad b/services/nomad/apps/popcorn.nomad index 6e78c1c9..5c05c4c9 100644 --- a/services/nomad/apps/popcorn.nomad +++ b/services/nomad/apps/popcorn.nomad @@ -69,13 +69,15 @@ EOF } service { - name = "popcorn-statrepo" - port = "statrepo" + provider = "nomad" + name = "popcorn-statrepo" + port = "statrepo" } service { - name = "popcorn-pqueryd" - port = "pqueryd" + provider = "nomad" + name = "popcorn-pqueryd" + port = "pqueryd" } volume "popcorn_data" { From acf59f0a6382280079544e6f0b5c3ddb644804d0 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Sun, 1 Nov 2026 16:26:33 -0500 Subject: [PATCH 5/5] services/nomad/monitoring/vector: use nomad service discovery --- services/nomad/monitoring/vector.nomad | 64 +++++++++++++------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/services/nomad/monitoring/vector.nomad b/services/nomad/monitoring/vector.nomad index 489fc93c..6ca1891a 100644 --- a/services/nomad/monitoring/vector.nomad +++ b/services/nomad/monitoring/vector.nomad @@ -54,38 +54,38 @@ job "vector" { } template { - data = yamlencode({ - sources = { - docker = { - type = "docker_logs" - exclude_containers = [ - "vector-", - "loki-", - "nomad_init_", - ] - } - } - sinks = { - vlogs = { - type = "elasticsearch" - inputs = ["docker"] - endpoints = ["http://vmlogs.service.consul:9428/insert/elasticsearch/"] - api_version = "v8" - compression = "gzip" - healthcheck = { enabled = false } - query = { - "_time_field" = "timestamp" - "_stream_fields" = join(",", formatlist("label.com.hashicorp.nomad.%s", [ - "namespace", "job_name", "task_group_name", "task_name", "alloc_id", - ])) - "_msg_field" = "message" - } - } - } - }) - destination = "local/vector.yaml" - left_delimiter = "[[" - right_delimiter = "]]" + data = <