From da255bb732ccc9762f2d08eee83fd70d097e0c3b Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Fri, 13 Nov 2026 02:52:26 -0600 Subject: [PATCH 1/5] Remove settings nothing reads and fix inaccurate comments Remove environment variables no service reads: SRC_GIT_SERVERS, SEARCHER_URL, and INDEXED_SEARCH_SERVERS on worker, and UID and GID on the executor. Remove the redis-cache and redis-store Prometheus targets: no Redis exporter runs, so both targets were always down. Scrape syntactic-code-intel-worker, which serves metrics but had no target. Correct comments that named ports nothing listens on, and document that the frontend address variables also accept a replica count. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d40c-fdd9-73bd-a123-0a3e4f0dc0c3 Co-authored-by: Amp --- docker-compose/dev/docker-compose.yaml | 4 +-- docker-compose/docker-compose.yaml | 32 +++++++++---------- .../docker-compose.override.yaml | 4 +-- .../executors/executor.docker-compose.yaml | 3 -- docker-compose/sourcegraph-uploads.env | 3 +- prometheus/prometheus_targets.yml | 17 ++++------ 6 files changed, 26 insertions(+), 37 deletions(-) diff --git a/docker-compose/dev/docker-compose.yaml b/docker-compose/dev/docker-compose.yaml index baecf3d0f..92eac99c7 100644 --- a/docker-compose/dev/docker-compose.yaml +++ b/docker-compose/dev/docker-compose.yaml @@ -1,5 +1,5 @@ -# Development overlay - has reduced resource requirements, and exposes the deployment on -# 127.0.0.1:8080 +# Development overlay - has reduced resource requirements, and also publishes the deployment +# on port 8080 of every host interface, in addition to ports 80 and 443 from the base file services: zoekt-webserver-0: diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 6febd829d..7abbfb2ee 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -118,13 +118,14 @@ services: # Description: Serves the frontend of Sourcegraph via HTTP(S). # # Disk: 128GB / non-persistent SSD - # Ports exposed to other Sourcegraph services: 6060/TCP, 3080 (HTTP), and/or 3443 (HTTPS) + # Ports exposed to other Sourcegraph services: 6060/TCP, 3080 (HTTP) # Ports exposed to the public internet: none # - # Note: SRC_GIT_SERVERS, and SEARCHER_URL are space-separated - # lists which each allow you to specify more container instances for scaling - # purposes. Be sure to also apply such a change here to the frontend-internal - # service. + # Note: SRC_GIT_SERVERS, SEARCHER_URL, INDEXED_SEARCH_SERVERS, and INDEXED_SEARCH_INDEXER + # each take either a space-separated list of addresses, or a number of replicas, + # ex. SRC_GIT_SERVERS=2 is the same as SRC_GIT_SERVERS=gitserver-0:3178 gitserver-1:3178. + # Only the two frontend services read them; be sure to apply the same change to the + # frontend-internal service. sourcegraph-frontend-0: container_name: sourcegraph-frontend-0 image: 'index.docker.io/sourcegraph/frontend:8.2.0@sha256:00adad80d2a9e49c586d6735a1ddbc2e89fc06d6af0a3b668aefbcb7802f2e95' @@ -357,7 +358,7 @@ services: # Description: Manages background processes. # # Disk: 128GB / non-persistent SSD - # Ports exposed to other Sourcegraph services: 3189/TCP 6060/TCP + # Ports exposed to other Sourcegraph services: 3189/TCP 6060/TCP 6996/TCP (executor metrics) # Ports exposed to the public internet: none # worker: @@ -371,9 +372,6 @@ services: environment: - 'SRC_FRONTEND_INTERNAL=sourcegraph-frontend-internal:3090' - 'OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317' - - 'SRC_GIT_SERVERS=gitserver-0:3178' - - 'SEARCHER_URL=http://searcher-0:3181' - - 'INDEXED_SEARCH_SERVERS=zoekt-webserver-0:6070' volumes: - 'worker:/mnt/cache' networks: @@ -463,8 +461,8 @@ services: # Add the following environment variables if you wish to use an auth proxy with Grafana: # # 'GF_AUTH_PROXY_ENABLED=true' - # 'GF_AUTH_PROXY_HEADER_NAME='X-Forwarded-User' - # 'GF_SERVER_ROOT_URL='https://grafana.example.com' + # 'GF_AUTH_PROXY_HEADER_NAME=X-Forwarded-User' + # 'GF_SERVER_ROOT_URL=https://grafana.example.com' grafana: container_name: grafana image: 'index.docker.io/sourcegraph/grafana:8.2.0@sha256:fd478ac1a5ce4552bfc388984a559548b5b7ea8001c2e65adcfb7e292bb54784' @@ -493,7 +491,7 @@ services: mem_limit: '1g' security_opt: - no-new-privileges:true - # You may set `privileged` to `false and `cadvisor` will run with reduced privileges. + # You may set `privileged` to `false` and `cadvisor` will run with reduced privileges. # `cadvisor` requires root privileges in order to display provisioning metrics. # These metrics provide critical information to help you scale the Sourcegraph deployment. # If you would like to bring your own infrastructure monitoring & alerting solution, @@ -549,7 +547,7 @@ services: # Description: PostgreSQL database for various data. # # Disk: 128GB / persistent SSD - # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP + # Ports exposed to other Sourcegraph services: 5432/TCP # Ports exposed to the public internet: none # pgsql: @@ -600,7 +598,7 @@ services: # # Disk: 128GB / persistent SSD # Network: 1Gbps - # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP + # Ports exposed to other Sourcegraph services: 5432/TCP # Ports exposed to the public internet: none # codeintel-db: @@ -651,7 +649,7 @@ services: # # Disk: 128GB / persistent SSD # Network: 1Gbps - # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP + # Ports exposed to other Sourcegraph services: 5432/TCP # Ports exposed to the public internet: none # codeinsights-db: @@ -732,7 +730,7 @@ services: # Description: Redis for storing short-lived caches. # # Disk: 128GB / persistent SSD - # Ports exposed to other Sourcegraph services: 6379/TCP 9121/TCP + # Ports exposed to other Sourcegraph services: 6379/TCP # Ports exposed to the public internet: none # redis-cache: @@ -750,7 +748,7 @@ services: # Description: Redis for storing semi-persistent data like user sessions. # # Disk: 128GB / persistent SSD - # Ports exposed to other Sourcegraph services: 6379/TCP 9121/TCP + # Ports exposed to other Sourcegraph services: 6379/TCP # Ports exposed to the public internet: none # redis-store: diff --git a/docker-compose/examples/external-databases/docker-compose.override.yaml b/docker-compose/examples/external-databases/docker-compose.override.yaml index 81451e000..9106e19a4 100644 --- a/docker-compose/examples/external-databases/docker-compose.override.yaml +++ b/docker-compose/examples/external-databases/docker-compose.override.yaml @@ -26,9 +26,7 @@ x-env-vars-database: &env-vars-database PGSSLMODE: require PGUSER: sg -# Uncomment if using AWS RDS with IAM authentication -# PG_CONNECTION_UPDATER: EC2_ROLE_CREDENTIALS -# AWS_STS_REGIONAL_ENDPOINTS: regional +# For AWS RDS with IAM authentication, use ../aws-rds-databases/docker-compose.override.yaml instead services: diff --git a/docker-compose/executors/executor.docker-compose.yaml b/docker-compose/executors/executor.docker-compose.yaml index 473bc7072..596c21376 100644 --- a/docker-compose/executors/executor.docker-compose.yaml +++ b/docker-compose/executors/executor.docker-compose.yaml @@ -19,9 +19,6 @@ services: - EXECUTOR_DOCKER_HOST_MOUNT_PATH=/scratch # Note: Must match right-hand side of scratch volume mount - TMPDIR=/scratch - # Run as root (required for docker daemon control) - - UID=1000 - - GID=1000 volumes: # Mount docker socket - '/var/run/docker.sock:/var/run/docker.sock' diff --git a/docker-compose/sourcegraph-uploads.env b/docker-compose/sourcegraph-uploads.env index b5f6a2097..af2004e69 100644 --- a/docker-compose/sourcegraph-uploads.env +++ b/docker-compose/sourcegraph-uploads.env @@ -3,4 +3,5 @@ SOURCEGRAPH_UPLOAD_BACKEND=blobstore SOURCEGRAPH_UPLOAD_AWS_ENDPOINT=http://blobstore:9000 # To use external S3 or GCS, replace the settings above and provide every -# container with credentials here. See README.md for supported settings. +# container with credentials here. For supported settings, see +# https://sourcegraph.com/docs/self-hosted/external-services/object-storage#sourcegraph-bucket diff --git a/prometheus/prometheus_targets.yml b/prometheus/prometheus_targets.yml index e7cc3e623..652316e31 100644 --- a/prometheus/prometheus_targets.yml +++ b/prometheus/prometheus_targets.yml @@ -25,7 +25,12 @@ job: precise-code-intel-worker targets: - precise-code-intel-worker:6060 - # Add new entries here for every searcher/symbol/gitserver replica. +- labels: + nodename: "sourcegraph-docker-compose-host" + job: syntactic-code-intel-worker + targets: + - syntactic-code-intel-worker:6060 +# Add a target to the matching job below for every gitserver, searcher, and zoekt replica. - labels: nodename: "sourcegraph-docker-compose-host" job: zoekt-indexserver @@ -66,16 +71,6 @@ job: codeinsights-db targets: - codeinsights-db-exporter:9187 -- labels: - nodename: "sourcegraph-docker-compose-host" - job: redis-cache - targets: - - redis-cache:9121 -- labels: - nodename: "sourcegraph-docker-compose-host" - job: redis-store - targets: - - redis-store:9121 - labels: nodename: "sourcegraph-docker-compose-host" job: node-exporter From 62d55fd2a0ed584366ada987d94c008dc25a61a4 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:16:01 -0600 Subject: [PATCH 2/5] prometheus: Fix the Docker metrics target for Linux Inside the prometheus container, localhost:9323 is the container itself, so the documented Linux target was always down. Name the host through host.docker.internal, mapped to host-gateway, and have the daemon listen on an address containers can reach. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d40c-fdd9-73bd-a123-0a3e4f0dc0c3 Co-authored-by: Amp --- prometheus/README.md | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/prometheus/README.md b/prometheus/README.md index 5bb031965..fee8fbe69 100644 --- a/prometheus/README.md +++ b/prometheus/README.md @@ -14,14 +14,27 @@ To scrape Docker itself for metrics, add the following file depending on your ho #### Linux -Create `prometheus/docker_targets.yml`: +Inside the `prometheus` container, `localhost` is the container itself, so the target must name the host. -```yaml -- labels: - job: docker - targets: - - localhost:9323 -``` +1. Set `"metrics-addr": "0.0.0.0:9323"` in `/etc/docker/daemon.json`, and restart Docker. Containers cannot reach a daemon which listens on `127.0.0.1:9323`. + +2. Add the host's name to the `prometheus` service in your `docker-compose.override.yaml`: + + ```yaml + services: + prometheus: + extra_hosts: + - 'host.docker.internal:host-gateway' + ``` + +3. Create `prometheus/docker_targets.yml`: + + ```yaml + - labels: + job: docker + targets: + - host.docker.internal:9323 + ``` #### Mac & Windows From df06df08409679de62979165404158f0d8977e9a Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:30:21 -0600 Subject: [PATCH 3/5] Fix the default collector and Grafana Jaeger datasource, drop unused volumes and ports The base file's collector config listens on localhost only, so no service can send it traces and Prometheus cannot scrape it. Use the mounted otel-collector/config.yaml by default, and make that file valid for the bundled collector. Remove the cache volumes, environment variables, published ports, and Caddy matcher which nothing uses, and correct the examples and comments which described them. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d40c-fdd9-73bd-a123-0a3e4f0dc0c3 Co-authored-by: Amp --- caddy/builtins/https.custom-cert.Caddyfile | 6 --- docker-compose/docker-compose.yaml | 45 ++++++++----------- .../aws-alb/docker-compose.override.yaml | 4 +- .../docker-compose.override.yaml | 45 ++----------------- .../docker-compose.override.yaml | 5 ++- docker-compose/jaeger/docker-compose.yaml | 7 +-- grafana/datasources/datasources.yaml | 4 +- otel-collector/config.yaml | 28 +++++++++--- 8 files changed, 54 insertions(+), 90 deletions(-) diff --git a/caddy/builtins/https.custom-cert.Caddyfile b/caddy/builtins/https.custom-cert.Caddyfile index 2a8986d8d..f182314dd 100644 --- a/caddy/builtins/https.custom-cert.Caddyfile +++ b/caddy/builtins/https.custom-cert.Caddyfile @@ -5,12 +5,6 @@ {$SRC_SITE_ADDRESS} -# Redirects all HTTP traffic to HTTPS (using custom SSL certificates disables the automatic HTTPS feature of Caddy, see https://caddyserver.com/docs/automatic-https) -@http { - protocol http -} -redir @http https://{host}{uri} - tls /sourcegraph.pem /sourcegraph.key # Add the reverse proxies IPs (or IP CIDR ranges) to the trusted_proxies list. diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 7abbfb2ee..b25a9eeb8 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -33,9 +33,8 @@ services: - CODEINTEL_PGDATABASE=sg - CODEINTEL_PGSSLMODE=disable - # If you are not running code insights, please see: - # https://sourcegraph.com/docs/self-hosted/deploy/docker-compose/operations#database-migrations for more information - # for information on how to configure Sourcegraph migrations + # For the migrator's environment variables and commands, see: + # https://sourcegraph.com/docs/self-hosted/updates/migrator/migrator-operations - CODEINSIGHTS_PGHOST=codeinsights-db - CODEINSIGHTS_PGPORT=5432 - CODEINSIGHTS_PGUSER=postgres @@ -117,7 +116,7 @@ services: # Description: Serves the frontend of Sourcegraph via HTTP(S). # - # Disk: 128GB / non-persistent SSD + # Disk: none # Ports exposed to other Sourcegraph services: 6060/TCP, 3080 (HTTP) # Ports exposed to the public internet: none # @@ -165,8 +164,6 @@ services: timeout: 10s retries: 10 start_period: 300s - volumes: - - 'sourcegraph-frontend-0:/mnt/cache' networks: - sourcegraph restart: always @@ -176,7 +173,7 @@ services: # Description: Serves the internal Sourcegraph frontend API. # - # Disk: 128GB / non-persistent SSD + # Disk: none # Ports exposed to other Sourcegraph services: 3090/TCP 6060/TCP # Ports exposed to the public internet: none # @@ -202,8 +199,6 @@ services: - 'GRAFANA_SERVER_URL=http://grafana:3370' - 'PROMETHEUS_URL=http://prometheus:9090' - 'OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317' - volumes: - - 'sourcegraph-frontend-internal-0:/mnt/cache' networks: - sourcegraph restart: always @@ -263,8 +258,6 @@ services: environment: - 'HOSTNAME=zoekt-webserver-0:6070' - 'SRC_FRONTEND_INTERNAL=http://sourcegraph-frontend-internal:3090' - - 'OPENTELEMETRY_DISABLED=false' - - 'OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317' volumes: - 'zoekt-0-shared:/data/index' networks: @@ -357,7 +350,7 @@ services: # Description: Manages background processes. # - # Disk: 128GB / non-persistent SSD + # Disk: none # Ports exposed to other Sourcegraph services: 3189/TCP 6060/TCP 6996/TCP (executor metrics) # Ports exposed to the public internet: none # @@ -372,8 +365,6 @@ services: environment: - 'SRC_FRONTEND_INTERNAL=sourcegraph-frontend-internal:3090' - 'OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317' - volumes: - - 'worker:/mnt/cache' networks: - sourcegraph restart: always @@ -431,7 +422,7 @@ services: # Description: Prometheus collects metrics and aggregates them into graphs. # # Disk: 200GB / persistent SSD - # Ports exposed to other Sourcegraph services: none + # Ports exposed to other Sourcegraph services: 9090/TCP # Ports exposed to the public internet: none (HTTP 9090 should be exposed to admins only) # prometheus: @@ -462,7 +453,16 @@ services: # # 'GF_AUTH_PROXY_ENABLED=true' # 'GF_AUTH_PROXY_HEADER_NAME=X-Forwarded-User' - # 'GF_SERVER_ROOT_URL=https://grafana.example.com' + # + # Add all of the following environment variables if you wish to use the `pgsql` datasource, + # to query a Postgres database from Grafana; the datasource does not work until all six are set: + # + # 'GRAFANA_PGSQL_HOST=pgsql' + # 'GRAFANA_PGSQL_PORT=5432' + # 'GRAFANA_PGSQL_USER=sg' + # 'GRAFANA_PGSQL_PASSWORD=sg' + # 'GRAFANA_PGSQL_DATABASE=sg' + # 'GRAFANA_PGSQL_SSLMODE=disable' grafana: container_name: grafana image: 'index.docker.io/sourcegraph/grafana:8.2.0@sha256:fd478ac1a5ce4552bfc388984a559548b5b7ea8001c2e65adcfb7e292bb54784' @@ -510,8 +510,6 @@ services: networks: - sourcegraph restart: always - expose: - - 48080 # Description: Publishes Prometheus metrics about the machine's hardware / operating system. # @@ -538,8 +536,6 @@ services: - '--no-collector.wifi' - '--no-collector.hwmon' - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' - expose: - - 9100 networks: - sourcegraph restart: always @@ -766,7 +762,7 @@ services: # Description: Ingests and exports OpenTelemetry data. # - # Ports exposed to other Sourcegraph services: 4317 (receiver), 55679 (zpages) + # Ports exposed to other Sourcegraph services: 4317 (OTLP gRPC), 4318 (OTLP HTTP), 8888 (metrics) # Ports exposed to the public internet: none otel-collector: container_name: otel-collector @@ -778,8 +774,8 @@ services: networks: - sourcegraph restart: always - # Default configurations are shipped in '/etc/otel-collector/configs/' - command: ['--config', '/etc/otel-collector/configs/logging.yaml'] + # Other configurations are shipped in '/etc/otel-collector/configs/' + command: ['--config', '/etc/otel-collector/config.yaml'] volumes: - '../otel-collector/config.yaml:/etc/otel-collector/config.yaml' @@ -794,10 +790,7 @@ volumes: prometheus-v2: redis-cache: redis-store: - worker: searcher-0: - sourcegraph-frontend-0: - sourcegraph-frontend-internal-0: zoekt-0-shared: networks: sourcegraph: diff --git a/docker-compose/examples/aws-alb/docker-compose.override.yaml b/docker-compose/examples/aws-alb/docker-compose.override.yaml index e904ba6ab..29dda96e9 100644 --- a/docker-compose/examples/aws-alb/docker-compose.override.yaml +++ b/docker-compose/examples/aws-alb/docker-compose.override.yaml @@ -1,5 +1,7 @@ # Docker Compose override snippet for customers using AWS ALBs to terminate TLS, -# and do not need TLS between the ALB and their Sourcegraph intance +# and do not need TLS between the ALB and their Sourcegraph instance +# Apply with `docker compose up -d --remove-orphans`, to remove the running caddy container, +# which otherwise holds port 80 ## Overrides for services defined in the base docker-compose.yaml files diff --git a/docker-compose/examples/aws-rds-databases/docker-compose.override.yaml b/docker-compose/examples/aws-rds-databases/docker-compose.override.yaml index a5a8ca557..b2527deea 100644 --- a/docker-compose/examples/aws-rds-databases/docker-compose.override.yaml +++ b/docker-compose/examples/aws-rds-databases/docker-compose.override.yaml @@ -3,19 +3,15 @@ -## Reusable blocks of environment variables +## Reusable block of environment variables -# For all containers, which connect to any of the databases -x-database-env-var-rds-iam-auth: &database-env-var-rds-iam-auth +# For migrator and both frontends, the only services which read these +# The other services get their database connection strings from frontend +x-database-env-vars: &database-env-vars # Using AWS RDS IAM authentication, not a password, so instruct the psql library to refresh the token PG_CONNECTION_UPDATER: EC2_ROLE_CREDENTIALS -# For both migrator and frontend -x-database-env-vars: &database-env-vars - - <<: *database-env-var-rds-iam-auth - # NOTE: Must clear / override the PGDATASOURCE env vars, otherwise frontends don't start up PGHOST: example-database-pgsql.cluster-example.us-west-2.rds.amazonaws.com PGPORT: 5432 @@ -47,11 +43,6 @@ x-database-env-vars: &database-env-vars services: - gitserver-0: - # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact - environment: - <<: *database-env-var-rds-iam-auth - migrator: @@ -86,18 +77,6 @@ services: # If migrator fails to start up due to drift detected, or other issues, please open a support ticket by emailing support@sourcegraph.com - precise-code-intel-worker: - # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact - environment: - <<: *database-env-var-rds-iam-auth - - - searcher-0: - # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact - environment: - <<: *database-env-var-rds-iam-auth - - sourcegraph-frontend-0: # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact environment: @@ -115,22 +94,6 @@ services: <<: *database-env-vars - syntactic-code-intel-worker: - # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact - environment: - <<: *database-env-var-rds-iam-auth - - - worker: - # Add / merge the env vars, to only overwrite the ones we need, and leave the others intact - environment: - <<: *database-env-var-rds-iam-auth - # Add additional env vars, as needed - # TELEMETRY_GATEWAY_EXPORTER_USERMETADATA_EXPORT_ENABLED: true - # TELEMETRY_GATEWAY_EXPORTER_USERMETADATA_EXPORT_ORGANIZATIONS: true - # TELEMETRY_GATEWAY_EXPORTER_USERMETADATA_EXPORT_USERS: true - - ## Disable services # Drop built-in database services and volumes from the base compose file diff --git a/docker-compose/examples/migrator-operations/docker-compose.override.yaml b/docker-compose/examples/migrator-operations/docker-compose.override.yaml index 4fc3d4a98..f1f247d66 100644 --- a/docker-compose/examples/migrator-operations/docker-compose.override.yaml +++ b/docker-compose/examples/migrator-operations/docker-compose.override.yaml @@ -22,8 +22,8 @@ services: # https://github.com/sourcegraph/artifacts/blob/main/migrations/codeinsights/1675347548_squashed_migrations_privileged/up.sql # After the extensions are successfully installed, manually mark these migrations as completed, - # by uncommenting these commands, one at a time, and running `docker compose up migrator` each time, - # then comment the last one out, and run the normal `docker compose up -d --remove-orphans` command, for a normal startup + # by replacing the `command: ["up"]` line above with each of these commands, one at a time, and running `docker compose up migrator` each time, + # then restore the `command: ["up"]` line, and run the normal `docker compose up -d --remove-orphans` command, for a normal startup # command: ["add-log", "-db=frontend", "-version=1680296731"] # command: ["add-log", "-db=codeintel", "-version=1679010276"] # command: ["add-log", "-db=codeinsights", "-version=1675347548"] @@ -33,6 +33,7 @@ services: ################################################################################ # Multi-version upgrades # See https://sourcegraph.com/docs/self-hosted/deploy/docker-compose/upgrade#multi-version-upgrades + # Replace the `command: ["up"]` line above with this command # command: ["upgrade", "--from=v6.4.0", "--to=v6.9.1277"] # Do not skip the drift check, or use other args, without instruction from Sourcegraph Customer Support Engineering diff --git a/docker-compose/jaeger/docker-compose.yaml b/docker-compose/jaeger/docker-compose.yaml index a74593689..5229b0b59 100644 --- a/docker-compose/jaeger/docker-compose.yaml +++ b/docker-compose/jaeger/docker-compose.yaml @@ -3,7 +3,7 @@ services: # (https://www.jaegertracing.io/docs/2.16/getting-started/) for distributed tracing. # # Disk: none - # Ports exposed to other Sourcegraph services: 5778/TCP 4317/TCP 4318/TCP + # Ports exposed to other Sourcegraph services: 4317/TCP (OTLP gRPC) 16686/HTTP # Ports exposed to the public internet: none # Ports exposed to site admins only: 16686/HTTP # @@ -17,11 +17,6 @@ services: ports: # Query port - '0.0.0.0:16686:16686' - # OTLP ports - - '0.0.0.0:4317:4317' # gRPC - - '0.0.0.0:4318:4318' # HTTP - # Remote sampling - - '0.0.0.0:5778:5778' networks: - sourcegraph restart: always diff --git a/grafana/datasources/datasources.yaml b/grafana/datasources/datasources.yaml index 95cf8c4b2..020edb78f 100644 --- a/grafana/datasources/datasources.yaml +++ b/grafana/datasources/datasources.yaml @@ -8,7 +8,7 @@ datasources: isDefault: true editable: false - name: Jaeger - type: Jaeger + type: jaeger access: proxy - url: http://jaeger-query:16686/-/debug/jaeger + url: http://jaeger:16686/-/debug/jaeger editable: false diff --git a/otel-collector/config.yaml b/otel-collector/config.yaml index a06de1961..5acf1b286 100644 --- a/otel-collector/config.yaml +++ b/otel-collector/config.yaml @@ -1,13 +1,21 @@ -# This is the template configuration for OpenTelemetry collector, and is not used by default. +# This is the default configuration for the OpenTelemetry collector. # It is mounted to '/etc/otel-collector/config.yaml' in the otel-collector container. receivers: otlp: protocols: - grpc: # port 4317 - http: # port 4321 + # The collector only listens on localhost by default, which other containers cannot reach + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" exporters: + # Logs a sample of the received traces to the container's output + debug: + verbosity: basic + sampling_initial: 5 + sampling_thereafter: 200 # Add your exporter(s) configuration here. For each exporter, make sure it is enabled # in the service configuration below. # Refer to our OpenTelemetry docs for information on how to configure different exporters: @@ -22,12 +30,20 @@ extensions: service: telemetry: metrics: - address: ":8888" + # Prometheus scrapes the collector's own metrics from port 8888 + readers: + - pull: + exporter: + prometheus: + host: "0.0.0.0" + port: 8888 + without_type_suffix: true + without_units: true pipelines: traces: exporters: - # Add the name of your exporter(s) here, e.g.: - # - logging + # Add the name of your exporter(s) here + - debug receivers: - otlp # Do not remove this receiver extensions: From 301672a775e399fab82ec9a846dfbed50eb66b7a Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:36:35 -0600 Subject: [PATCH 4/5] List both code intel workers as debug services, drop a rule which matches nothing The frontend's /-/debug/ page lists the services in SRC_PROF_SERVICES. precise-code-intel-worker and syntactic-code-intel-worker both serve a debug server on 6060, but were not in the list. container:process_cpu_seconds_total:ratio_rate5m divides a series with only an instance label by engine_daemon_engine_cpus_cpus, which also has a job label, so the two sides never match and the rule records nothing, even when Prometheus scrapes the Docker daemon. Nothing reads it. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d40c-fdd9-73bd-a123-0a3e4f0dc0c3 Co-authored-by: Amp --- docker-compose/docker-compose.yaml | 2 ++ prometheus/docker_rules.yml | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index b25a9eeb8..40bf8e86a 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -154,6 +154,8 @@ services: { "Name": "frontend-internal-0", "Host": "sourcegraph-frontend-internal:6060" }, { "Name": "gitserver-0", "Host": "gitserver-0:6060" }, { "Name": "searcher-0", "Host": "searcher-0:6060" }, + { "Name": "precise-code-intel-worker", "Host": "precise-code-intel-worker:6060" }, + { "Name": "syntactic-code-intel-worker", "Host": "syntactic-code-intel-worker:6060" }, { "Name": "worker", "Host": "worker:6060" }, { "Name": "zoekt-indexserver-0", "Host": "zoekt-indexserver-0:6072" }, { "Name": "zoekt-webserver-0", "Host": "zoekt-webserver-0:6070", "DefaultPath": "/debug/requests/" } diff --git a/prometheus/docker_rules.yml b/prometheus/docker_rules.yml index 0786eb780..b59be0075 100644 --- a/prometheus/docker_rules.yml +++ b/prometheus/docker_rules.yml @@ -3,8 +3,6 @@ groups: - name: docker.rules rules: - - record: container:process_cpu_seconds_total:ratio_rate5m - expr: sum by (instance) (rate(process_cpu_seconds_total[5m])) / engine_daemon_engine_cpus_cpus - record: container:process_cpu_seconds_total:sum expr: sum by (instance) (irate(process_cpu_seconds_total[1m])) - record: container:process_resident_memory_bytes:max From bffc6b73b903a7dbe0b86061d2eda5fed9fa3dbe Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:41:30 -0600 Subject: [PATCH 5/5] examples: Note the exporter targets which stay down, and Git's lowercase http_proxy With the built-in databases removed, Prometheus keeps the three exporter targets and shows them as down. Git ignores the uppercase HTTP_PROXY for http:// URLs. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d40c-fdd9-73bd-a123-0a3e4f0dc0c3 Co-authored-by: Amp --- .../examples/aws-rds-databases/docker-compose.override.yaml | 1 + .../examples/external-databases/docker-compose.override.yaml | 1 + .../examples/outbound-http-proxy/docker-compose.override.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/docker-compose/examples/aws-rds-databases/docker-compose.override.yaml b/docker-compose/examples/aws-rds-databases/docker-compose.override.yaml index b2527deea..17bf1689f 100644 --- a/docker-compose/examples/aws-rds-databases/docker-compose.override.yaml +++ b/docker-compose/examples/aws-rds-databases/docker-compose.override.yaml @@ -97,6 +97,7 @@ services: ## Disable services # Drop built-in database services and volumes from the base compose file + # Prometheus still lists the three removed exporters as targets, and shows them as down; this is expected pgsql: !reset {} pgsql-exporter: !reset {} codeintel-db: !reset {} diff --git a/docker-compose/examples/external-databases/docker-compose.override.yaml b/docker-compose/examples/external-databases/docker-compose.override.yaml index 9106e19a4..46cd14927 100644 --- a/docker-compose/examples/external-databases/docker-compose.override.yaml +++ b/docker-compose/examples/external-databases/docker-compose.override.yaml @@ -49,6 +49,7 @@ services: # Disable built-in databases + # Prometheus still lists the three removed exporters as targets, and shows them as down; this is expected codeinsights-db-exporter: !reset {} codeinsights-db: !reset {} codeintel-db-exporter: !reset {} diff --git a/docker-compose/examples/outbound-http-proxy/docker-compose.override.yaml b/docker-compose/examples/outbound-http-proxy/docker-compose.override.yaml index 3045b3ed3..1a1d7c45f 100644 --- a/docker-compose/examples/outbound-http-proxy/docker-compose.override.yaml +++ b/docker-compose/examples/outbound-http-proxy/docker-compose.override.yaml @@ -56,6 +56,7 @@ services: <<: *proxy-env-vars # Uncomment if connections to code hosts outside your private network need to use the proxy + # Git ignores the uppercase HTTP_PROXY for http:// code host URLs, so also set lowercase http_proxy for those # gitserver-0: # environment: # <<: *proxy-env-vars