From 666b797f213d844c1b82c692d1f316d4aeb2e37c Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:38:45 -0600 Subject: [PATCH 1/2] self-hosted: Correct Docker Compose settings which Sourcegraph does not read - INDEXED_SEARCH_INDEXERS is not a variable; the frontend reads INDEXED_SEARCH_INDEXER. - Nothing reads SYMBOLS_URL or AWS_STS_REGIONAL_ENDPOINTS. - Only the frontends read SRC_GIT_SERVERS, so the gitserver shard example no longer sets it on worker. - The collector has a debug exporter, not logging, and listens on localhost unless the receiver names an address. - The Compose debug ports were those of the removed single-container image. - The external database steps edited the base file of the 3.37 branch; point at the override examples in deploy-sourcegraph-docker instead, and list the code insights database variables. - The Caddyfile volume lines did not match the base file. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d40c-fdd9-73bd-a123-0a3e4f0dc0c3 Co-authored-by: Amp --- .../deploy/docker-compose/configuration.mdx | 10 ++--- .../deploy/without-service-discovery.mdx | 3 +- .../external-services/postgres.mdx | 38 +++++++++---------- docs/self-hosted/http-https-configuration.mdx | 6 +-- .../observability/opentelemetry.mdx | 9 +++-- docs/self-hosted/pprof.mdx | 14 +++---- 6 files changed, 37 insertions(+), 43 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/configuration.mdx b/docs/self-hosted/deploy/docker-compose/configuration.mdx index 20e56a50f..273ef8c6e 100644 --- a/docs/self-hosted/deploy/docker-compose/configuration.mdx +++ b/docs/self-hosted/deploy/docker-compose/configuration.mdx @@ -130,14 +130,14 @@ When adding replicas for `gitserver`, `searcher`, `zoekt-indexserver`, or `zoekt services: sourcegraph-frontend-0: environment: - - 'INDEXED_SEARCH_INDEXERS=1' + - 'INDEXED_SEARCH_INDEXER=1' - 'INDEXED_SEARCH_SERVERS=1' - 'SEARCHER_URL=1' - 'SRC_GIT_SERVERS=1' sourcegraph-frontend-internal: environment: - - 'INDEXED_SEARCH_INDEXERS=1' + - 'INDEXED_SEARCH_INDEXER=1' - 'INDEXED_SEARCH_SERVERS=1' - 'SEARCHER_URL=1' - 'SRC_GIT_SERVERS=1' @@ -179,15 +179,11 @@ services: cpus: 6 mem_limit: '6g' # Set the following environment variables to generate the replica endpoints - environment: &env_gitserver + environment: - 'SRC_GIT_SERVERS=2' # Use the same override values as sourcegraph-frontend-0 above sourcegraph-frontend-internal: <<: *frontend - # Add the updated environment for gitserver from frontend to worker using anchor - worker: - environment: - - *env_gitserver # Add a new volume assigned to the new gitserver replica volumes: gitserver-1: diff --git a/docs/self-hosted/deploy/without-service-discovery.mdx b/docs/self-hosted/deploy/without-service-discovery.mdx index bc581a16c..8753d0968 100644 --- a/docs/self-hosted/deploy/without-service-discovery.mdx +++ b/docs/self-hosted/deploy/without-service-discovery.mdx @@ -13,6 +13,5 @@ If you are deploying Sourcegraph in an environment **without Kubernetes service - INDEXED_SEARCH_SERVERS - SEARCHER_URL - SRC_GIT_SERVERS -- SYMBOLS_URL -For detailed instructions on how to set these environment variables in a Docker Compose deployment, see the [Docker Compose configuration guide](/self-hosted/deploy/docker-compose/configuration#set-environment-variables). +For detailed instructions on how to set these environment variables in a Docker Compose deployment, see the [Docker Compose configuration guide](/self-hosted/deploy/docker-compose/configuration#add-replicas). diff --git a/docs/self-hosted/external-services/postgres.mdx b/docs/self-hosted/external-services/postgres.mdx index 6340ef665..1b7bdca5b 100644 --- a/docs/self-hosted/external-services/postgres.mdx +++ b/docs/self-hosted/external-services/postgres.mdx @@ -41,6 +41,15 @@ To externalize the _code navigation database_, use the following prefixed `CODEI - `CODEINTEL_PGDATABASE` - `CODEINTEL_PGSSLMODE` +To externalize the _code insights database_, use the following prefixed `CODEINSIGHTS_PG*` variables: + +- `CODEINSIGHTS_PGHOST` +- `CODEINSIGHTS_PGPORT` +- `CODEINSIGHTS_PGUSER` +- `CODEINSIGHTS_PGPASSWORD` +- `CODEINSIGHTS_PGDATABASE` +- `CODEINSIGHTS_PGSSLMODE` + If you have configured both the frontend (pgsql) and code navigation (codeintel-db) databases with the same values, the Sourcegraph instance will @@ -51,27 +60,16 @@ To externalize the _code navigation database_, use the following prefixed `CODEI ### Docker Compose -1. Add/modify the following environment variables to all of the `sourcegraph-frontend-*` services, the `sourcegraph-frontend-internal` service, and the `migrator` service (for Sourcegraph versions 3.37+) in [docker-compose.yaml](https://github.com/sourcegraph/deploy-sourcegraph-docker/blob/3.37/docker-compose/docker-compose.yaml): +1. Copy the [external databases example](https://github.com/sourcegraph/deploy-sourcegraph-docker/blob/main/docker-compose/examples/external-databases/docker-compose.override.yaml) into your `docker-compose.override.yaml` file, and set the values for your databases. The example sets the variables on the `migrator`, `sourcegraph-frontend-0`, and `sourcegraph-frontend-internal` services, and removes the built-in database services and their dependencies. -```yaml -sourcegraph-frontend-0: - environment: - - 'PGHOST=psql1.example.com' - - 'PGUSER=sourcegraph' - - 'PGPASSWORD=secret' - - 'PGDATABASE=sourcegraph' - - 'PGSSLMODE=require' - - 'CODEINTEL_PGHOST=psql2.example.com' - - 'CODEINTEL_PGUSER=sourcegraph' - - 'CODEINTEL_PGPASSWORD=secret' - - 'CODEINTEL_PGDATABASE=sourcegraph-codeintel' - - 'CODEINTEL_PGSSLMODE=require' +2. Apply the override file: + +```bash +docker compose -f docker-compose.yaml -f docker-compose.override.yaml up -d --remove-orphans ``` See [Environment variables in Compose](https://docs.docker.com/compose/environment-variables/) for other ways to pass these environment variables to the relevant services (including from the command line, a `.env` file, etc.). -1. Comment out / remove the internal `pgsql` and `codeintel-db` services in [docker-compose.yaml](https://github.com/sourcegraph/deploy-sourcegraph-docker/blob/3.37/docker-compose/docker-compose.yaml) since Sourcegraph is using the external one now. - ### Kubernetes Update the `PG*` and `CODEINTEL_PG*` environment variables in the `sourcegraph-frontend` deployment YAML file to point to the external frontend (`pgsql`) and code navigation (`codeintel-db`) PostgreSQL instances, respectively. Again, these must not point to the same database or the Sourcegraph instance will refuse to start. @@ -142,9 +140,7 @@ Configure the following environment variables for the **`sourcegraph-frontend`** - `CODEINSIGHTS_PGUSER=<>` - this should be the database accounts created above - `CODEINSIGHTS_PGDATABASE=<>` -**Optional:** If your workloads are in a specific AWS Region and you want to use regional STS endpoints for improved latency and resiliency, also set: - -- `AWS_STS_REGIONAL_ENDPOINTS=regional` - This configures the AWS SDK to use regional STS endpoints (e.g., `sts.us-west-2.amazonaws.com`) instead of the global endpoint (`sts.amazonaws.com`). This is recommended for better performance and resilience. See [AWS documentation](https://docs.aws.amazon.com/sdkref/latest/guide/feature-sts-regionalized-endpoints.html) for more details. +For Docker Compose, start from the [AWS RDS example](https://github.com/sourcegraph/deploy-sourcegraph-docker/blob/main/docker-compose/examples/aws-rds-databases/docker-compose.override.yaml) override file. ## Usage with PgBouncer @@ -154,7 +150,7 @@ When [PgBouncer] is used, we need to include `statement_cache_mode=describe` in ### Docker Compose -1. Add/modify the following environment variables to all of the `sourcegraph-frontend-*` services, the `sourcegraph-frontend-internal` service, and the `migrator` service (for Sourcegraph versions 3.37+) in [docker-compose.yaml](https://github.com/sourcegraph/deploy-sourcegraph-docker/blob/3.37/docker-compose/docker-compose.yaml): +1. Set the following environment variables on the `migrator`, `sourcegraph-frontend-0`, and `sourcegraph-frontend-internal` services in your `docker-compose.override.yaml` file: ```yaml sourcegraph-frontend-0: @@ -165,7 +161,7 @@ sourcegraph-frontend-0: See ["Environment variables in Compose"](https://docs.docker.com/compose/environment-variables/) for other ways to pass these environment variables to the relevant services (including from the command line, a `.env` file, etc.). -1. Comment out / remove the internal `pgsql` and `codeintel-db` services in [docker-compose.yaml](https://github.com/sourcegraph/deploy-sourcegraph-docker/blob/3.37/docker-compose/docker-compose.yaml) since Sourcegraph is using the external one now. +2. Remove the built-in database services and their dependencies, as the [external databases example](https://github.com/sourcegraph/deploy-sourcegraph-docker/blob/main/docker-compose/examples/external-databases/docker-compose.override.yaml) does. ### Kubernetes diff --git a/docs/self-hosted/http-https-configuration.mdx b/docs/self-hosted/http-https-configuration.mdx index 697efcba7..8bba88141 100644 --- a/docs/self-hosted/http-https-configuration.mdx +++ b/docs/self-hosted/http-https-configuration.mdx @@ -17,7 +17,7 @@ Sourcegraph's [Docker Compose deployment](/self-hosted/deploy/docker-compose/) u - HTTPS with automatically provisioned Let's Encrypt certificates - HTTPS with custom certificates that you provide -Usage instructions are provided via [the `caddy` service's inline comments inside the `docker-compose.yaml` definition](https://github.com/sourcegraph/deploy-sourcegraph-docker/blob/3.14/docker-compose/docker-compose.yaml#L3:L58). Detailed steps are found below. +Usage instructions are provided via [the `caddy` service's inline comments inside the `docker-compose.yaml` definition](https://github.com/sourcegraph/deploy-sourcegraph-docker/blob/main/docker-compose/docker-compose.yaml). Detailed steps are found below. **Important:** When setting up caddy's automatic Lets Encrypt TLS certification for HTTPS we strongly recommended to test with the staging configuration first. However, the TLS certs provided by Lets Encrypt staging will not be verifiable, and will display an `unknown certificate` error. Once you are done testing, you will need to comment out the staging env variable and uncomment the production env variable for fully functional HTTPS. @@ -36,13 +36,13 @@ In your [docker-compose.yaml](https://github.com/sourcegraph/deploy-sourcegraph- 1. In Volumes section of the compose file comment out the following line ``` - - '//builtins/http.Caddyfile:/etc/caddy/Caddyfile' + - '../caddy/builtins/http.Caddyfile:/etc/caddy/Caddyfile' ``` 1. In Volumes section of the compose file uncomment the following line: ``` - - '//builtins/https.custom-cert.Caddyfile:/etc/caddy/Caddyfile' + - '../caddy/builtins/https.custom-cert.Caddyfile:/etc/caddy/Caddyfile' ``` 1. In Volumes section of the compose file uncomment and update the following line with your custom cert path: diff --git a/docs/self-hosted/observability/opentelemetry.mdx b/docs/self-hosted/observability/opentelemetry.mdx index ab7ef45e4..afed2e227 100644 --- a/docs/self-hosted/observability/opentelemetry.mdx +++ b/docs/self-hosted/observability/opentelemetry.mdx @@ -41,12 +41,15 @@ To enable a backend, it must be adding to both the `exporters` block and the `se receivers: otlp: protocols: + # The collector listens on localhost by default, which other containers cannot reach grpc: + endpoint: '0.0.0.0:4317' http: + endpoint: '0.0.0.0:4318' exporters: - logging: # Export traces as log events - loglevel: warn + debug: # Export traces as log events + verbosity: basic sampling_initial: 5 sampling_thereafter: 200 @@ -56,7 +59,7 @@ service: receivers: - otlp exporters: - - logging # The exporter name must be added here to enable it + - debug # The exporter name must be added here to enable it ``` ## Sampling traces diff --git a/docs/self-hosted/pprof.mdx b/docs/self-hosted/pprof.mdx index 116329ff5..a82f9634a 100644 --- a/docs/self-hosted/pprof.mdx +++ b/docs/self-hosted/pprof.mdx @@ -53,13 +53,13 @@ go tool pprof frontend-bin heap.out This is a table of Sourcegraph backend debug ports in the two deployment contexts: -| | kubernetes | docker | -| ----------------- | ---------- | ------ | -| frontend | 6060 | 6063 | -| gitserver | 6060 | 6068 | -| searcher | 6060 | 6069 | -| zoekt-indexserver | 6060 | 6072 | -| zoekt-webserver | 6060 | 3070 | +| | Kubernetes | Docker Compose | +| ----------------- | ---------- | -------------- | +| frontend | 6060 | 6060 | +| gitserver | 6060 | 6060 | +| searcher | 6060 | 6060 | +| zoekt-indexserver | 6060 | 6072 | +| zoekt-webserver | 6060 | 6070 | ## Profiling kinds From 10613c2d62a5780a51d8966e53f3b07c655ef615 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:45:57 -0600 Subject: [PATCH 2/2] self-hosted: Fix Docker Compose snippets which fail when followed Each snippet was run on Sourcegraph 8.2.0 with Docker Compose: - Disable a service: mark the dependency on the disabled service as not required, or Compose rejects the project - Git SSH: the gitserver user (100:101) must own the files; `chmod -R 600` on the directory makes it unreadable - OpenTelemetry: `$JAEGER_HOST` is not expanded, `${env:JAEGER_HOST}` is; the image's jaeger.yaml also reads JAEGER_OTLP_GRPC_PORT; the googlecloud exporter rejects `retry_on_failure` - Grafana: the base file publishes no Grafana port, so show the override which publishes it on loopback; give the Compose authentication override its own block under `services:` Amp-Thread-ID: https://ampcode.com/threads/T-01a1d40c-fdd9-73bd-a123-0a3e4f0dc0c3 Co-authored-by: Amp --- .../deploy/docker-compose/configuration.mdx | 28 +++++++++++++++++-- docs/self-hosted/observability/metrics.mdx | 24 ++++++++++++---- .../observability/opentelemetry.mdx | 8 ++---- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/docs/self-hosted/deploy/docker-compose/configuration.mdx b/docs/self-hosted/deploy/docker-compose/configuration.mdx index 273ef8c6e..fe16ad93c 100644 --- a/docs/self-hosted/deploy/docker-compose/configuration.mdx +++ b/docs/self-hosted/deploy/docker-compose/configuration.mdx @@ -113,7 +113,17 @@ services: - '~/sg/volume-maps/gitserver/.ssh:/home/sourcegraph/.ssh' ``` -> WARNING: The permissions on your SSH / Git configuration must be set to be readable by the user in the `gitserver` container. For example, run `chmod -v -R 600 ~/sg/volume-maps/gitserver/.ssh` in the folder on the host machine. +> WARNING: The `gitserver` container runs as user ID `100` and group ID `101`. SSH ignores or rejects the configuration unless that user owns the directory and its files, the directory has mode `700`, and the files have mode `600`. + +Set the owner and the modes on the host machine: + +```bash +chmod 700 ~/sg/volume-maps/gitserver/.ssh +chmod 600 ~/sg/volume-maps/gitserver/.ssh/* +sudo chown -R 100:101 ~/sg/volume-maps/gitserver/.ssh +``` + +After the `chown`, your host user can no longer read the directory, so later changes to these files need `sudo`. ### Git HTTP(S) basic username + password authentication @@ -193,7 +203,9 @@ volumes: You can disable services by assigning them to one or more [profiles](https://docs.docker.com/compose/profiles/), so that when running the `docker compose up` command, services assigned to profiles will not be started unless explicitly specified in the command (e.g., `docker compose --profile disabled up`). -For example, when you need to disable the bundled databases to use external databases, you can assign the bundled database containers to a profile called `disabled`: +Services which list the disabled service under `depends_on` fail to start with `depends on undefined service`, unless you mark that dependency as not required. + +For example, to disable the bundled `codeintel-db` container, assign it to a profile called `disabled`, and mark it as not required on the two services which depend on it: ```yaml # docker-compose.override.yaml @@ -201,8 +213,20 @@ services: codeintel-db: profiles: - disabled + migrator: + depends_on: + codeintel-db: + condition: service_healthy + required: false + sourcegraph-frontend-internal: + depends_on: + codeintel-db: + condition: service_healthy + required: false ``` +To replace the bundled databases with external ones, see [using your own PostgreSQL server](/self-hosted/external-services/postgres). + ## Expose debug port To generate [pprof profiling data](/self-hosted/pprof), you must configure your deployment to expose port 6060 on one of your frontend containers, for example: diff --git a/docs/self-hosted/observability/metrics.mdx b/docs/self-hosted/observability/metrics.mdx index cac6f88eb..667bfb266 100644 --- a/docs/self-hosted/observability/metrics.mdx +++ b/docs/self-hosted/observability/metrics.mdx @@ -81,16 +81,25 @@ kubectl port-forward svc/grafana 3370:30070 Grafana will be available http://localhost:3370/-/debug/grafana. -If you are using the [Docker Compose deployment option](/self-hosted/deploy/), Grafana is available locally at http://localhost:3370/-/debug/grafana without any additional setup. +If you are using the [Docker Compose deployment option](/self-hosted/deploy/docker-compose/), publish Grafana's port on the host's loopback address in your override file, then redeploy: -If Sourcegraph is deployed to a remote server, then access via an SSH tunnel using a tool such as [sshuttle](https://github.com/sshuttle/sshuttle) is required to establish a secure connection to Grafana. -To access the remote server using `sshuttle` from your local machine: +```yaml +# docker-compose.override.yaml +services: + grafana: + ports: + - '127.0.0.1:3370:3370' +``` + +Grafana will be available on the host at http://localhost:3370/-/debug/grafana. + +If Sourcegraph is deployed to a remote server, forward the port over SSH from your local machine: ```bash -sshuttle -r user@sourcegraph.example.com 0/0 +ssh -N -L 3370:localhost:3370 user@sourcegraph.example.com ``` -Grafana will be available at http://sourcegraph.example.com:3370/-/debug/grafana. +Grafana will be available on your local machine at http://localhost:3370/-/debug/grafana. ### Grafana security @@ -127,8 +136,11 @@ grafana: value: 'custom-admin-username' GF_SECURITY_ADMIN_PASSWORD: value: 'custom-admin-password' +``` - # Docker Compose override +```yaml +# Docker Compose override +services: grafana: environment: - 'GF_AUTH_ANONYMOUS_ENABLED=false' diff --git a/docs/self-hosted/observability/opentelemetry.mdx b/docs/self-hosted/observability/opentelemetry.mdx index afed2e227..2c9580394 100644 --- a/docs/self-hosted/observability/opentelemetry.mdx +++ b/docs/self-hosted/observability/opentelemetry.mdx @@ -212,18 +212,18 @@ If you're looking for information about Sourcegraph's bundled Jaeger instance, h Refer to the [Jaeger](https://opentelemetry.io/docs/languages/js/exporters/#jaeger) documentation for options. -If you must use your own Jaeger instance, and if the bundled otel-collector's basic configuration with the Jaeger OTLP exporter enabled meets your needs, configure the otel-collector's startup command to `/usr/bin/otelcol-sourcegraph --config=/etc/otel-collector/configs/jaeger.yaml`. Note that this requires the environment variable `$JAEGER_HOST` to be set on the otel-collector service / container: +If you must use your own Jaeger instance, and if the bundled otel-collector's basic configuration with the Jaeger OTLP exporter enabled meets your needs, configure the otel-collector's startup command to `/usr/bin/otelcol-sourcegraph --config=/etc/otel-collector/configs/jaeger.yaml`. This requires the environment variables `JAEGER_HOST` and `JAEGER_OTLP_GRPC_PORT` to be set on the otel-collector service / container. In your own config file, the equivalent exporter is: ```yaml # otel-collector config.yaml exporters: otlp/jaeger: - endpoint: '$JAEGER_HOST:4317' + endpoint: '${env:JAEGER_HOST}:4317' tls: insecure: true ``` -The Sourcegraph frontend automatically proxies Jaeger's web UI to make it available at `/-/debug/jaeger`. You can proxy your own Jaeger instance instead by configuring the `JAEGER_SERVER_URL` environment variable on the `frontend` containers, and the `QUERY_BASE_PATH='/-/debug/jaeger'` environment variable on your `jaeger` container. +The Sourcegraph frontend automatically proxies Jaeger's web UI to make it available at `/-/debug/jaeger`. You can proxy your own Jaeger instance instead by configuring the `JAEGER_SERVER_URL` environment variable on the `frontend` containers, and serving your Jaeger instance's web UI under the base path `/-/debug/jaeger`. ### Google Cloud @@ -234,6 +234,4 @@ exporters: googlecloud: # See docs project: project-name # Project name can also be fetched from secrets - retry_on_failure: - enabled: false ```