amp: Add Amp Orb project setup

.agents/setup

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Prepares an Amp orb (https://ampcode.com/docs/orbs) to run Sourcegraph with Docker Compose.
+# Amp saves the result as a snapshot for new orbs, so this only installs.
+# .amp/services.yaml starts Sourcegraph.
+set -euo pipefail
+
+step() { echo "==> $* (${SECONDS}s)"; }
+
+step 'Start Docker'
+sudo systemctl enable --now docker
+
+if ! sudo docker compose version >/dev/null 2>&1; then
+	step 'Install Docker Compose'
+	sudo apt-get update
+	sudo apt-get install -y docker-compose-plugin
+fi
+
+step 'Write guidance for agents'
+guidance_file="$HOME/.config/amp/AGENTS.md"
+guidance_heading='## Sourcegraph in this orb'
+mkdir -p "$(dirname "$guidance_file")"
+touch "$guidance_file"
+# Replace the section on every run, because Amp reruns setup on its earlier snapshot
+other_guidance=$(awk -v heading="$guidance_heading" '$0 == heading { skip = 1; next } /^## / { skip = 0 } !skip' "$guidance_file")
+{
+	if [[ -n "$other_guidance" ]]; then
+		printf '%s\n\n' "$other_guidance"
+	fi
+	cat <<EOF
+$guidance_heading
+
+This orb is for developing and testing the Docker Compose deployment in this repository. To develop
+Sourcegraph itself, use \`sg\` in the sourcegraph/sourcegraph repository instead.
+
+- \`amp orb services ensure\` starts Sourcegraph with Jaeger and an executor, from the Compose files in
+  \`docker-compose/\`, \`docker-compose/jaeger/\`, and \`docker-compose/executors/\`, plus
+  \`docker-compose/docker-compose.override.yaml\` when it exists, and prints the Executors and Jaeger
+  portal URLs. The Sourcegraph portal URL is the Executors URL without its path.
+- The first site admin's username, email, and password are in \`\$SOURCEGRAPH_ADMIN_USERNAME\`,
+  \`\$SOURCEGRAPH_ADMIN_EMAIL\`, and \`\$SOURCEGRAPH_ADMIN_PASSWORD\`, from the Amp project's environment variables.
+  When you start Sourcegraph for the user, give them the portal URL, the username, and the password.
+- \`amp orb service logs sourcegraph\` shows the logs of every container.
+  After changing a Compose file, run \`amp orb service restart sourcegraph\`.
+- Docker needs \`sudo\`, such as \`sudo docker ps\` or \`sudo docker logs executor\`.
+EOF
+} >"$guidance_file.new"
+mv "$guidance_file.new" "$guidance_file"
+
+step 'Done'

.amp/Caddyfile

@@ -0,0 +1,20 @@
+# caddy/builtins/http.Caddyfile, changed so Sourcegraph loads in the orb's Portal tab, which shows it in an iframe.
+# Sourcegraph refuses requests from an iframe, and tells browsers not to show its pages in one.
+# This turns off Sourcegraph's clickjacking protection, which is fine only for a disposable test deployment
+:80
+
+# The health check in .amp/services.yaml: 404 until .amp/sourcegraph.sh has set up the site admin and site configuration
+handle /-/amp-ready {
+	root * /amp-ready
+	rewrite * /ready
+	file_server
+}
+
+handle {
+	reverse_proxy {
+		to {$SRC_FRONTEND_ADDRESSES}
+		trusted_proxies 0.0.0.0/0
+		header_up -Sec-Fetch-Dest
+		header_down -X-Frame-Options
+	}
+}

.amp/docker-compose.override.yaml

@@ -0,0 +1,26 @@
+# Changes the Docker Compose deployment for an Amp orb. .amp/sourcegraph.sh sets both variables
+services:
+  # Publishes the port Amp chose for the service, which Amp checks as a user who cannot open port 80,
+  # and port 80 for the containers the executor starts. Not port 443, where the portal proxy in the orb listens
+  caddy:
+    ports: !override
+      - '${PORT:?}:80'
+      - '80:80'
+    # Relative to docker-compose/, the directory of the first Compose file
+    volumes:
+      - '../.amp/Caddyfile:/etc/caddy/Caddyfile'
+      - '${READY_DIRECTORY:?}:/amp-ready:ro'
+  # Connects the executor from docker-compose/executors/ to Sourcegraph,
+  # with the token .amp/sourcegraph.sh also sets as executors.accessToken in the site configuration
+  executor:
+    environment:
+      - EXECUTOR_FRONTEND_PASSWORD=${EXECUTOR_TOKEN:?}
+      # Every queue. The base file leaves EXECUTOR_QUEUE_NAME empty, which the executor requires with this
+      - EXECUTOR_QUEUE_NAMES=batches,batches-hooks,codeintel
+      # Lets the containers the executor starts reach Caddy at host.docker.internal, the executors.frontendURL
+      - EXECUTOR_DOCKER_ADD_HOST_GATEWAY=true
+  # Docker refuses a CPU limit above the CPUs in the orb, and an a1.medium orb has 4
+  zoekt-indexserver-0:
+    cpus: 4
+  zoekt-webserver-0:
+    cpus: 4

.amp/services.yaml

@@ -0,0 +1,13 @@
+# Services Amp runs in an orb: https://ampcode.com/docs/orbs/portals
+# `amp orb services ensure` starts them, and so does opening the Portal tab
+services:
+  sourcegraph:
+    command: .amp/sourcegraph.sh
+    # Served by Caddy once .amp/sourcegraph.sh has created the site admin and configured the site
+    health: /-/amp-ready
+    # .amp/sourcegraph.sh adds the Sourcegraph link, with the admin password in its note
+    portals:
+      - url: /admin/executors
+        title: Executors
+      - url: /-/debug/jaeger/
+        title: Jaeger

.amp/sourcegraph.sh

@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# Runs Sourcegraph in an Amp orb, as the `sourcegraph` service in .amp/services.yaml.
+# Starts the Docker Compose deployment with Jaeger and an executor, creates the first site admin,
+# then follows the logs until Amp stops the service.
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+frontend_url="http://localhost:${PORT}"
+portal_url=${PUBLIC_URL%/}
+# The first site admin, from the Amp project's environment variables, so it is the same in every orb
+admin_username=${SOURCEGRAPH_ADMIN_USERNAME:?"Set it in the Amp project's environment variables"}
+admin_email=${SOURCEGRAPH_ADMIN_EMAIL:?"Set it in the Amp project's environment variables"}
+admin_password=${SOURCEGRAPH_ADMIN_PASSWORD:?"Set it in the Amp project's environment variables"}
+secrets_directory="$HOME/.config/sourcegraph"
+
+# Each orb makes its own executor token
+executor_token_file="$secrets_directory/executor-token"
+if [[ ! -s "$executor_token_file" ]]; then
+	(umask 077 && mkdir -p "$secrets_directory" && openssl rand -hex 16 >"$executor_token_file")
+fi
+executor_token=$(<"$executor_token_file")
+# The token last written to the site configuration, which shows it only as REDACTED
+configured_token_file="$secrets_directory/configured-executor-token"
+
+# Caddy serves this file at /-/amp-ready, the health check in .amp/services.yaml, so Amp reports the service
+# ready only once the site admin exists and the site is configured
+ready_directory="$secrets_directory/ready"
+mkdir -p "$ready_directory"
+rm -f "$ready_directory/ready"
+
+# Variables for .amp/docker-compose.override.yaml. A file keeps the token out of the sudo log
+compose_variables="$secrets_directory/compose.env"
+(umask 077 && printf 'PORT=%s\nEXECUTOR_TOKEN=%s\nREADY_DIRECTORY=%s\n' "$PORT" "$executor_token" "$ready_directory" \
+	>"$compose_variables")
+compose=(sudo docker compose --env-file "$compose_variables"
+	--file docker-compose/docker-compose.yaml
+	--file docker-compose/jaeger/docker-compose.yaml
+	--file docker-compose/executors/executor.docker-compose.yaml)
+# Naming files turns off Compose's own pick-up of the override file customers keep their changes in
+if [[ -f docker-compose/docker-compose.override.yaml ]]; then
+	compose+=(--file docker-compose/docker-compose.override.yaml)
+fi
+compose+=(--file .amp/docker-compose.override.yaml)
+
+# When Amp stops the service, or a step fails, stop the containers and remove these files
+cookie_jar=$(mktemp)
+portal_name=sourcegraph-admin
+trap 'rm -f "$ready_directory/ready"; "${compose[@]}" stop; rm -f "$cookie_jar" ".amp/portals/$portal_name.json"' EXIT
+trap exit TERM
+
+# Start every service, then wait until the frontend is healthy.
+# Waiting for every service fails, because searcher-0 reports unhealthy for a few seconds while it starts.
+# The first start pulls the images, and Docker Hub times out now and then.
+# --remove-orphans removes the containers of services the Compose files no longer have, such as after a rename
+"${compose[@]}" up --remove-orphans --detach || { sleep 10 && "${compose[@]}" up --remove-orphans --detach; }
+"${compose[@]}" up --remove-orphans --detach --wait sourcegraph-frontend-0
+
+# Posts JSON, keeps the session cookie, and prints Sourcegraph's reason when it refuses
+post() {
+	curl --silent --show-error --fail-with-body --cookie-jar "$cookie_jar" \
+		--header 'Content-Type: application/json' --data "$2" "$frontend_url$1"
+}
+credentials=$(jq --null-input --arg email "$admin_email" --arg username "$admin_username" \
+	--arg password "$admin_password" '{$email, $username, $password}')
+# Sign in, or create the first site admin, which also signs in. The session authorizes the GraphQL requests below
+if ! sign_in_error=$(post /-/sign-in "$credentials" 2>&1) &&
+	! site_init_error=$(post /-/site-init "$credentials" 2>&1); then
+	echo "Could not sign in as $admin_username: ${sign_in_error//$'\n'/ }" >&2
+	echo "Could not create that site admin: ${site_init_error//$'\n'/ }" >&2
+	echo "Sourcegraph creates only the first site admin, so after changing the SOURCEGRAPH_ADMIN_* variables," \
+		"change that user to match" >&2
+	exit 1
+fi
+
+# Prints the response, and fails with its errors
+graphql() {
+	curl --silent --show-error --fail-with-body --cookie "$cookie_jar" --header 'Content-Type: application/json' \
+		--data "$1" "$frontend_url/.api/graphql" |
+		jq --exit-status 'if .errors then "GraphQL errors: \(.errors)\n" | halt_error else . end'
+}
+# The site configuration is JSON with comments, so drop whole-line comments and trailing commas to read it
+site=$(graphql '{"query":"{ site { configuration { id effectiveContents } } }"}' | jq '.data.site.configuration |
+	{id, contents: (.effectiveContents | gsub("(?m)^\\s*//.*$"; "") | gsub(",(?=\\s*[]}])"; "") | fromjson? // null)}')
+# Configure the site when it has no executor token, has an older one, or has another portal URL.
+# Otherwise leave it alone, so restarts keep later changes and its comments
+if jq --exit-status '.contents == null' <<<"$site" >/dev/null; then
+	echo 'Could not parse the site configuration, so left it alone' >&2
+elif [[ "$executor_token" != "$(cat "$configured_token_file" 2>/dev/null)" ]] ||
+	jq --exit-status --arg url "$portal_url" \
+		'.contents | (has("executors.accessToken") | not) or .externalURL != $url' <<<"$site" >/dev/null; then
+	# Gives Sourcegraph its portal URL, lets the executor and the containers it starts connect,
+	# lets code intelligence jobs be queued, and sends traces to Jaeger
+	site_configuration=$(jq --arg url "$portal_url" --arg token "$executor_token" '.contents + {
+		"externalURL": $url,
+		"executors.accessToken": $token,
+		"executors.frontendURL": "http://host.docker.internal",
+		"codeIntelAutoIndexing.enabled": true,
+		"observability.tracing": {"urlTemplate": "{{ .ExternalURL }}/-/debug/jaeger/trace/{{ .TraceID }}"}
+	}' <<<"$site")
+	graphql "$(jq --arg input "$site_configuration" \
+		'{query: "mutation($id: Int!, $input: String!) { updateSiteConfiguration(lastID: $id, input: $input) }",
+		variables: {id, input: $input}}' <<<"$site")" >/dev/null
+	(umask 077 && cp "$executor_token_file" "$configured_token_file")
+fi
+
+# Shows the sign-in in the Portal tab, as a note next to the Sourcegraph link, until the service stops
+amp orb portal "$PORT" --name "$portal_name" --title Sourcegraph \
+	--description "Sign in as $admin_username with the password $admin_password"
+touch "$ready_directory/ready"
+
+# Free the disk that images of earlier versions hold, once the deployment's images change.
+# Every container of this deployment exists by now, so Docker keeps their images.
+# Not on every start, because that also removes the images of executor jobs, which then pull again
+images_file="$secrets_directory/images"
+images=$("${compose[@]}" config --images | sort)
+if [[ "$images" != "$(cat "$images_file" 2>/dev/null)" ]]; then
+	sudo docker image prune --all --force | tail -n 1
+	echo "$images" >"$images_file"
+fi
+
+# Stay in the foreground. Stopping the service stops the containers
+"${compose[@]}" up --remove-orphans

.gitignore

@@ -3,3 +3,4 @@
 .vagrant
 *.log
 node_modules/
+.amp/portals/

AGENTS.md

@@ -8,6 +8,7 @@ This repository is the deployment reference for [Sourcegraph with Docker Compose
 - `docker-compose/db-only-migrate.docker-compose.yaml`: a copy of the `pgsql` and `codeintel-db` services for backup and restore. Make every change to those two services in both files.
 - `docker-compose/examples/`: example override files. `docker-compose/executors/` and `docker-compose/jaeger/` are optional extra compose files. `docker-compose/dev/` is an overlay for developing this repository.
 - `test/` and `.buildkite/`: the smoke test and the CI pipeline.
+- `.agents/setup` and `.amp/`: run the deployment in an [Amp orb](https://ampcode.com/docs/orbs), to test changes to it. `.amp/docker-compose.override.yaml` names base services, and `.amp/Caddyfile` copies `caddy/builtins/http.Caddyfile`; keep both in step with the base files, because CI checks neither.
 - `release.yaml`: the release steps that `sg release` runs.
 
 ## Checks