diff --git a/packages/amp/src/arrow-flight/service.ts b/packages/amp/src/arrow-flight/service.ts index f13e225..230bc07 100644 --- a/packages/amp/src/arrow-flight/service.ts +++ b/packages/amp/src/arrow-flight/service.ts @@ -121,8 +121,10 @@ const make = Effect.gen(function* () { // Setup the query headers const headers = new Headers() + // `amp-stream` overrides the query's `SETTINGS stream`, so send the value + // the caller asked for: `stream: false` must run a batch query. if (Predicate.isNotUndefined(options?.stream)) { - headers.set("amp-stream", "true") + headers.set("amp-stream", options.stream ? "true" : "false") } if (Predicate.isNotUndefined(options?.resumeWatermark)) { headers.set("amp-resume", blockRangesToResumeWatermark(options.resumeWatermark)) @@ -169,6 +171,14 @@ const make = Effect.gen(function* () { return flightDataStream.pipe( Stream.mapEffect( Effect.fnUntraced(function* (flightData): Effect.fn.Return, ArrowFlightError> { + // While a streaming query waits for new blocks, Amp sends an empty + // keep-alive message (no data header, body, or app metadata) every + // `keep_alive_interval` seconds. It isn't part of the result: skip it + // instead of parsing an empty flatbuffer, which throws a RangeError. + if (flightData.dataHeader.length === 0) { + return Option.none() + } + const messageType = yield* Effect.orDie(getMessageType(flightData)) switch (messageType) { diff --git a/packages/amp/src/protocol-stream/service.ts b/packages/amp/src/protocol-stream/service.ts index 14d1827..00ad676 100644 --- a/packages/amp/src/protocol-stream/service.ts +++ b/packages/amp/src/protocol-stream/service.ts @@ -181,10 +181,17 @@ const make = Effect.gen(function* () { resumeWatermark: options?.resumeWatermark }) - const initialState: ProtocolStreamState = { - previous: [], - initialized: false - } + // When resuming, seed the state with the resume watermark. The server checks + // the cursor's hash and, if the chain reorganized while the client was away, + // re-sends the replaced blocks with a range that jumps back before the + // cursor. Comparing the first batch against the cursor is the only way to + // detect that reorg: without it, the re-sent rows arrive as plain `Data` and + // the client keeps both the orphaned and the replacement rows. + const resumeWatermark = options?.resumeWatermark + const initialState: ProtocolStreamState = + resumeWatermark !== undefined && resumeWatermark.length > 0 + ? { previous: resumeWatermark, initialized: true } + : { previous: [], initialized: false } return rawStream.pipe( // Map Arrow Flight errors to protocol errors @@ -208,15 +215,22 @@ const make = Effect.gen(function* () { // Detect reorgs const invalidations = state.initialized ? detectReorgs(state.previous, incoming) : [] - // Determine message type - let message: ProtocolMessage + // Determine the messages for this batch + const messages: Array = [] if (invalidations.length > 0) { - message = protocolReorg(state.previous, incoming, invalidations) - } else if (metadata.rangesComplete && batchData.length === 0) { - message = protocolWatermark(incoming) - } else { - message = protocolData(batchData, incoming) + messages.push(protocolReorg(state.previous, incoming, invalidations)) + } + + // The batch that reveals a reorg carries the replacement blocks' rows + // (or, for a range without rows, its completion marker). Emit them after + // the `Reorg` instead of dropping them. + if (batchData.length > 0) { + messages.push(protocolData(batchData, incoming)) + } else if (metadata.rangesComplete) { + messages.push(protocolWatermark(incoming)) + } else if (invalidations.length === 0) { + messages.push(protocolData(batchData, incoming)) } const newState: ProtocolStreamState = { @@ -224,7 +238,7 @@ const make = Effect.gen(function* () { initialized: true } - return [newState, [message]] as const + return [newState, messages] as const }) ), Stream.withSpan("ProtocolStream.stream") diff --git a/packages/amp/test/arrow-flight/service.test.ts b/packages/amp/test/arrow-flight/service.test.ts index 4437b38..e61754f 100644 --- a/packages/amp/test/arrow-flight/service.test.ts +++ b/packages/amp/test/arrow-flight/service.test.ts @@ -102,7 +102,92 @@ const makeCapturingTransport = (capturedSql: { value: string }) => }) }) +/** + * A transport that records the `amp-stream` header of each `getFlightInfo` + * call, and returns no data. + */ +const makeHeaderCapturingTransport = (capturedStreamHeaders: Array) => + createRouterTransport((router) => { + const ticket = create(TicketSchema, { ticket: encoder.encode("ticket") }) + const endpoint = create(FlightEndpointSchema, { + appMetadata: new Uint8Array(0), + location: [], + ticket + }) + const flightInfo = create(FlightInfoSchema, { + appMetadata: new Uint8Array(0), + endpoint: [endpoint], + ordered: true, + schema: new Uint8Array(0), + totalBytes: 0n, + totalRecords: 0n + }) + + router.service(FlightService, { + doGet() { + async function* messages(): AsyncGenerator {} + return messages() + }, + getFlightInfo(_req, context) { + capturedStreamHeaders.push(context.requestHeader.get("amp-stream")) + return flightInfo + } + }) + }) + +/** + * The empty message Amp sends every `keep_alive_interval` seconds while a + * streaming query waits for new blocks: no data header, body, or app metadata. + */ +const keepAlive = (): FlightData => create(FlightDataSchema, {}) + describe("ArrowFlight", () => { + it.effect("skips keep-alive messages in a stream", ({ expect }) => + Effect.gen(function* () { + const testSchema = SchemaBuilder.schema().binary("bin").build() + + const generated = yield* FlightDataGenerator.generateFlightData(testSchema, { + defaultNullRate: 0, + fields: { bin: { minLength: 3, maxLength: 3 } }, + numRows: 1, + seed: 444 + }) + + const transport = makeTransport([ + toProtoFlightData(generated.schemaFlightData, new Uint8Array(0)), + keepAlive(), + toProtoFlightData(generated.recordBatchFlightData, recordBatchMetadata), + keepAlive() + ]) + const layer = ArrowFlight.layer.pipe(Layer.provide(Layer.succeed(ArrowFlight.Transport, transport))) + + const results = yield* Effect.gen(function* () { + const flight = yield* ArrowFlight.ArrowFlight + return yield* flight.query("SELECT bin FROM test", { binaryHandling: "hex" }) + }).pipe(Effect.provide(layer)) + + expect(results).toHaveLength(1) + expect(results[0]?.data).toHaveLength(1) + }) + ) + + it.effect("sends the amp-stream header as the stream option says", ({ expect }) => + Effect.gen(function* () { + const captured: Array = [] + const transport = makeHeaderCapturingTransport(captured) + const layer = ArrowFlight.layer.pipe(Layer.provide(Layer.succeed(ArrowFlight.Transport, transport))) + + yield* Effect.gen(function* () { + const flight = yield* ArrowFlight.ArrowFlight + yield* flight.query("SELECT 1", { stream: true }) + yield* flight.query("SELECT 1", { stream: false }) + yield* flight.query("SELECT 1") + }).pipe(Effect.provide(layer)) + + expect(captured).toEqual(["true", "false", null]) + }) + ) + it.effect("explain prepends EXPLAIN to the SQL by default", ({ expect }) => Effect.gen(function* () { const captured = { value: "" } diff --git a/packages/amp/test/protocol-stream/stream.test.ts b/packages/amp/test/protocol-stream/stream.test.ts new file mode 100644 index 0000000..a69ae32 --- /dev/null +++ b/packages/amp/test/protocol-stream/stream.test.ts @@ -0,0 +1,182 @@ +/** + * ProtocolStream end-to-end tests. + * + * `reorg.test.ts` checks a copy of the reorg detection logic. These tests run + * the real `ProtocolStream` service over a mock `ArrowFlight` that replays the + * batches Amp sends, to check the messages a client actually receives. + */ +import * as ArrowFlight from "@edgeandnode/amp/arrow-flight" +import type { QueryOptions, QueryResult } from "@edgeandnode/amp/arrow-flight" +import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/core" +import { layer as protocolStreamLayer, type ProtocolMessage, ProtocolStream } from "@edgeandnode/amp/protocol-stream" +import { describe, it } from "@effect/vitest" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Stream from "effect/Stream" + +// ============================================================================= +// Helpers +// ============================================================================= + +const NETWORK = "anvil" + +/** + * A deterministic block hash. Blocks of different epochs, like the blocks of + * two forks, get different hashes. The `ab` marker keeps block 0's hash from + * being the zero hash, which the protocol reserves for "no parent". + */ +const makeHash = (block: number, epoch: number): BlockHash => + `0x${epoch.toString(16).padStart(2, "0")}ab${"0".repeat(44)}${block.toString(16).padStart(16, "0")}` as BlockHash + +/** + * A block range from `start` to `end` on the fork `epoch`, whose parent is on + * the fork `parentEpoch` (the same fork unless the range starts a new one). + */ +const range = (start: number, end: number, epoch: number, parentEpoch: number = epoch): BlockRange => ({ + network: NETWORK as Network, + numbers: { start: start as BlockNumber, end: end as BlockNumber }, + hash: makeHash(end, epoch), + prevHash: start > 0 ? makeHash(start - 1, parentEpoch) : undefined +}) + +const rows = (start: number, end: number): Array> => + Array.from({ length: end - start + 1 }, (_, i) => ({ block_num: start + i })) + +/** A batch of rows for a range, as Amp sends it before the range is complete. */ +const dataBatch = (r: BlockRange): QueryResult> => ({ + data: rows(r.numbers.start, r.numbers.end), + metadata: { ranges: [r], rangesComplete: false } +}) + +/** The empty batch that marks a range complete. */ +const completeBatch = (r: BlockRange): QueryResult> => ({ + data: [], + metadata: { ranges: [r], rangesComplete: true } +}) + +/** + * A mock ArrowFlight whose `streamQuery` replays `batches`, and records the + * options it was called with. + */ +const mockArrowFlightLayer = ( + batches: ReadonlyArray>>, + calls: Array +): Layer.Layer => + Layer.succeed(ArrowFlight.ArrowFlight, { + client: undefined as never, + query: () => Effect.die("query is not used by ProtocolStream"), + explain: () => Effect.die("explain is not used by ProtocolStream"), + streamQuery: ((_sql: string, options?: QueryOptions) => { + calls.push(options) + return Stream.fromIterable(batches) + }) as never + }) + +/** Runs a protocol stream over `batches`, and collects its messages. */ +const collect = ( + batches: ReadonlyArray>>, + resumeWatermark?: ReadonlyArray +) => { + const calls: Array = [] + return Effect.gen(function* () { + const protocolStream = yield* ProtocolStream + const messages = yield* Stream.runCollect( + protocolStream.stream("SELECT block_num FROM blocks", resumeWatermark ? { resumeWatermark } : undefined) + ) + return { messages: Array.from(messages) as Array, calls } + }).pipe(Effect.provide(protocolStreamLayer.pipe(Layer.provide(mockArrowFlightLayer(batches, calls))))) +} + +const summarize = (messages: ReadonlyArray) => + messages.map((message) => { + switch (message._tag) { + case "Data": + return `Data ${message.data.map((row) => row.block_num).join(",")}` + case "Watermark": + return `Watermark ${message.ranges.map((r) => `${r.numbers.start}-${r.numbers.end}`).join(",")}` + case "Reorg": + return `Reorg ${message.invalidation.map((r) => `${r.start}-${r.end}`).join(",")}` + } + }) + +// ============================================================================= +// Reorgs within a stream +// ============================================================================= + +describe("ProtocolStream reorgs", () => { + it.effect("emits the replacement rows that arrive with a reorg", ({ expect }) => + Effect.gen(function* () { + // Blocks 0-2, then a reorg replaces 1-2 and adds 3: the new range starts + // at block 1, and its parent is block 0, the last block both forks share. + const original = range(0, 2, 0) + const replacement = range(1, 3, 1, 0) + + const { messages } = yield* collect([ + dataBatch(original), + completeBatch(original), + dataBatch(replacement), + completeBatch(replacement) + ]) + + expect(summarize(messages)).toEqual(["Data 0,1,2", "Watermark 0-2", "Reorg 1-3", "Data 1,2,3", "Watermark 1-3"]) + }) + ) + + it.effect("emits the watermark of a reorg range that has no rows", ({ expect }) => + Effect.gen(function* () { + const original = range(0, 2, 0) + const replacement = range(1, 3, 1, 0) + + const { messages } = yield* collect([dataBatch(original), completeBatch(original), completeBatch(replacement)]) + + expect(summarize(messages)).toEqual(["Data 0,1,2", "Watermark 0-2", "Reorg 1-3", "Watermark 1-3"]) + }) + ) + + it.effect("emits a plain data message when the chain continues", ({ expect }) => + Effect.gen(function* () { + const first = range(0, 2, 0) + const next = range(3, 4, 0) + + const { messages } = yield* collect([ + dataBatch(first), + completeBatch(first), + dataBatch(next), + completeBatch(next) + ]) + + expect(summarize(messages)).toEqual(["Data 0,1,2", "Watermark 0-2", "Data 3,4", "Watermark 3-4"]) + }) + ) +}) + +// ============================================================================= +// Reorgs while the stream was stopped +// ============================================================================= + +describe("ProtocolStream resume", () => { + it.effect("detects a reorg that happened while the stream was stopped", ({ expect }) => + Effect.gen(function* () { + // The client saved its cursor at block 2. Meanwhile, a reorg replaced + // blocks 1-2, so Amp re-sends from block 1 on the new fork. + const cursor = range(0, 2, 0) + const replacement = range(1, 3, 1, 0) + + const { calls, messages } = yield* collect([dataBatch(replacement), completeBatch(replacement)], [cursor]) + + expect(calls[0]?.resumeWatermark).toEqual([cursor]) + expect(summarize(messages)).toEqual(["Reorg 1-3", "Data 1,2,3", "Watermark 1-3"]) + }) + ) + + it.effect("resumes without a reorg when the chain continued", ({ expect }) => + Effect.gen(function* () { + const cursor = range(0, 2, 0) + const next = range(3, 4, 0) + + const { messages } = yield* collect([dataBatch(next), completeBatch(next)], [cursor]) + + expect(summarize(messages)).toEqual(["Data 3,4", "Watermark 3-4"]) + }) + ) +})