Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion packages/amp/src/arrow-flight/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,10 @@ const make = Effect.gen(function* () {

// Setup the query headers
const headers = new Headers()
// `amp-stream` overrides the query's `SETTINGS stream`, so send the value
// the caller asked for: `stream: false` must run a batch query.
if (Predicate.isNotUndefined(options?.stream)) {
headers.set("amp-stream", "true")
headers.set("amp-stream", options.stream ? "true" : "false")
}
if (Predicate.isNotUndefined(options?.resumeWatermark)) {
headers.set("amp-resume", blockRangesToResumeWatermark(options.resumeWatermark))
Expand Down Expand Up @@ -169,6 +171,14 @@ const make = Effect.gen(function* () {
return flightDataStream.pipe(
Stream.mapEffect(
Effect.fnUntraced(function* (flightData): Effect.fn.Return<Option.Option<Result>, ArrowFlightError> {
// While a streaming query waits for new blocks, Amp sends an empty
// keep-alive message (no data header, body, or app metadata) every
// `keep_alive_interval` seconds. It isn't part of the result: skip it
// instead of parsing an empty flatbuffer, which throws a RangeError.
if (flightData.dataHeader.length === 0) {
return Option.none<Result>()
}

const messageType = yield* Effect.orDie(getMessageType(flightData))

switch (messageType) {
Expand Down
38 changes: 26 additions & 12 deletions packages/amp/src/protocol-stream/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,10 +181,17 @@ const make = Effect.gen(function* () {
resumeWatermark: options?.resumeWatermark
})

const initialState: ProtocolStreamState = {
previous: [],
initialized: false
}
// When resuming, seed the state with the resume watermark. The server checks
// the cursor's hash and, if the chain reorganized while the client was away,
// re-sends the replaced blocks with a range that jumps back before the
// cursor. Comparing the first batch against the cursor is the only way to
// detect that reorg: without it, the re-sent rows arrive as plain `Data` and
// the client keeps both the orphaned and the replacement rows.
const resumeWatermark = options?.resumeWatermark
const initialState: ProtocolStreamState =
resumeWatermark !== undefined && resumeWatermark.length > 0
? { previous: resumeWatermark, initialized: true }
: { previous: [], initialized: false }

return rawStream.pipe(
// Map Arrow Flight errors to protocol errors
Expand All @@ -208,23 +215,30 @@ const make = Effect.gen(function* () {
// Detect reorgs
const invalidations = state.initialized ? detectReorgs(state.previous, incoming) : []

// Determine message type
let message: ProtocolMessage
// Determine the messages for this batch
const messages: Array<ProtocolMessage> = []

if (invalidations.length > 0) {
message = protocolReorg(state.previous, incoming, invalidations)
} else if (metadata.rangesComplete && batchData.length === 0) {
message = protocolWatermark(incoming)
} else {
message = protocolData(batchData, incoming)
messages.push(protocolReorg(state.previous, incoming, invalidations))
}

// The batch that reveals a reorg carries the replacement blocks' rows
// (or, for a range without rows, its completion marker). Emit them after
// the `Reorg` instead of dropping them.
if (batchData.length > 0) {
messages.push(protocolData(batchData, incoming))
} else if (metadata.rangesComplete) {
messages.push(protocolWatermark(incoming))
} else if (invalidations.length === 0) {
messages.push(protocolData(batchData, incoming))
}

const newState: ProtocolStreamState = {
previous: incoming,
initialized: true
}

return [newState, [message]] as const
return [newState, messages] as const
})
),
Stream.withSpan("ProtocolStream.stream")
Expand Down
85 changes: 85 additions & 0 deletions packages/amp/test/arrow-flight/service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,92 @@ const makeCapturingTransport = (capturedSql: { value: string }) =>
})
})

/**
* A transport that records the `amp-stream` header of each `getFlightInfo`
* call, and returns no data.
*/
const makeHeaderCapturingTransport = (capturedStreamHeaders: Array<string | null>) =>
createRouterTransport((router) => {
const ticket = create(TicketSchema, { ticket: encoder.encode("ticket") })
const endpoint = create(FlightEndpointSchema, {
appMetadata: new Uint8Array(0),
location: [],
ticket
})
const flightInfo = create(FlightInfoSchema, {
appMetadata: new Uint8Array(0),
endpoint: [endpoint],
ordered: true,
schema: new Uint8Array(0),
totalBytes: 0n,
totalRecords: 0n
})

router.service(FlightService, {
doGet() {
async function* messages(): AsyncGenerator<FlightData> {}
return messages()
},
getFlightInfo(_req, context) {
capturedStreamHeaders.push(context.requestHeader.get("amp-stream"))
return flightInfo
}
})
})

/**
* The empty message Amp sends every `keep_alive_interval` seconds while a
* streaming query waits for new blocks: no data header, body, or app metadata.
*/
const keepAlive = (): FlightData => create(FlightDataSchema, {})

describe("ArrowFlight", () => {
it.effect("skips keep-alive messages in a stream", ({ expect }) =>
Effect.gen(function* () {
const testSchema = SchemaBuilder.schema().binary("bin").build()

const generated = yield* FlightDataGenerator.generateFlightData(testSchema, {
defaultNullRate: 0,
fields: { bin: { minLength: 3, maxLength: 3 } },
numRows: 1,
seed: 444
})

const transport = makeTransport([
toProtoFlightData(generated.schemaFlightData, new Uint8Array(0)),
keepAlive(),
toProtoFlightData(generated.recordBatchFlightData, recordBatchMetadata),
keepAlive()
])
const layer = ArrowFlight.layer.pipe(Layer.provide(Layer.succeed(ArrowFlight.Transport, transport)))

const results = yield* Effect.gen(function* () {
const flight = yield* ArrowFlight.ArrowFlight
return yield* flight.query("SELECT bin FROM test", { binaryHandling: "hex" })
}).pipe(Effect.provide(layer))

expect(results).toHaveLength(1)
expect(results[0]?.data).toHaveLength(1)
})
)

it.effect("sends the amp-stream header as the stream option says", ({ expect }) =>
Effect.gen(function* () {
const captured: Array<string | null> = []
const transport = makeHeaderCapturingTransport(captured)
const layer = ArrowFlight.layer.pipe(Layer.provide(Layer.succeed(ArrowFlight.Transport, transport)))

yield* Effect.gen(function* () {
const flight = yield* ArrowFlight.ArrowFlight
yield* flight.query("SELECT 1", { stream: true })
yield* flight.query("SELECT 1", { stream: false })
yield* flight.query("SELECT 1")
}).pipe(Effect.provide(layer))

expect(captured).toEqual(["true", "false", null])
})
)

it.effect("explain prepends EXPLAIN to the SQL by default", ({ expect }) =>
Effect.gen(function* () {
const captured = { value: "" }
Expand Down
182 changes: 182 additions & 0 deletions packages/amp/test/protocol-stream/stream.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
/**
* ProtocolStream end-to-end tests.
*
* `reorg.test.ts` checks a copy of the reorg detection logic. These tests run
* the real `ProtocolStream` service over a mock `ArrowFlight` that replays the
* batches Amp sends, to check the messages a client actually receives.
*/
import * as ArrowFlight from "@edgeandnode/amp/arrow-flight"
import type { QueryOptions, QueryResult } from "@edgeandnode/amp/arrow-flight"
import type { BlockHash, BlockNumber, BlockRange, Network } from "@edgeandnode/amp/core"
import { layer as protocolStreamLayer, type ProtocolMessage, ProtocolStream } from "@edgeandnode/amp/protocol-stream"
import { describe, it } from "@effect/vitest"
import * as Effect from "effect/Effect"
import * as Layer from "effect/Layer"
import * as Stream from "effect/Stream"

// =============================================================================
// Helpers
// =============================================================================

const NETWORK = "anvil"

/**
* A deterministic block hash. Blocks of different epochs, like the blocks of
* two forks, get different hashes. The `ab` marker keeps block 0's hash from
* being the zero hash, which the protocol reserves for "no parent".
*/
const makeHash = (block: number, epoch: number): BlockHash =>
`0x${epoch.toString(16).padStart(2, "0")}ab${"0".repeat(44)}${block.toString(16).padStart(16, "0")}` as BlockHash

/**
* A block range from `start` to `end` on the fork `epoch`, whose parent is on
* the fork `parentEpoch` (the same fork unless the range starts a new one).
*/
const range = (start: number, end: number, epoch: number, parentEpoch: number = epoch): BlockRange => ({
network: NETWORK as Network,
numbers: { start: start as BlockNumber, end: end as BlockNumber },
hash: makeHash(end, epoch),
prevHash: start > 0 ? makeHash(start - 1, parentEpoch) : undefined
})

const rows = (start: number, end: number): Array<Record<string, unknown>> =>
Array.from({ length: end - start + 1 }, (_, i) => ({ block_num: start + i }))

/** A batch of rows for a range, as Amp sends it before the range is complete. */
const dataBatch = (r: BlockRange): QueryResult<Record<string, unknown>> => ({
data: rows(r.numbers.start, r.numbers.end),
metadata: { ranges: [r], rangesComplete: false }
})

/** The empty batch that marks a range complete. */
const completeBatch = (r: BlockRange): QueryResult<Record<string, unknown>> => ({
data: [],
metadata: { ranges: [r], rangesComplete: true }
})

/**
* A mock ArrowFlight whose `streamQuery` replays `batches`, and records the
* options it was called with.
*/
const mockArrowFlightLayer = (
batches: ReadonlyArray<QueryResult<Record<string, unknown>>>,
calls: Array<QueryOptions | undefined>
): Layer.Layer<ArrowFlight.ArrowFlight> =>
Layer.succeed(ArrowFlight.ArrowFlight, {
client: undefined as never,
query: () => Effect.die("query is not used by ProtocolStream"),
explain: () => Effect.die("explain is not used by ProtocolStream"),
streamQuery: ((_sql: string, options?: QueryOptions) => {
calls.push(options)
return Stream.fromIterable(batches)
}) as never
})

/** Runs a protocol stream over `batches`, and collects its messages. */
const collect = (
batches: ReadonlyArray<QueryResult<Record<string, unknown>>>,
resumeWatermark?: ReadonlyArray<BlockRange>
) => {
const calls: Array<QueryOptions | undefined> = []
return Effect.gen(function* () {
const protocolStream = yield* ProtocolStream
const messages = yield* Stream.runCollect(
protocolStream.stream("SELECT block_num FROM blocks", resumeWatermark ? { resumeWatermark } : undefined)
)
return { messages: Array.from(messages) as Array<ProtocolMessage>, calls }
}).pipe(Effect.provide(protocolStreamLayer.pipe(Layer.provide(mockArrowFlightLayer(batches, calls)))))
}

const summarize = (messages: ReadonlyArray<ProtocolMessage>) =>
messages.map((message) => {
switch (message._tag) {
case "Data":
return `Data ${message.data.map((row) => row.block_num).join(",")}`
case "Watermark":
return `Watermark ${message.ranges.map((r) => `${r.numbers.start}-${r.numbers.end}`).join(",")}`
case "Reorg":
return `Reorg ${message.invalidation.map((r) => `${r.start}-${r.end}`).join(",")}`
}
})

// =============================================================================
// Reorgs within a stream
// =============================================================================

describe("ProtocolStream reorgs", () => {
it.effect("emits the replacement rows that arrive with a reorg", ({ expect }) =>
Effect.gen(function* () {
// Blocks 0-2, then a reorg replaces 1-2 and adds 3: the new range starts
// at block 1, and its parent is block 0, the last block both forks share.
const original = range(0, 2, 0)
const replacement = range(1, 3, 1, 0)

const { messages } = yield* collect([
dataBatch(original),
completeBatch(original),
dataBatch(replacement),
completeBatch(replacement)
])

expect(summarize(messages)).toEqual(["Data 0,1,2", "Watermark 0-2", "Reorg 1-3", "Data 1,2,3", "Watermark 1-3"])
})
)

it.effect("emits the watermark of a reorg range that has no rows", ({ expect }) =>
Effect.gen(function* () {
const original = range(0, 2, 0)
const replacement = range(1, 3, 1, 0)

const { messages } = yield* collect([dataBatch(original), completeBatch(original), completeBatch(replacement)])

expect(summarize(messages)).toEqual(["Data 0,1,2", "Watermark 0-2", "Reorg 1-3", "Watermark 1-3"])
})
)

it.effect("emits a plain data message when the chain continues", ({ expect }) =>
Effect.gen(function* () {
const first = range(0, 2, 0)
const next = range(3, 4, 0)

const { messages } = yield* collect([
dataBatch(first),
completeBatch(first),
dataBatch(next),
completeBatch(next)
])

expect(summarize(messages)).toEqual(["Data 0,1,2", "Watermark 0-2", "Data 3,4", "Watermark 3-4"])
})
)
})

// =============================================================================
// Reorgs while the stream was stopped
// =============================================================================

describe("ProtocolStream resume", () => {
it.effect("detects a reorg that happened while the stream was stopped", ({ expect }) =>
Effect.gen(function* () {
// The client saved its cursor at block 2. Meanwhile, a reorg replaced
// blocks 1-2, so Amp re-sends from block 1 on the new fork.
const cursor = range(0, 2, 0)
const replacement = range(1, 3, 1, 0)

const { calls, messages } = yield* collect([dataBatch(replacement), completeBatch(replacement)], [cursor])

expect(calls[0]?.resumeWatermark).toEqual([cursor])
expect(summarize(messages)).toEqual(["Reorg 1-3", "Data 1,2,3", "Watermark 1-3"])
})
)

it.effect("resumes without a reorg when the chain continued", ({ expect }) =>
Effect.gen(function* () {
const cursor = range(0, 2, 0)
const next = range(3, 4, 0)

const { messages } = yield* collect([dataBatch(next), completeBatch(next)], [cursor])

expect(summarize(messages)).toEqual(["Data 3,4", "Watermark 3-4"])
})
)
})