From f3c03e9bb545d0762432d38c09a79a97fd4c5d44 Mon Sep 17 00:00:00 2001 From: Matt Johnston Date: Wed, 23 Sep 2026 07:58:22 -0300 Subject: [PATCH] feat(cli): wait for the rollout in compute push MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `push` stopped as soon as the image landed, which on a bare-image redeploy or a skipped build is true the moment the deploy is accepted — so it printed "Deployed" while every instance still served the previous code. It now polls until the declared instances are all ready, current and not stale. The report gains the live instance tally and how long the wait took, split at the image landing, so a deploy that felt slow says where the time went. --- .../experimental/compute/compute.format.ts | 20 ++ .../compute/compute.format.unit.test.ts | 23 ++- .../experimental/compute/push/SIDE_EFFECTS.md | 38 ++-- .../experimental/compute/push/push.command.ts | 10 +- .../experimental/compute/push/push.handler.ts | 104 +++++++--- .../compute/push/push.integration.test.ts | 178 ++++++++++++++++-- apps/cli/src/shared/compute/compute-api.ts | 105 +++++++++-- .../shared/compute/compute-api.unit.test.ts | 50 +++++ apps/cli/src/shared/compute/compute.errors.ts | 14 ++ 9 files changed, 471 insertions(+), 71 deletions(-) create mode 100644 apps/cli/src/shared/compute/compute-api.unit.test.ts diff --git a/apps/cli/src/commands/experimental/compute/compute.format.ts b/apps/cli/src/commands/experimental/compute/compute.format.ts index 56e9e8002e..e46aecac64 100644 --- a/apps/cli/src/commands/experimental/compute/compute.format.ts +++ b/apps/cli/src/commands/experimental/compute/compute.format.ts @@ -22,3 +22,23 @@ export function renderComputeDetails(rows: ReadonlyArray label.length)); return `${present.map(([label, value]) => ` ${label.padEnd(width)} ${value}`).join("\n")}\n`; } + +/** + * A wait, for the deploy summary: `48s`, `3m21s`, `1h04m`. + * + * Whole seconds throughout — these measure a server-side build and rollout, where sub-second + * precision would imply the CLI polled far more tightly than it does. + */ +export function formatWaited(millis: number): string { + const total = Math.max(0, Math.round(millis / 1000)); + const seconds = total % 60; + const minutes = Math.floor(total / 60) % 60; + const hours = Math.floor(total / 3600); + if (hours > 0) { + return `${hours}h${String(minutes).padStart(2, "0")}m`; + } + if (minutes > 0) { + return `${minutes}m${String(seconds).padStart(2, "0")}s`; + } + return `${seconds}s`; +} diff --git a/apps/cli/src/commands/experimental/compute/compute.format.unit.test.ts b/apps/cli/src/commands/experimental/compute/compute.format.unit.test.ts index 79aabfe23a..c3b213d8e3 100644 --- a/apps/cli/src/commands/experimental/compute/compute.format.unit.test.ts +++ b/apps/cli/src/commands/experimental/compute/compute.format.unit.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { renderComputeDetails } from "./compute.format.ts"; +import { formatWaited, renderComputeDetails } from "./compute.format.ts"; describe("renderComputeDetails", () => { it("pads every label to the widest one", () => { @@ -32,3 +32,24 @@ describe("renderComputeDetails", () => { ).toBe(""); }); }); + +describe("formatWaited", () => { + it("reports a sub-minute wait in seconds", () => { + expect(formatWaited(48_000)).toBe("48s"); + }); + + it("pads the seconds so minute-scale waits align in a column", () => { + expect(formatWaited(201_000)).toBe("3m21s"); + expect(formatWaited(184_000)).toBe("3m04s"); + }); + + it("drops to minutes past an hour, where the seconds are noise", () => { + expect(formatWaited(3_840_000)).toBe("1h04m"); + }); + + // Rounds rather than truncates, so a poll that lands at 1999ms is not reported as one second. + it("rounds to whole seconds", () => { + expect(formatWaited(1_600)).toBe("2s"); + expect(formatWaited(0)).toBe("0s"); + }); +}); diff --git a/apps/cli/src/commands/experimental/compute/push/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/compute/push/SIDE_EFFECTS.md index 1201d594f9..eb6dc513e5 100644 --- a/apps/cli/src/commands/experimental/compute/push/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/compute/push/SIDE_EFFECTS.md @@ -33,18 +33,23 @@ and the command handler does not run. See the [Compute command guide](../../../. ## API Routes -| Method | Path | Auth | Request body | Response (used fields) | -| ------ | -------------------------------------------- | ------------------------------------------- | --------------------------------------------------- | ------------------------------------------------------ | -| `POST` | `/v2/projects/{ref}/compute/{name}/uploads` | Bearer token | none | `data.id`, `data.attributes.url/method` | -| `PUT` | presigned upload URL (control-plane storage) | URL signature — **no** Supabase credentials | `.tar.gz` build context | status only | -| `POST` | `/v2/projects/{ref}/compute/{name}/deploy` | Bearer token | `{data:{type,attributes:{spec,context_upload_id}}}` | `data.attributes.build_state` | -| `GET` | `/v2/projects/{ref}/compute/{name}` | Bearer token | none | `build_state`, `state_reason`, `image_version`, `spec` | -| `GET` | `/v1/projects/{ref}` | Bearer token | none | linked-project cache miss only — name, org, region | - -`GET /v2/projects/{ref}/compute/{name}` is polled until `build_state` leaves -`building`. It is skipped entirely in two cases: under `--no-wait`, and when -the deploy response already carried a terminal `build_state`. Either way the -run reports the accepted spec the deploy response returned. +| Method | Path | Auth | Request body | Response (used fields) | +| ------ | -------------------------------------------- | ------------------------------------------- | --------------------------------------------------- | ------------------------------------------------------------------- | +| `POST` | `/v2/projects/{ref}/compute/{name}/uploads` | Bearer token | none | `data.id`, `data.attributes.url/method` | +| `PUT` | presigned upload URL (control-plane storage) | URL signature — **no** Supabase credentials | `.tar.gz` build context | status only | +| `POST` | `/v2/projects/{ref}/compute/{name}/deploy` | Bearer token | `{data:{type,attributes:{spec,context_upload_id}}}` | `data.attributes.build_state` | +| `GET` | `/v2/projects/{ref}/compute/{name}` | Bearer token | none | `build_state`, `state_reason`, `image_version`, `spec`, `instances` | +| `GET` | `/v1/projects/{ref}` | Bearer token | none | linked-project cache miss only — name, org, region | + +`GET /v2/projects/{ref}/compute/{name}` is polled until the deploy's code is +serving: `build_state` has left `building` and the instance tally reports every +declared instance ready, current and not stale. The poll starts at 2s and backs +off geometrically to 15s, bounded at 22 minutes. + +It is skipped in two cases: under `--no-wait`, and when the deploy response +already carried `build_state: failed`. An `active` deploy response is **not** +one of them — that is the bare-image case, where nothing was built and the +instances keep serving the previous code until the control plane recycles them. ## Exit Codes @@ -59,6 +64,7 @@ run reports the accepted spec the deploy response returned. | `1` | a compute's source links to a path outside itself | | `1` | build context upload failed | | `1` | the build reached `failed`, or never left `building` | +| `1` | the build landed but the instances never all reached serving inside the poll budget | | `1` | with `--no-wait`: the deploy was answered with `build_state: failed` | | `1` | API error, or project not enrolled in the alpha | @@ -125,6 +131,14 @@ while `build_state` is `building`. The deploy response may carry an `image_version` — a re-push of a compute that is already serving echoes the image it is serving now — and that is the previous build's, not this one's. +The deploy report carries an `Instances` row (the live tally, `2/2 serving`, +falling back to the declared count when the API reports none) and a `Waited` +row (`3m21s (1m04s building)`). Machine payloads carry the same as +`instances_ready`, `instances_stale`, `waited_ms` and `waited_build_ms`. The +build split is measured from this run's own polls, so it is "when the CLI first +observed the image had landed" — it includes the control plane's detection lag +rather than isolating build time, which the API publishes no stamp for. + The presigned `PUT` above is the one request whose URL is itself a credential. `--debug` logs every request URL, so `httpClientLayer` redacts query strings that carry a signature. diff --git a/apps/cli/src/commands/experimental/compute/push/push.command.ts b/apps/cli/src/commands/experimental/compute/push/push.command.ts index fd56fc69ab..a762853d42 100644 --- a/apps/cli/src/commands/experimental/compute/push/push.command.ts +++ b/apps/cli/src/commands/experimental/compute/push/push.command.ts @@ -35,12 +35,12 @@ const config = { Flag.optional, ), noWait: Flag.boolean("no-wait").pipe( - // The deploy POST returns once the platform accepts the spec and context; the - // server-side build that follows can run for minutes. Waiting stays the - // default so a plain push reports the build's verdict; `--no-wait` opts out - // for callers that only need the deploy accepted. + // The deploy POST returns once the platform accepts the spec and context; the build and the + // rollout that follows it can run for minutes between them. Waiting stays the default so a + // plain push reports what is actually serving; `--no-wait` opts out for callers that only + // need the deploy accepted. Flag.withDescription( - "Return once the deploy is accepted, without waiting for the server-side build to finish.", + "Return once the deploy is accepted, without waiting for the new code to roll out.", ), Flag.withDefault(false), ), diff --git a/apps/cli/src/commands/experimental/compute/push/push.handler.ts b/apps/cli/src/commands/experimental/compute/push/push.handler.ts index 3f297e0e6e..50aa71027a 100644 --- a/apps/cli/src/commands/experimental/compute/push/push.handler.ts +++ b/apps/cli/src/commands/experimental/compute/push/push.handler.ts @@ -1,8 +1,8 @@ -import { Effect, FileSystem, Option, Path, Predicate, type Schedule } from "effect"; +import { Clock, Effect, FileSystem, Option, Path, Predicate, type Schedule } from "effect"; import type { PlatformError } from "effect/PlatformError"; import { Output } from "../../../../shared/output/output.service.ts"; import { emitSuccessTrailer } from "../../../../shared/cli/success-trailer.ts"; -import { renderComputeDetails } from "../compute.format.ts"; +import { formatWaited, renderComputeDetails } from "../compute.format.ts"; import { emitComputeMachineOutput, rejectComputeEnvOutput, @@ -35,11 +35,13 @@ import { } from "../../../../shared/compute/compute-runtimes.ts"; import { computeUrl } from "../../../../shared/compute/compute-url.ts"; import { - awaitComputeBuild, + awaitComputeServing, createComputeUpload, deployCompute, + isComputeServing, uploadBuildContext, type ComputeDeploySpec, + type ComputeRecord, } from "../../../../shared/compute/compute-api.ts"; import { NoComputeToDeployError, @@ -66,8 +68,9 @@ import type { ComputePushFlags } from "./push.command.ts"; * `supabase compute push [name...]` — deploys the named compute * (or every compute, if none are named), reading runtime, size, exposure, and * source from `[compute.]`; an unrecorded runtime is guessed and - * reported. Builds run server-side from an uploaded context and are waited on - * by default; `--no-wait` returns once the deploy is accepted. + * reported. Builds run server-side from an uploaded context, and a plain push waits for the + * deploy's code to actually be serving — the build landing is not the end of it; `--no-wait` + * returns once the deploy is accepted. */ const resolveRuntime = Effect.fnUntraced(function* (options: { @@ -216,6 +219,29 @@ function addYourCode(sourceDisplay: string): string { return `Add your compute's code to ${sourceDisplay}, then run this command again.`; } +/** + * The instance row, and the progress line while waiting on a rollout. + * + * Falls back to the declared count alone whenever the tally is absent — it is omitted until the + * first image lands, and replaced by `instances_error` when the control plane's read-through + * fails, neither of which says anything about how many instances are up. + */ +function describeTally(compute: ComputeRecord): string { + const { instances } = compute; + if (instances === undefined) { + return `${compute.spec.instances} declared`; + } + const stale = instances.stale > 0 ? `, ${instances.stale} stale` : ""; + return `${instances.ready}/${instances.declared} serving${stale}`; +} + +/** `3m21s (1m04s building)`, or just the total when nothing was built. */ +function describeWaited(waited: { readonly total: number; readonly build: number }): string { + const total = formatWaited(waited.total); + // Under a second is the poll cadence, not a build worth attributing. + return waited.build < 1000 ? total : `${total} (${formatWaited(waited.build)} building)`; +} + const deployOneCompute = Effect.fnUntraced(function* (input: { readonly project: ComputeProject; readonly name: string; @@ -370,25 +396,46 @@ const deployOneCompute = Effect.fnUntraced(function* (input: { Effect.tapError(() => deploying.fail()), ); - // Polled only when the deploy response left the build unresolved. - // `V2DeployAWorkerOutput` permits a terminal `active` or `failed` on the - // deploy itself, and that verdict is this deploy's — a fresh `GET` can only - // contradict it: `awaitComputeBuild` reads a post-deploy 404 as "still - // building", so an already-`failed` deploy could burn the whole poll budget - // and surface as a timeout, and a concurrent deployment could answer with a - // state that belongs to someone else's build. - const settled = - input.noWait || accepted.buildState !== "building" - ? accepted - : yield* awaitComputeBuild(api, projectRef, name, { - schedule: input.pollSchedule, - retrySchedule: input.pollRetrySchedule, - refSuffix: input.refSuffix, - onPoll: (polled) => - polled.buildState === "building" - ? deploying.message("Building compute...") - : Effect.void, - }).pipe(Effect.tapError(() => deploying.fail())); + // A terminal `active` on the deploy response is no longer a reason to skip the poll. It is + // exactly the case where nothing was built — a bare-image redeploy, or a build skipped as + // unchanged — and the instances go on serving the previous code until the control plane + // recycles them. Stopping there is what let `push` report "Deployed" over a rollout that had + // not started. A `failed` response still short-circuits: that verdict is this deploy's, and a + // fresh `GET` can only contradict it — `awaitComputeServing` reads a post-deploy 404 as "not + // settled yet", so an already-failed deploy would burn the whole poll budget and surface as a + // timeout, and a concurrent deployment could answer with a state belonging to someone else's + // build. + const shortCircuit = input.noWait || accepted.buildState === "failed"; + + // Transitions as this deploy's own polls saw them, at the poll cadence's resolution. This is + // the only timing available to the CLI: the API publishes no per-stage stamps, so the build + // mark is "when we first observed the image landed", which includes the control plane's own + // detection lag rather than the build alone. + const startedAt = yield* Clock.currentTimeMillis; + let builtAt: number | undefined = accepted.buildState === "building" ? undefined : startedAt; + + const settled = shortCircuit + ? accepted + : yield* awaitComputeServing(api, projectRef, name, { + schedule: input.pollSchedule, + retrySchedule: input.pollRetrySchedule, + refSuffix: input.refSuffix, + onPoll: (polled) => + Effect.gen(function* () { + if (polled.buildState === "building") { + return yield* deploying.message("Building compute..."); + } + builtAt ??= yield* Clock.currentTimeMillis; + if (!isComputeServing(polled)) { + yield* deploying.message(`Starting instances... ${describeTally(polled)}`); + } + }), + }).pipe(Effect.tapError(() => deploying.fail())); + + const finishedAt = yield* Clock.currentTimeMillis; + const waited = shortCircuit + ? undefined + : { total: finishedAt - startedAt, build: (builtAt ?? finishedAt) - startedAt }; // Checked regardless of whether the build was waited on: the verdict can // arrive on the deploy response as readily as on a poll. @@ -435,6 +482,11 @@ const deployOneCompute = Effect.fnUntraced(function* (input: { // empty-valued row. ["Image", imageVersion ?? ""], ["Access", settled.spec.exposure], + ["Instances", describeTally(settled)], + // How long the wait actually took, and how much of it was over before the image + // landed. The API publishes no per-stage timing, so this is the only attribution a + // caller gets for a deploy that felt slow. + ["Waited", waited === undefined ? "" : describeWaited(waited)], ["URL", url ?? ""], ]), ); @@ -461,6 +513,10 @@ const deployOneCompute = Effect.fnUntraced(function* (input: { // deploy completed. Same reason `url` is spread below. ...(imageVersion === undefined ? {} : { image_version: imageVersion }), build_state: settled.buildState, + ...(settled.instances === undefined ? {} : { instances_ready: settled.instances.ready }), + ...(settled.instances === undefined ? {} : { instances_stale: settled.instances.stale }), + // Flat scalars rather than a nested object: `-o env` and `-o toml` both take this payload. + ...(waited === undefined ? {} : { waited_ms: waited.total, waited_build_ms: waited.build }), ...(url === undefined ? {} : { url }), }; }); diff --git a/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts b/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts index 9cc6252af7..49244d5580 100644 --- a/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts +++ b/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts @@ -19,6 +19,7 @@ import { ComputeBuildFailedError, ComputeBuildTimeoutError, ComputeProjectNotFoundError, + ComputeRolloutTimeoutError, ComputeRouteNotFoundError, ComputeUnavailableError, ComputeSourceEscapingLinkError, @@ -79,6 +80,20 @@ function project(files: Readonly> = {}) { }); } +/** + * A compute that has finished rolling out, for the `GET` a poll reads. + * + * `push` waits for the instance tally to converge, so a fake standing in for a healthy deploy + * has to carry one: an absent tally is deliberately not convergence, since the field is omitted + * until the first image lands. + */ +function rolledOut(options: Parameters[0]) { + return computeResource({ + instanceCounts: { declared: 1, live: 1, ready: 1, stale: 0 }, + ...options, + }); +} + function routes(overrides: ComputeHttpRoutes = {}): ComputeHttpRoutes { return { [`POST ${computeRoute("/api/uploads")}`]: { status: 201, body: uploadSlot }, @@ -90,7 +105,7 @@ function routes(overrides: ComputeHttpRoutes = {}): ComputeHttpRoutes { [`GET ${computeRoute("/api")}`]: { status: 200, body: { - data: computeResource({ + data: rolledOut({ name: "api", runtime: "node", buildState: "active", @@ -186,12 +201,132 @@ describe("compute push", () => { }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); - // `V2DeployAWorkerOutput` permits a terminal state on the deploy response - // itself, and that verdict is this deploy's. A poll on top of it can only - // contradict it — `awaitComputeBuild` reads a post-deploy 404 as "still - // building", so an already-settled deploy would burn the poll budget and - // surface as a timeout instead of the answer the platform already gave. - describe("honours a terminal deploy response instead of polling", () => { + describe("waiting for the rollout", () => { + const tally = (counts: { declared: number; live: number; ready: number; stale: number }) => ({ + status: 200, + body: { + data: computeResource({ + name: "api", + runtime: "node", + buildState: "active" as const, + imageVersion: "v1", + instanceCounts: counts, + }), + }, + }); + + it.live("keeps polling until every declared instance is serving the new code", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer, out } = setupCompute({ + workdir: repo.dir, + routes: routes({ + [`GET ${computeRoute("/api")}`]: [ + tally({ declared: 2, live: 0, ready: 0, stale: 0 }), + tally({ declared: 2, live: 2, ready: 1, stale: 0 }), + tally({ declared: 2, live: 2, ready: 2, stale: 0 }), + ], + }), + }); + + return yield* Effect.gen(function* () { + yield* push({ instances: Option.some(2) }); + + expect(out.stdoutText).toContain("Deployed Compute api"); + // The live tally, not the declared count: the whole point of waiting. + expect(out.stdoutText).toContain("2/2 serving"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // A redeploy that produces no new image version leaves the rotation cutoff as the only + // thing marking instances still on the old code, so `stale` is what keeps the wait open. + it.live("keeps waiting while an instance is still on the previous code", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer, out } = setupCompute({ + workdir: repo.dir, + routes: routes({ + [`GET ${computeRoute("/api")}`]: [ + tally({ declared: 1, live: 1, ready: 1, stale: 1 }), + tally({ declared: 1, live: 1, ready: 1, stale: 0 }), + ], + }), + }); + + return yield* Effect.gen(function* () { + yield* push(); + + expect(out.stdoutText).toContain("1/1 serving"); + expect(out.stdoutText).not.toContain("stale"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // `instances_error` replaces the tally when the control plane's read-through fails. It says + // nothing about how many instances are up, so it must not be read as convergence. + it.live("keeps waiting when the instance read-through fails", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer, out } = setupCompute({ + workdir: repo.dir, + routes: routes({ + [`GET ${computeRoute("/api")}`]: [ + { + status: 200, + body: { + data: computeResource({ + name: "api", + runtime: "node", + buildState: "active", + imageVersion: "v1", + instancesError: "instances are temporarily unavailable", + }), + }, + }, + tally({ declared: 1, live: 1, ready: 1, stale: 0 }), + ], + }), + }); + + return yield* Effect.gen(function* () { + yield* push(); + expect(out.stdoutText).toContain("1/1 serving"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // A build that lands and a rollout that never converges have different remedies, so the + // exhausted budget must not report the second as the first. + it.live("fails with a rollout timeout, not a build timeout, when instances never serve", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer } = setupCompute({ + workdir: repo.dir, + routes: routes({ + [`GET ${computeRoute("/api")}`]: tally({ + declared: 2, + live: 1, + ready: 1, + stale: 0, + }), + }), + }); + + return yield* Effect.gen(function* () { + const error = yield* push({ instances: Option.some(2) }).pipe(Effect.flip); + expect(error).toBeInstanceOf(ComputeRolloutTimeoutError); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + }); + + // `V2DeployAWorkerOutput` permits a terminal state on the deploy response itself. A `failed` + // verdict is this deploy's and a poll could only contradict it — `awaitComputeServing` reads a + // post-deploy 404 as "not settled yet", so an already-failed deploy would burn the poll budget + // and surface as a timeout instead of the answer the platform already gave. An `active` one is + // different: it means nothing was built, not that the rollout is done. + describe("a terminal deploy response", () => { const settledOnDeploy = (repoDir: string, state: "active" | "failed") => setupCompute({ workdir: repoDir, @@ -210,7 +345,11 @@ describe("compute push", () => { }), }); - it.live("reports a deploy that came back already active", () => + // The bare-image case: a redeploy whose code never enters the image builds nothing, so the + // deploy answers `active` while every instance still serves the previous code. Taking that + // as the end of the story is what let `push` report "Deployed" over a rollout that had not + // started, so an `active` response is polled like any other. + it.live("still waits for the rollout when the deploy came back already active", () => Effect.gen(function* () { const repo = yield* project(); const { layer, out, http } = settledOnDeploy(repo.dir, "active"); @@ -218,9 +357,10 @@ describe("compute push", () => { return yield* Effect.gen(function* () { yield* push(); - expect(http.routeKeys).not.toContain(`GET ${computeRoute("/api")}`); + expect(http.routeKeys).toContain(`GET ${computeRoute("/api")}`); expect(out.stdoutText).toContain("Deployed Compute api"); expect(out.stdoutText).toContain("v1"); + expect(out.stdoutText).toContain("1/1 serving"); }).pipe(Effect.provide(layer)); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); @@ -251,7 +391,7 @@ describe("compute push", () => { routes: routes({ [`GET ${computeRoute("/api")}`]: { status: 200, - body: { data: computeResource({ name: "api", buildState: "active" }) }, + body: { data: rolledOut({ name: "api", buildState: "active" }) }, }, }), }); @@ -701,7 +841,7 @@ describe("compute push", () => { { status: 200, body: { - data: computeResource({ name: "api", buildState: "active", imageVersion: "v2" }), + data: rolledOut({ name: "api", buildState: "active", imageVersion: "v2" }), }, }, ], @@ -785,7 +925,7 @@ describe("compute push", () => { [`GET ${computeRoute("/api")}`]: { status: 200, body: { - data: computeResource({ + data: rolledOut({ name: "api", runtime: "node", buildState: "active", @@ -1392,7 +1532,7 @@ describe("compute push", () => { { status: 500, body: { message: "blip" } }, { status: 200, - body: { data: computeResource({ name: "api", buildState: "active" }) }, + body: { data: rolledOut({ name: "api", buildState: "active" }) }, }, ], }), @@ -1483,7 +1623,7 @@ describe("compute push", () => { }, [`GET ${computeRoute("/web")}`]: { status: 200, - body: { data: computeResource({ name: "web", runtime: "node", buildState: "active" }) }, + body: { data: rolledOut({ name: "web", runtime: "node", buildState: "active" }) }, }, }, }); @@ -1765,6 +1905,12 @@ describe("compute push", () => { build_state: "active", image_version: "v1", url: `https://${COMPUTE_PROJECT_REF}.supabase.co/compute/v1/api`, + instances_ready: 1, + instances_stale: 0, + // Wall-clock, so only its presence is assertable. This is what a machine consumer + // reads to see where a slow deploy spent its time. + waited_ms: expect.any(Number), + waited_build_ms: expect.any(Number), }, ]); }).pipe(Effect.provide(layer)); @@ -1909,7 +2055,7 @@ describe("compute push", () => { }, [`GET ${computeRoute("/web")}`]: { status: 200, - body: { data: computeResource({ name: "web", runtime: "node", buildState: "active" }) }, + body: { data: rolledOut({ name: "web", runtime: "node", buildState: "active" }) }, }, }, }); @@ -2000,7 +2146,7 @@ describe("compute push", () => { routes: routes({ [`GET ${computeRoute("/api")}`]: { status: 200, - body: { data: computeResource({ name: "api", runtime: "node", buildState: "active" }) }, + body: { data: rolledOut({ name: "api", runtime: "node", buildState: "active" }) }, }, }), }); diff --git a/apps/cli/src/shared/compute/compute-api.ts b/apps/cli/src/shared/compute/compute-api.ts index ef1a6662db..c5c901b425 100644 --- a/apps/cli/src/shared/compute/compute-api.ts +++ b/apps/cli/src/shared/compute/compute-api.ts @@ -6,7 +6,7 @@ import { V2ListAllComputeInstancesOutput, type ApiClient, } from "@supabase/api/effect"; -import { Effect, Option, Schedule } from "effect"; +import { Duration, Effect, Option, Schedule } from "effect"; import * as HttpClient from "effect/unstable/http/HttpClient"; import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; import { @@ -22,6 +22,7 @@ import { import { ComputeBuildTimeoutError, ComputeProjectNotFoundError, + ComputeRolloutTimeoutError, ComputeRouteNotFoundError, ComputeUnavailableError, ComputeUploadFailedError, @@ -317,13 +318,56 @@ export const deleteCompute = Effect.fnUntraced(function* ( return yield* unexpectedStatus(operation, response); }); +/** + * Whether every declared instance is up, current, and accepting connections. + * + * `stale` is the term that carries a redeploy: a bare-image deploy produces no new image version, + * so the rotation cutoff is the only thing distinguishing instances still on the old code. `live` + * carries a scale-down, where surplus instances keep serving until the control plane trims them. + * + * An absent tally is never convergence. The field is omitted entirely until the first image lands + * — precisely when the wait matters most — and it is replaced by `instances_error` when the + * read-through fails, which is a reason to keep waiting rather than to believe anything. + */ +export function isComputeServing(compute: ComputeRecord): boolean { + const { instances } = compute; + if (instances === undefined) { + return false; + } + return ( + instances.ready === instances.declared && + instances.live === instances.declared && + instances.stale === 0 + ); +} + +/** + * How long `push` polls before giving up. + * + * Tracks what the control plane's own end-to-end suite allows a healthy deploy — 15 minutes for + * a build to reach `active`, and 6 more for it to serve. The previous bound here was 10 minutes + * for the build alone, under that, so a slow-but-healthy build surfaced as a CLI timeout rather + * than as the wait it was. + */ +const COMPUTE_WAIT_BUDGET = "22 minutes"; + +/** Ceiling on the poll interval, before which the backoff below grows geometrically. */ +const COMPUTE_POLL_MAX_INTERVAL = Duration.seconds(15); + /** * The build runs asynchronously — deploy answers 202 and the compute reaches `active` or `failed` - * later — so `push` polls `get` until `build_state` leaves `building`. Overridable so tests can - * drive the loop without waiting on wall-clock delays. + * later — so `push` polls `get` until the rollout that follows the build has converged. + * + * The cadence starts tight so a quick deploy still feels immediate, then backs off. A flat 2s was + * affordable when the wait ended at the build; it is not across a rollout, which runs minutes + * longer and whose instance counts the control plane reads through to its backend rather than + * serving from cache. Overridable so tests can drive the loop without wall-clock delays. */ -const COMPUTE_BUILD_POLL_SCHEDULE = Schedule.spaced("2 seconds").pipe( - Schedule.upTo({ duration: "10 minutes" }), +const COMPUTE_POLL_SCHEDULE = Schedule.exponential("2 seconds", 1.5).pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, COMPUTE_POLL_MAX_INTERVAL)), + ), + Schedule.upTo({ duration: COMPUTE_WAIT_BUDGET }), ); /** @@ -344,7 +388,29 @@ const isPermanentReadFailure = (error: unknown) => error instanceof ComputeUnavailableError || error instanceof ComputeProjectNotFoundError; -export const awaitComputeBuild = Effect.fnUntraced(function* ( +/** + * The record a poll may stop on, or undefined to keep waiting. + * + * A failed build settles too: the caller raises it, and no rollout follows a build that produced + * nothing to roll out. + */ +function settled(compute: ComputeRecord): ComputeRecord | undefined { + if (compute.buildState === "building") { + return undefined; + } + if (compute.buildState === "failed") { + return compute; + } + return isComputeServing(compute) ? compute : undefined; +} + +/** + * Polls `get` until the compute's declared instances are all serving the deploy's code, and + * fails with the timeout that names what it was still waiting on — a build that never landed + * and a rollout that never converged have different remedies, and reporting the second as the + * first sends people to their Dockerfile. + */ +export const awaitComputeServing = Effect.fnUntraced(function* ( api: ApiClient, projectRef: string, name: string, @@ -361,6 +427,10 @@ export const awaitComputeBuild = Effect.fnUntraced(function* ( readonly refSuffix?: string; } = {}, ) { + // The last thing a poll actually saw, so an exhausted budget can say which stage it gave up + // in rather than assuming the build. + let last: ComputeRecord | undefined; + const poll = Effect.gen(function* () { // A build runs for minutes; one blip on one read must not abandon a deploy that is fine. const compute = yield* getCompute(api, projectRef, name).pipe( @@ -373,25 +443,34 @@ export const awaitComputeBuild = Effect.fnUntraced(function* ( // The deploy was accepted, so a 404 here is the read racing the write, not an absence. return undefined; } + last = compute.value; if (options.onPoll !== undefined) { yield* options.onPoll(compute.value); } - return compute.value.buildState === "building" ? undefined : compute.value; + return settled(compute.value); }); - const settled = yield* poll.pipe( + const result = yield* poll.pipe( Effect.repeat({ - schedule: options.schedule ?? COMPUTE_BUILD_POLL_SCHEDULE, - until: (result) => result !== undefined, + schedule: options.schedule ?? COMPUTE_POLL_SCHEDULE, + until: (value) => value !== undefined, }), ); - if (settled === undefined) { + if (result === undefined) { + const refSuffix = options.refSuffix ?? ""; + const check = `Check on it with \`supabase compute status ${name}${refSuffix}\`.`; + if (last?.buildState === "active") { + return yield* new ComputeRolloutTimeoutError({ + detail: `"${name}" built, but its instances were not all serving when this command stopped waiting.`, + suggestion: `${check} \`supabase compute logs ${name}${refSuffix}\` shows an instance that is failing to start.`, + }); + } return yield* new ComputeBuildTimeoutError({ detail: `"${name}" was still building when this command stopped waiting.`, - suggestion: `Check on it with \`supabase compute status ${name}${options.refSuffix ?? ""}\`.`, + suggestion: check, }); } - return settled; + return result; }); diff --git a/apps/cli/src/shared/compute/compute-api.unit.test.ts b/apps/cli/src/shared/compute/compute-api.unit.test.ts new file mode 100644 index 0000000000..92cdd7c23d --- /dev/null +++ b/apps/cli/src/shared/compute/compute-api.unit.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { isComputeServing, type ComputeRecord } from "./compute-api.ts"; + +const record = (overrides: Partial = {}): ComputeRecord => ({ + name: "api", + spec: { size: "2gb-1vcpu", exposure: "public", instances: 2 }, + buildState: "active", + ...overrides, +}); + +describe("isComputeServing", () => { + it("is satisfied when every declared instance is ready and current", () => { + expect( + isComputeServing(record({ instances: { declared: 2, live: 2, ready: 2, stale: 0 } })), + ).toBe(true); + }); + + // The field is omitted entirely until the first image lands — which is exactly when a wait + // matters most, so an absent tally must never read as convergence. + it("is not satisfied when the tally is absent", () => { + expect(isComputeServing(record())).toBe(false); + }); + + // `instances_error` replaces the tally when the read-through fails. It reports nothing about + // how many instances are up, so it is a reason to keep waiting. + it("is not satisfied when the read-through failed", () => { + expect(isComputeServing(record({ instancesError: "instances are unavailable" }))).toBe(false); + }); + + it("is not satisfied while instances are still starting", () => { + expect( + isComputeServing(record({ instances: { declared: 2, live: 2, ready: 1, stale: 0 } })), + ).toBe(false); + }); + + // The term that carries a bare-image redeploy: no new image version is produced, so the + // rotation cutoff is the only thing marking instances still on the old code. + it("is not satisfied while an instance is still on the previous code", () => { + expect( + isComputeServing(record({ instances: { declared: 1, live: 1, ready: 1, stale: 1 } })), + ).toBe(false); + }); + + // A scale-down leaves surplus instances serving until the control plane trims them. + it("is not satisfied while surplus instances are still live", () => { + expect( + isComputeServing(record({ instances: { declared: 1, live: 3, ready: 3, stale: 0 } })), + ).toBe(false); + }); +}); diff --git a/apps/cli/src/shared/compute/compute.errors.ts b/apps/cli/src/shared/compute/compute.errors.ts index ae29737c74..09f4b100d3 100644 --- a/apps/cli/src/shared/compute/compute.errors.ts +++ b/apps/cli/src/shared/compute/compute.errors.ts @@ -170,6 +170,20 @@ export class ComputeBuildTimeoutError extends Data.TaggedError("ComputeBuildTime } } +/** + * The build landed, but the declared instances were not all serving inside the polling budget. + * Distinct from a build failure: the image is good, so the remedy is the instances' own logs + * rather than the Dockerfile. + */ +export class ComputeRolloutTimeoutError extends Data.TaggedError("ComputeRolloutTimeoutError")<{ + readonly detail: string; + readonly suggestion: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.apiStatus; + } +} + /** PUTting the build context to the presigned slot failed. */ export class ComputeUploadFailedError extends Data.TaggedError("ComputeUploadFailedError")<{ readonly detail: string;