diff --git a/apps/cli/src/commands/experimental/compute/compute.format.ts b/apps/cli/src/commands/experimental/compute/compute.format.ts index 56e9e8002e..e46aecac64 100644 --- a/apps/cli/src/commands/experimental/compute/compute.format.ts +++ b/apps/cli/src/commands/experimental/compute/compute.format.ts @@ -22,3 +22,23 @@ export function renderComputeDetails(rows: ReadonlyArray label.length)); return `${present.map(([label, value]) => ` ${label.padEnd(width)} ${value}`).join("\n")}\n`; } + +/** + * A wait, for the deploy summary: `48s`, `3m21s`, `1h04m`. + * + * Whole seconds throughout — these measure a server-side build and rollout, where sub-second + * precision would imply the CLI polled far more tightly than it does. + */ +export function formatWaited(millis: number): string { + const total = Math.max(0, Math.round(millis / 1000)); + const seconds = total % 60; + const minutes = Math.floor(total / 60) % 60; + const hours = Math.floor(total / 3600); + if (hours > 0) { + return `${hours}h${String(minutes).padStart(2, "0")}m`; + } + if (minutes > 0) { + return `${minutes}m${String(seconds).padStart(2, "0")}s`; + } + return `${seconds}s`; +} diff --git a/apps/cli/src/commands/experimental/compute/compute.format.unit.test.ts b/apps/cli/src/commands/experimental/compute/compute.format.unit.test.ts index 79aabfe23a..c3b213d8e3 100644 --- a/apps/cli/src/commands/experimental/compute/compute.format.unit.test.ts +++ b/apps/cli/src/commands/experimental/compute/compute.format.unit.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { renderComputeDetails } from "./compute.format.ts"; +import { formatWaited, renderComputeDetails } from "./compute.format.ts"; describe("renderComputeDetails", () => { it("pads every label to the widest one", () => { @@ -32,3 +32,24 @@ describe("renderComputeDetails", () => { ).toBe(""); }); }); + +describe("formatWaited", () => { + it("reports a sub-minute wait in seconds", () => { + expect(formatWaited(48_000)).toBe("48s"); + }); + + it("pads the seconds so minute-scale waits align in a column", () => { + expect(formatWaited(201_000)).toBe("3m21s"); + expect(formatWaited(184_000)).toBe("3m04s"); + }); + + it("drops to minutes past an hour, where the seconds are noise", () => { + expect(formatWaited(3_840_000)).toBe("1h04m"); + }); + + // Rounds rather than truncates, so a poll that lands at 1999ms is not reported as one second. + it("rounds to whole seconds", () => { + expect(formatWaited(1_600)).toBe("2s"); + expect(formatWaited(0)).toBe("0s"); + }); +}); diff --git a/apps/cli/src/commands/experimental/compute/push/push.command.ts b/apps/cli/src/commands/experimental/compute/push/push.command.ts index bdb2640610..5237e2a971 100644 --- a/apps/cli/src/commands/experimental/compute/push/push.command.ts +++ b/apps/cli/src/commands/experimental/compute/push/push.command.ts @@ -72,12 +72,12 @@ const config = { Flag.optional, ), noWait: Flag.Boolean("no-wait").pipe( - // The deploy POST returns once the platform accepts the spec and context; the - // server-side build that follows can run for minutes. Waiting stays the - // default so a plain push reports the build's verdict; `--no-wait` opts out - // for callers that only need the deploy accepted. + // The deploy POST returns once the platform accepts the spec and context; the build and the + // rollout that follows it can run for minutes between them. Waiting stays the default so a + // plain push reports what is actually serving; `--no-wait` opts out for callers that only + // need the deploy accepted. Flag.withDescription( - "Return once the deploy is accepted, without waiting for the server-side build to finish.", + "Return once the deploy is accepted, without waiting for the new code to roll out.", ), Flag.withDefault(false), ), diff --git a/apps/cli/src/commands/experimental/compute/push/push.handler.ts b/apps/cli/src/commands/experimental/compute/push/push.handler.ts index fff6ab0fa2..8a8acfbd77 100644 --- a/apps/cli/src/commands/experimental/compute/push/push.handler.ts +++ b/apps/cli/src/commands/experimental/compute/push/push.handler.ts @@ -1,8 +1,8 @@ import { cliConfigFormatForPath } from "@supabase/config/internal"; -import { Effect, FileSystem, Option, Path, type Schedule } from "effect"; +import { Clock, Effect, FileSystem, Option, Path, type Schedule } from "effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { emitSuccessTrailer } from "../../../../shared/cli/success-trailer.ts"; -import { renderComputeDetails } from "../compute.format.ts"; +import { formatWaited, renderComputeDetails } from "../compute.format.ts"; import { emitComputeMachineOutput, rejectComputeEnvOutput, @@ -42,12 +42,14 @@ import { } from "../../../../shared/compute/compute-runtimes.ts"; import { computeUrl } from "../../../../shared/compute/compute-url.ts"; import { - awaitComputeBuild, + awaitComputeServing, createComputeUpload, deployCompute, + isComputeServing, parseComputeSecretsScopeToAPIFormat, uploadBuildContext, type ComputeDeploySpec, + type ComputeRecord, } from "../../../../shared/compute/compute-api.ts"; import { NoComputeToDeployError, @@ -76,8 +78,9 @@ import type { ComputePushFlags } from "./push.command.ts"; * `supabase compute push [name...]` — deploys the named compute * (or every compute, if none are named), reading runtime, size, exposure, and * source from `[compute.]`; an unrecorded runtime is guessed and - * reported. Builds run server-side from an uploaded context and are waited on - * by default; `--no-wait` returns once the deploy is accepted. + * reported. Builds run server-side from an uploaded context, and a plain push waits for the + * deploy's code to actually be serving — the build landing is not the end of it; `--no-wait` + * returns once the deploy is accepted. */ const entryRef = ({ format }: ConfigFileRef, name: string): string => @@ -288,6 +291,29 @@ function addYourCode(sourceDisplay: string): string { return `Add your compute's code to ${sourceDisplay}, then run this command again.`; } +/** + * The instance row, and the progress line while waiting on a rollout. + * + * Falls back to the declared count alone whenever the tally is absent — it is omitted until the + * first image lands, and replaced by `instances_error` when the control plane's read-through + * fails, neither of which says anything about how many instances are up. + */ +function describeTally(compute: ComputeRecord): string { + const { instances } = compute; + if (instances === undefined) { + return `${compute.spec.instances} declared`; + } + const stale = instances.stale > 0 ? `, ${instances.stale} stale` : ""; + return `${instances.ready}/${instances.declared} serving${stale}`; +} + +/** `3m21s (1m04s building)`, or just the total when nothing was built. */ +function describeWaited(waited: { readonly total: number; readonly build: number }): string { + const total = formatWaited(waited.total); + // Under a second is the poll cadence, not a build worth attributing. + return waited.build < 1000 ? total : `${total} (${formatWaited(waited.build)} building)`; +} + const deployOneCompute = Effect.fn("compute.push.deployOne")(function* (input: { readonly project: ComputeProject; readonly name: string; @@ -483,25 +509,46 @@ const deployOneCompute = Effect.fn("compute.push.deployOne")(function* (input: { Effect.tapError(() => deploying.fail()), ); - // Polled only when the deploy response left the build unresolved. - // `V2DeployAWorkerOutput` permits a terminal `active` or `failed` on the - // deploy itself, and that verdict is this deploy's — a fresh `GET` can only - // contradict it: `awaitComputeBuild` reads a post-deploy 404 as "still - // building", so an already-`failed` deploy could burn the whole poll budget - // and surface as a timeout, and a concurrent deployment could answer with a - // state that belongs to someone else's build. - const settled = - input.noWait || accepted.buildState !== "building" - ? accepted - : yield* awaitComputeBuild(api, projectRef, name, { - schedule: input.pollSchedule, - retrySchedule: input.pollRetrySchedule, - refSuffix: input.refSuffix, - onPoll: (polled) => - polled.buildState === "building" - ? deploying.message("Building compute...") - : Effect.void, - }).pipe(Effect.tapError(() => deploying.fail())); + // A terminal `active` on the deploy response is no longer a reason to skip the poll. It is + // exactly the case where nothing was built — a bare-image redeploy, or a build skipped as + // unchanged — and the instances go on serving the previous code until the control plane + // recycles them. Stopping there is what let `push` report "Deployed" over a rollout that had + // not started. A `failed` response still short-circuits: that verdict is this deploy's, and a + // fresh `GET` can only contradict it — `awaitComputeServing` reads a post-deploy 404 as "not + // settled yet", so an already-failed deploy would burn the whole poll budget and surface as a + // timeout, and a concurrent deployment could answer with a state belonging to someone else's + // build. + const shortCircuit = input.noWait || accepted.buildState === "failed"; + + // Transitions as this deploy's own polls saw them, at the poll cadence's resolution. This is + // the only timing available to the CLI: the API publishes no per-stage stamps, so the build + // mark is "when we first observed the image landed", which includes the control plane's own + // detection lag rather than the build alone. + const startedAt = yield* Clock.currentTimeMillis; + let builtAt: number | undefined = accepted.buildState === "building" ? undefined : startedAt; + + const settled = shortCircuit + ? accepted + : yield* awaitComputeServing(api, projectRef, name, { + schedule: input.pollSchedule, + retrySchedule: input.pollRetrySchedule, + refSuffix: input.refSuffix, + onPoll: (polled) => + Effect.gen(function* () { + if (polled.buildState === "building") { + return yield* deploying.message("Building compute..."); + } + builtAt ??= yield* Clock.currentTimeMillis; + if (!isComputeServing(polled)) { + yield* deploying.message(`Starting instances... ${describeTally(polled)}`); + } + }), + }).pipe(Effect.tapError(() => deploying.fail())); + + const finishedAt = yield* Clock.currentTimeMillis; + const waited = shortCircuit + ? undefined + : { total: finishedAt - startedAt, build: (builtAt ?? finishedAt) - startedAt }; yield* Effect.annotateCurrentSpan({ "compute.waited": !input.noWait && accepted.buildState === "building", @@ -558,6 +605,11 @@ const deployOneCompute = Effect.fn("compute.push.deployOne")(function* (input: { // empty-valued row. ["Image", imageVersion ?? ""], ["Access", settled.spec.exposure], + ["Instances", describeTally(settled)], + // How long the wait actually took, and how much of it was over before the image + // landed. The API publishes no per-stage timing, so this is the only attribution a + // caller gets for a deploy that felt slow. + ["Waited", waited === undefined ? "" : describeWaited(waited)], ["Secrets Scope", formatSecretsScope(settledSecretsScope ?? "")], ["URL", url ?? ""], ]), @@ -586,6 +638,10 @@ const deployOneCompute = Effect.fn("compute.push.deployOne")(function* (input: { // deploy completed. Same reason `url` is spread below. ...(imageVersion === undefined ? {} : { image_version: imageVersion }), build_state: settled.buildState, + ...(settled.instances === undefined ? {} : { instances_ready: settled.instances.ready }), + ...(settled.instances === undefined ? {} : { instances_stale: settled.instances.stale }), + // Flat scalars rather than a nested object: `-o env` and `-o toml` both take this payload. + ...(waited === undefined ? {} : { waited_ms: waited.total, waited_build_ms: waited.build }), ...(url === undefined ? {} : { url }), }; }); diff --git a/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts b/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts index 8a72f6319c..326dd545b2 100644 --- a/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts +++ b/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts @@ -26,6 +26,7 @@ import { ComputeBuildFailedError, ComputeBuildTimeoutError, ComputeProjectNotFoundError, + ComputeRolloutTimeoutError, ComputeRouteNotFoundError, ComputeUnavailableError, ComputeSourceEscapingLinkError, @@ -98,6 +99,20 @@ function project(files: Readonly> = {}) { }); } +/** + * A compute that has finished rolling out, for the `GET` a poll reads. + * + * `push` waits for the instance tally to converge, so a fake standing in for a healthy deploy + * has to carry one: an absent tally is deliberately not convergence, since the field is omitted + * until the first image lands. + */ +function rolledOut(options: Parameters[0]) { + return computeResource({ + instanceCounts: { declared: 1, live: 1, ready: 1, stale: 0 }, + ...options, + }); +} + function routes(overrides: ComputeHttpRoutes = {}): ComputeHttpRoutes { return { [`POST ${computeRoute("/api/uploads")}`]: { status: 201, body: uploadSlot }, @@ -109,7 +124,7 @@ function routes(overrides: ComputeHttpRoutes = {}): ComputeHttpRoutes { [`GET ${computeRoute("/api")}`]: { status: 200, body: { - data: computeResource({ + data: rolledOut({ name: "api", runtime: "node", buildState: "active", @@ -231,6 +246,126 @@ describe("compute push", () => { }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); + describe("waiting for the rollout", () => { + const tally = (counts: { declared: number; live: number; ready: number; stale: number }) => ({ + status: 200, + body: { + data: computeResource({ + name: "api", + runtime: "node", + buildState: "active" as const, + imageVersion: "v1", + instanceCounts: counts, + }), + }, + }); + + it.live("keeps polling until every declared instance is serving the new code", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer, out } = setupCompute({ + workdir: repo.dir, + routes: routes({ + [`GET ${computeRoute("/api")}`]: [ + tally({ declared: 2, live: 0, ready: 0, stale: 0 }), + tally({ declared: 2, live: 2, ready: 1, stale: 0 }), + tally({ declared: 2, live: 2, ready: 2, stale: 0 }), + ], + }), + }); + + return yield* Effect.gen(function* () { + yield* push({ instances: Option.some(2) }); + + expect(out.stdoutText).toContain("Deployed Compute api"); + // The live tally, not the declared count: the whole point of waiting. + expect(out.stdoutText).toContain("2/2 serving"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // A redeploy that produces no new image version leaves the rotation cutoff as the only + // thing marking instances still on the old code, so `stale` is what keeps the wait open. + it.live("keeps waiting while an instance is still on the previous code", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer, out } = setupCompute({ + workdir: repo.dir, + routes: routes({ + [`GET ${computeRoute("/api")}`]: [ + tally({ declared: 1, live: 1, ready: 1, stale: 1 }), + tally({ declared: 1, live: 1, ready: 1, stale: 0 }), + ], + }), + }); + + return yield* Effect.gen(function* () { + yield* push(); + + expect(out.stdoutText).toContain("1/1 serving"); + expect(out.stdoutText).not.toContain("stale"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // `instances_error` replaces the tally when the control plane's read-through fails. It says + // nothing about how many instances are up, so it must not be read as convergence. + it.live("keeps waiting when the instance read-through fails", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer, out } = setupCompute({ + workdir: repo.dir, + routes: routes({ + [`GET ${computeRoute("/api")}`]: [ + { + status: 200, + body: { + data: computeResource({ + name: "api", + runtime: "node", + buildState: "active", + imageVersion: "v1", + instancesError: "instances are temporarily unavailable", + }), + }, + }, + tally({ declared: 1, live: 1, ready: 1, stale: 0 }), + ], + }), + }); + + return yield* Effect.gen(function* () { + yield* push(); + expect(out.stdoutText).toContain("1/1 serving"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // A build that lands and a rollout that never converges have different remedies, so the + // exhausted budget must not report the second as the first. + it.live("fails with a rollout timeout, not a build timeout, when instances never serve", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer } = setupCompute({ + workdir: repo.dir, + routes: routes({ + [`GET ${computeRoute("/api")}`]: tally({ + declared: 2, + live: 1, + ready: 1, + stale: 0, + }), + }), + }); + + return yield* Effect.gen(function* () { + const error = yield* push({ instances: Option.some(2) }).pipe(Effect.flip); + expect(error).toBeInstanceOf(ComputeRolloutTimeoutError); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + }); + it.effect("picks the project, deploys the compute and records the linked project", () => Effect.gen(function* () { const repo = yield* project(); @@ -273,12 +408,12 @@ describe("compute push", () => { }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); - // `V2DeployAWorkerOutput` permits a terminal state on the deploy response - // itself, and that verdict is this deploy's. A poll on top of it can only - // contradict it — `awaitComputeBuild` reads a post-deploy 404 as "still - // building", so an already-settled deploy would burn the poll budget and - // surface as a timeout instead of the answer the platform already gave. - describe("honours a terminal deploy response instead of polling", () => { + // `V2DeployAWorkerOutput` permits a terminal state on the deploy response itself. A `failed` + // verdict is this deploy's and a poll could only contradict it — `awaitComputeServing` reads a + // post-deploy 404 as "not settled yet", so an already-failed deploy would burn the poll budget + // and surface as a timeout instead of the answer the platform already gave. An `active` one is + // different: it means nothing was built, not that the rollout is done. + describe("a terminal deploy response", () => { const settledOnDeploy = (repoDir: string, state: "active" | "failed") => setupPush({ workdir: repoDir, @@ -297,7 +432,11 @@ describe("compute push", () => { }), }); - it.effect("reports a deploy that came back already active", () => + // The bare-image case: a redeploy whose code never enters the image builds nothing, so the + // deploy answers `active` while every instance still serves the previous code. Taking that + // as the end of the story is what let `push` report "Deployed" over a rollout that had not + // started, so an `active` response is polled like any other. + it.live("still waits for the rollout when the deploy came back already active", () => Effect.gen(function* () { const repo = yield* project(); const { layer, out, http } = settledOnDeploy(repo.dir, "active"); @@ -305,9 +444,10 @@ describe("compute push", () => { return yield* Effect.gen(function* () { yield* push(); - expect(http.routeKeys).not.toContain(`GET ${computeRoute("/api")}`); + expect(http.routeKeys).toContain(`GET ${computeRoute("/api")}`); expect(out.stdoutText).toContain("Deployed Compute api"); expect(out.stdoutText).toContain("v1"); + expect(out.stdoutText).toContain("1/1 serving"); }).pipe(Effect.provide(layer)); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); @@ -338,7 +478,7 @@ describe("compute push", () => { routes: routes({ [`GET ${computeRoute("/api")}`]: { status: 200, - body: { data: computeResource({ name: "api", buildState: "active" }) }, + body: { data: rolledOut({ name: "api", buildState: "active" }) }, }, }), }); @@ -891,7 +1031,7 @@ describe("compute push", () => { { status: 200, body: { - data: computeResource({ name: "api", buildState: "active", imageVersion: "v2" }), + data: rolledOut({ name: "api", buildState: "active", imageVersion: "v2" }), }, }, ], @@ -975,7 +1115,7 @@ describe("compute push", () => { [`GET ${computeRoute("/api")}`]: { status: 200, body: { - data: computeResource({ + data: rolledOut({ name: "api", runtime: "node", buildState: "active", @@ -1584,7 +1724,7 @@ describe("compute push", () => { { status: 500, body: { message: "blip" } }, { status: 200, - body: { data: computeResource({ name: "api", buildState: "active" }) }, + body: { data: rolledOut({ name: "api", buildState: "active" }) }, }, ], }), @@ -1675,7 +1815,7 @@ describe("compute push", () => { }, [`GET ${computeRoute("/web")}`]: { status: 200, - body: { data: computeResource({ name: "web", runtime: "node", buildState: "active" }) }, + body: { data: rolledOut({ name: "web", runtime: "node", buildState: "active" }) }, }, }, }); @@ -1957,6 +2097,12 @@ describe("compute push", () => { build_state: "active", image_version: "v1", url: `https://${COMPUTE_PROJECT_REF}.supabase.co/compute/v1/api`, + instances_ready: 1, + instances_stale: 0, + // Wall-clock, so only its presence is assertable. This is what a machine consumer + // reads to see where a slow deploy spent its time. + waited_ms: expect.any(Number), + waited_build_ms: expect.any(Number), }, ]); }).pipe(Effect.provide(layer)); @@ -2101,7 +2247,7 @@ describe("compute push", () => { }, [`GET ${computeRoute("/web")}`]: { status: 200, - body: { data: computeResource({ name: "web", runtime: "node", buildState: "active" }) }, + body: { data: rolledOut({ name: "web", runtime: "node", buildState: "active" }) }, }, }, }); @@ -2192,7 +2338,7 @@ describe("compute push", () => { routes: routes({ [`GET ${computeRoute("/api")}`]: { status: 200, - body: { data: computeResource({ name: "api", runtime: "node", buildState: "active" }) }, + body: { data: rolledOut({ name: "api", runtime: "node", buildState: "active" }) }, }, }), }); diff --git a/apps/cli/src/shared/compute/compute-api.ts b/apps/cli/src/shared/compute/compute-api.ts index f35f30eaf5..f4b8214650 100644 --- a/apps/cli/src/shared/compute/compute-api.ts +++ b/apps/cli/src/shared/compute/compute-api.ts @@ -6,7 +6,7 @@ import { V2ListAllComputeInstancesOutput, type ApiClient, } from "@supabase/api/effect"; -import { Effect, Option, Schedule } from "effect"; +import { Duration, Effect, Option, Schedule } from "effect"; import { block } from "../../command-internal/command-permissions/model.ts"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientRequest from "effect/http/HttpClientRequest"; @@ -23,6 +23,7 @@ import { import { ComputeBuildTimeoutError, ComputeProjectNotFoundError, + ComputeRolloutTimeoutError, ComputeRouteNotFoundError, ComputeUnavailableError, ComputeUploadFailedError, @@ -347,13 +348,56 @@ export const deleteCompute = Effect.fnUntraced(function* ( return yield* unexpectedStatus(operation, response); }); +/** + * Whether every declared instance is up, current, and accepting connections. + * + * `stale` is the term that carries a redeploy: a bare-image deploy produces no new image version, + * so the rotation cutoff is the only thing distinguishing instances still on the old code. `live` + * carries a scale-down, where surplus instances keep serving until the control plane trims them. + * + * An absent tally is never convergence. The field is omitted entirely until the first image lands + * — precisely when the wait matters most — and it is replaced by `instances_error` when the + * read-through fails, which is a reason to keep waiting rather than to believe anything. + */ +export function isComputeServing(compute: ComputeRecord): boolean { + const { instances } = compute; + if (instances === undefined) { + return false; + } + return ( + instances.ready === instances.declared && + instances.live === instances.declared && + instances.stale === 0 + ); +} + +/** + * How long `push` polls before giving up. + * + * Tracks what the control plane's own end-to-end suite allows a healthy deploy — 15 minutes for + * a build to reach `active`, and 6 more for it to serve. The previous bound here was 10 minutes + * for the build alone, under that, so a slow-but-healthy build surfaced as a CLI timeout rather + * than as the wait it was. + */ +const COMPUTE_WAIT_BUDGET = "22 minutes"; + +/** Ceiling on the poll interval, before which the backoff below grows geometrically. */ +const COMPUTE_POLL_MAX_INTERVAL = Duration.seconds(15); + /** * The build runs asynchronously — deploy answers 202 and the compute reaches `active` or `failed` - * later — so `push` polls `get` until `build_state` leaves `building`. Overridable so tests can - * drive the loop without waiting on wall-clock delays. + * later — so `push` polls `get` until the rollout that follows the build has converged. + * + * The cadence starts tight so a quick deploy still feels immediate, then backs off. A flat 2s was + * affordable when the wait ended at the build; it is not across a rollout, which runs minutes + * longer and whose instance counts the control plane reads through to its backend rather than + * serving from cache. Overridable so tests can drive the loop without wall-clock delays. */ -const COMPUTE_BUILD_POLL_SCHEDULE = Schedule.spaced("2 seconds").pipe( - Schedule.upTo({ duration: "10 minutes" }), +const COMPUTE_POLL_SCHEDULE = Schedule.exponential("2 seconds", 1.5).pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, COMPUTE_POLL_MAX_INTERVAL)), + ), + Schedule.upTo({ duration: COMPUTE_WAIT_BUDGET }), ); /** @@ -374,7 +418,29 @@ const isPermanentReadFailure = (error: unknown) => error instanceof ComputeUnavailableError || error instanceof ComputeProjectNotFoundError; -export const awaitComputeBuild = Effect.fn("Compute.awaitBuild")(function* ( +/** + * The record a poll may stop on, or undefined to keep waiting. + * + * A failed build settles too: the caller raises it, and no rollout follows a build that produced + * nothing to roll out. + */ +function settled(compute: ComputeRecord): ComputeRecord | undefined { + if (compute.buildState === "building") { + return undefined; + } + if (compute.buildState === "failed") { + return compute; + } + return isComputeServing(compute) ? compute : undefined; +} + +/** + * Polls `get` until the compute's declared instances are all serving the deploy's code, and + * fails with the timeout that names what it was still waiting on — a build that never landed + * and a rollout that never converged have different remedies, and reporting the second as the + * first sends people to their Dockerfile. + */ +export const awaitComputeServing = Effect.fn("Compute.awaitServing")(function* ( api: ApiClient, projectRef: string, name: string, @@ -391,6 +457,9 @@ export const awaitComputeBuild = Effect.fn("Compute.awaitBuild")(function* ( readonly refSuffix?: string; } = {}, ) { + // The last thing a poll actually saw, so an exhausted budget can say which stage it gave up + // in rather than assuming the build. + let last: ComputeRecord | undefined; let attempts = 0; // Polls emit no request spans; this span's attempt count stands in for them. const poll = Effect.gen(function* () { @@ -406,26 +475,35 @@ export const awaitComputeBuild = Effect.fn("Compute.awaitBuild")(function* ( // The deploy was accepted, so a 404 here is the read racing the write, not an absence. return undefined; } + last = compute.value; if (options.onPoll !== undefined) { yield* options.onPoll(compute.value); } - return compute.value.buildState === "building" ? undefined : compute.value; + return settled(compute.value); }).pipe(Effect.withTracerEnabled(false)); - const settled = yield* poll.pipe( + const result = yield* poll.pipe( Effect.repeat({ - schedule: options.schedule ?? COMPUTE_BUILD_POLL_SCHEDULE, - until: (result) => result !== undefined, + schedule: options.schedule ?? COMPUTE_POLL_SCHEDULE, + until: (value) => value !== undefined, }), ); yield* Effect.annotateCurrentSpan({ "poll.attempt_count": attempts }); - if (settled === undefined) { + if (result === undefined) { + const refSuffix = options.refSuffix ?? ""; + const check = `Check on it with \`supabase compute status ${name}${refSuffix}\`.`; + if (last?.buildState === "active") { + return yield* new ComputeRolloutTimeoutError({ + detail: `"${name}" built, but its instances were not all serving when this command stopped waiting.`, + suggestion: `${check} \`supabase compute logs ${name}${refSuffix}\` shows an instance that is failing to start.`, + }); + } return yield* new ComputeBuildTimeoutError({ detail: `"${name}" was still building when this command stopped waiting.`, - suggestion: `Check on it with \`supabase compute status ${name}${options.refSuffix ?? ""}\`.`, + suggestion: check, }); } - return settled; + return result; }); diff --git a/apps/cli/src/shared/compute/compute-api.unit.test.ts b/apps/cli/src/shared/compute/compute-api.unit.test.ts new file mode 100644 index 0000000000..92cdd7c23d --- /dev/null +++ b/apps/cli/src/shared/compute/compute-api.unit.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { isComputeServing, type ComputeRecord } from "./compute-api.ts"; + +const record = (overrides: Partial = {}): ComputeRecord => ({ + name: "api", + spec: { size: "2gb-1vcpu", exposure: "public", instances: 2 }, + buildState: "active", + ...overrides, +}); + +describe("isComputeServing", () => { + it("is satisfied when every declared instance is ready and current", () => { + expect( + isComputeServing(record({ instances: { declared: 2, live: 2, ready: 2, stale: 0 } })), + ).toBe(true); + }); + + // The field is omitted entirely until the first image lands — which is exactly when a wait + // matters most, so an absent tally must never read as convergence. + it("is not satisfied when the tally is absent", () => { + expect(isComputeServing(record())).toBe(false); + }); + + // `instances_error` replaces the tally when the read-through fails. It reports nothing about + // how many instances are up, so it is a reason to keep waiting. + it("is not satisfied when the read-through failed", () => { + expect(isComputeServing(record({ instancesError: "instances are unavailable" }))).toBe(false); + }); + + it("is not satisfied while instances are still starting", () => { + expect( + isComputeServing(record({ instances: { declared: 2, live: 2, ready: 1, stale: 0 } })), + ).toBe(false); + }); + + // The term that carries a bare-image redeploy: no new image version is produced, so the + // rotation cutoff is the only thing marking instances still on the old code. + it("is not satisfied while an instance is still on the previous code", () => { + expect( + isComputeServing(record({ instances: { declared: 1, live: 1, ready: 1, stale: 1 } })), + ).toBe(false); + }); + + // A scale-down leaves surplus instances serving until the control plane trims them. + it("is not satisfied while surplus instances are still live", () => { + expect( + isComputeServing(record({ instances: { declared: 1, live: 3, ready: 3, stale: 0 } })), + ).toBe(false); + }); +}); diff --git a/apps/cli/src/shared/compute/compute.errors.ts b/apps/cli/src/shared/compute/compute.errors.ts index 412c127903..cb72dd11ca 100644 --- a/apps/cli/src/shared/compute/compute.errors.ts +++ b/apps/cli/src/shared/compute/compute.errors.ts @@ -186,6 +186,20 @@ export class ComputeBuildTimeoutError extends Data.TaggedError("ComputeBuildTime } } +/** + * The build landed, but the declared instances were not all serving inside the polling budget. + * Distinct from a build failure: the image is good, so the remedy is the instances' own logs + * rather than the Dockerfile. + */ +export class ComputeRolloutTimeoutError extends Data.TaggedError("ComputeRolloutTimeoutError")<{ + readonly detail: string; + readonly suggestion: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.apiStatus; + } +} + /** PUTting the build context to the presigned slot failed. */ export class ComputeUploadFailedError extends Data.TaggedError("ComputeUploadFailedError")<{ readonly detail: string;