Keep CI moving through transient npm failures (#2039)

* fix(ci): retry npm installs

Transient registry and package-download failures should delay a job briefly instead of requiring a manual rerun. Apply one bounded retry policy across CI, deploy, and release workflows.

* fix(ci): preserve retries for historical Android tags

Manual Android rebuilds may check out releases from before the shared retry script existed. Keep that install step self-contained and allow the shared helper's injected runner to be asynchronous.

* fix(ci): preserve retries for desktop release refs

* refactor(ci): use shared npm retry helper everywhere

.github/workflows/android-apk-release.yml

@@ -71,7 +71,7 @@ jobs:
           scope: "@boudra"
 
       - name: Install JS dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
         env:
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/ci.yml

@@ -33,7 +33,7 @@ jobs:
           cache: "npm"
 
       - name: Install dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
 
       - name: Check formatting
         run: npx oxfmt --check .
@@ -51,7 +51,7 @@ jobs:
           cache: "npm"
 
       - name: Install dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
 
       - name: Lint lockfile
         run: npx --yes lockfile-lint --path package-lock.json --type npm --allowed-hosts npm --validate-https --validate-integrity
@@ -75,7 +75,7 @@ jobs:
           cache: "npm"
 
       - name: Install dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
       - name: Build server stack
         run: npm run build:server
 
@@ -111,9 +111,9 @@ jobs:
         run: git fetch --no-tags origin main:refs/remotes/origin/main
 
       - name: Install dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
       - name: Install agent CLIs for provider tests
-        run: npm install -g @anthropic-ai/claude-code opencode-ai
+        run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code opencode-ai
 
       - name: Build server dependencies
         run: npm run build:server-deps
@@ -139,35 +139,8 @@ jobs:
           node-version: "22"
           cache: "npm"
 
-      - name: Install dependencies with Electron retry
+      - name: Install dependencies with retry
-        if: runner.os != 'Windows'
+        run: node scripts/npm-retry.mjs ci
-        run: |
-          for attempt in 1 2 3; do
-            if npm ci; then
-              exit 0
-            else
-              exit_code=$?
-            fi
-            if [ "$attempt" -eq 3 ]; then
-              exit $exit_code
-            fi
-            sleep $((attempt * 20))
-          done
-
-      - name: Install dependencies with Electron retry
-        if: runner.os == 'Windows'
-        shell: pwsh
-        run: |
-          for ($attempt = 1; $attempt -le 3; $attempt++) {
-            npm ci
-            if ($LASTEXITCODE -eq 0) {
-              exit 0
-            }
-            if ($attempt -eq 3) {
-              exit $LASTEXITCODE
-            }
-            Start-Sleep -Seconds (20 * $attempt)
-          }
       - name: Build server stack
         run: npm run build:server
 
@@ -187,18 +160,7 @@ jobs:
           cache: "npm"
 
       - name: Install dependencies with retry
-        run: |
+        run: node scripts/npm-retry.mjs ci
-          for attempt in 1 2 3; do
-            if npm ci; then
-              exit 0
-            else
-              exit_code=$?
-            fi
-            if [ "$attempt" -eq 3 ]; then
-              exit $exit_code
-            fi
-            sleep $((attempt * 20))
-          done
       - name: Install Playwright browsers
         timeout-minutes: 10
         run: npx playwright install chromium
@@ -222,7 +184,7 @@ jobs:
           cache: "npm"
 
       - name: Install dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
       - name: Build client dependencies
         run: npm run build:client
 
@@ -258,18 +220,7 @@ jobs:
           cache: "npm"
 
       - name: Install dependencies with retry
-        run: |
+        run: node scripts/npm-retry.mjs ci
-          for attempt in 1 2 3; do
-            if npm ci; then
-              exit 0
-            else
-              exit_code=$?
-            fi
-            if [ "$attempt" -eq 3 ]; then
-              exit $exit_code
-            fi
-            sleep $((attempt * 20))
-          done
       - name: Install Playwright browsers
         timeout-minutes: 10
         run: npx playwright install chromium
@@ -282,7 +233,7 @@ jobs:
 
       - name: Install agent CLIs for provider tests
         if: ${{ !matrix.desktop }}
-        run: npm install -g @anthropic-ai/claude-code @openai/codex@0.105.0 opencode-ai
+        run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code @openai/codex@0.105.0 opencode-ai
 
       - name: Run Playwright E2E tests
         if: ${{ !matrix.desktop }}
@@ -317,7 +268,7 @@ jobs:
           cache: "npm"
 
       - name: Install dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
 
       - name: Build relay
         run: npm run build:relay
@@ -343,10 +294,10 @@ jobs:
           cache: "npm"
 
       - name: Install dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
 
       - name: Install agent CLIs for provider tests
-        run: npm install -g @anthropic-ai/claude-code @openai/codex@0.105.0 opencode-ai
+        run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code @openai/codex@0.105.0 opencode-ai
 
       - name: Run CLI tests
         run: npm run test --workspace=@getpaseo/cli

.github/workflows/deploy-app.yml

@@ -24,7 +24,7 @@ jobs:
           scope: "@boudra"
 
       - name: Install dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
         env:
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Build app dependencies

.github/workflows/deploy-relay.yml

@@ -18,7 +18,7 @@ jobs:
           cache: "npm"
 
       - name: Install dependencies
-        run: npm ci --workspace=@getpaseo/relay --include-workspace-root
+        run: node scripts/npm-retry.mjs ci --workspace=@getpaseo/relay --include-workspace-root
 
       - name: Typecheck
         run: npm run typecheck --workspace=@getpaseo/relay

.github/workflows/deploy-website.yml

@@ -29,7 +29,7 @@ jobs:
           cache: "npm"
 
       - name: Install dependencies
-        run: npm ci --workspace=@getpaseo/website --include-workspace-root
+        run: node scripts/npm-retry.mjs ci --workspace=@getpaseo/website --include-workspace-root
 
       - name: Typecheck
         run: npm run typecheck --workspace=@getpaseo/website

.github/workflows/desktop-release.yml

@@ -127,7 +127,7 @@ jobs:
           scope: "@boudra"
 
       - name: Install JS dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
         env:
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -227,7 +227,7 @@ jobs:
           scope: "@boudra"
 
       - name: Install JS dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
         env:
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -325,7 +325,7 @@ jobs:
           scope: "@boudra"
 
       - name: Install JS dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
         env:
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -425,7 +425,7 @@ jobs:
           scope: "@boudra"
 
       - name: Install JS dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
         env:
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/desktop-rollout.yml

@@ -47,7 +47,7 @@ jobs:
           scope: "@boudra"
 
       - name: Install JS dependencies
-        run: npm ci
+        run: node scripts/npm-retry.mjs ci
         env:
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

scripts/npm-retry.mjs

@@ -0,0 +1,59 @@
+import { spawnSync } from "node:child_process";
+import { resolve as resolvePath } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const DEFAULT_ATTEMPTS = 3;
+const DEFAULT_BACKOFF_MS = 20_000;
+
+function runNpm(args) {
+  const command = process.platform === "win32" ? "npm.cmd" : "npm";
+  const result = spawnSync(command, args, {
+    shell: process.platform === "win32",
+    stdio: "inherit",
+  });
+
+  if (result.error) {
+    console.error(`Failed to start npm: ${result.error.message}`);
+    return 1;
+  }
+
+  return result.status ?? 1;
+}
+
+function wait(delayMs) {
+  return new Promise((resolve) => setTimeout(resolve, delayMs));
+}
+
+export async function runWithRetry(
+  args,
+  { attempts = DEFAULT_ATTEMPTS, backoffMs = DEFAULT_BACKOFF_MS, run = runNpm, sleep = wait } = {},
+) {
+  let exitCode = 1;
+
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    exitCode = await run(args);
+    if (exitCode === 0) {
+      return 0;
+    }
+
+    if (attempt < attempts) {
+      const delayMs = attempt * backoffMs;
+      console.warn(
+        `npm failed with exit code ${exitCode}; retrying in ${delayMs / 1000}s (${attempt + 1}/${attempts})`,
+      );
+      await sleep(delayMs);
+    }
+  }
+
+  return exitCode;
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === resolvePath(process.argv[1])) {
+  const args = process.argv.slice(2);
+  if (args.length === 0) {
+    console.error("Usage: node scripts/npm-retry.mjs <npm arguments...>");
+    process.exitCode = 2;
+  } else {
+    process.exitCode = await runWithRetry(args);
+  }
+}