ci(desktop): retry release asset uploads

uploads.github.com returned HTTP 500 five times during the v0.9.0-beta.1
release, each costing a ~50 minute rebuild. The upload step passed every
artifact to one gh invocation, so a single failed asset discarded the rest
and failed the job.

Upload each asset separately with bounded retries, shared by the macOS,
Linux and Windows jobs.

.github/workflows/ci.yml

@@ -52,7 +52,7 @@ jobs:
           filters: .github/ci-paths.yml
 
       - name: Validate CI contracts
-        run: node --test scripts/ci-workflow.test.mjs scripts/daemon-launch-contract.test.mjs scripts/sync-fdroid-changelogs.test.mjs
+        run: node --test scripts/ci-workflow.test.mjs scripts/daemon-launch-contract.test.mjs scripts/sync-fdroid-changelogs.test.mjs scripts/upload-release-assets.test.mjs
 
   format:
     name: format

.github/workflows/desktop-release.yml

@@ -191,16 +191,8 @@ jobs:
         run: |
           set -euo pipefail
           release_dir="${DESKTOP_PACKAGE_PATH}/release"
-          files=()
-          while IFS= read -r -d '' f; do
-            files+=("$f")
-          done < <(find "$release_dir" -maxdepth 1 -type f ! -name '*.yml' -print0 | sort -z)
-          if (( ${#files[@]} == 0 )); then
-            echo "::error::No release artifacts found in $release_dir"
-            exit 1
-          fi
           release_lookup="$(node scripts/github-release.mjs --repo "${{ github.repository }}" --tag "$RELEASE_TAG")"
-          gh release upload "$release_lookup" "${files[@]}" --clobber --repo "${{ github.repository }}"
+          node scripts/upload-release-assets.mjs "$release_lookup" "$release_dir" "${{ github.repository }}"
 
       - name: Upload desktop artifacts to workflow
         if: env.SHOULD_PUBLISH != 'true'
@@ -300,16 +292,8 @@ jobs:
         run: |
           set -euo pipefail
           release_dir="${DESKTOP_PACKAGE_PATH}/release"
-          files=()
-          while IFS= read -r -d '' f; do
-            files+=("$f")
-          done < <(find "$release_dir" -maxdepth 1 -type f ! -name '*.yml' -print0 | sort -z)
-          if (( ${#files[@]} == 0 )); then
-            echo "::error::No release artifacts found in $release_dir"
-            exit 1
-          fi
           release_lookup="$(node scripts/github-release.mjs --repo "${{ github.repository }}" --tag "$RELEASE_TAG")"
-          gh release upload "$release_lookup" "${files[@]}" --clobber --repo "${{ github.repository }}"
+          node scripts/upload-release-assets.mjs "$release_lookup" "$release_dir" "${{ github.repository }}"
 
       - name: Upload desktop artifacts to workflow
         if: env.SHOULD_PUBLISH != 'true'
@@ -406,16 +390,8 @@ jobs:
         run: |
           set -euo pipefail
           release_dir="${DESKTOP_PACKAGE_PATH}/release"
-          files=()
-          while IFS= read -r -d '' f; do
-            files+=("$f")
-          done < <(find "$release_dir" -maxdepth 1 -type f ! -name '*.yml' -print0 | sort -z)
-          if (( ${#files[@]} == 0 )); then
-            echo "::error::No release artifacts found in $release_dir"
-            exit 1
-          fi
           release_lookup="$(node scripts/github-release.mjs --repo "${{ github.repository }}" --tag "$RELEASE_TAG")"
-          gh release upload "$release_lookup" "${files[@]}" --clobber --repo "${{ github.repository }}"
+          node scripts/upload-release-assets.mjs "$release_lookup" "$release_dir" "${{ github.repository }}"
 
       - name: Upload desktop artifacts to workflow
         if: env.SHOULD_PUBLISH != 'true'

scripts/upload-release-assets.mjs

@@ -0,0 +1,88 @@
+import { spawnSync } from "node:child_process";
+import { readdirSync, statSync } from "node:fs";
+import path from "node:path";
+
+import { isMainModule } from "./is-main-module.mjs";
+
+// uploads.github.com returns HTTP 500 on large desktop binaries often enough that a
+// release regularly loses a ~50 minute build to one failed asset. Upload each file on
+// its own so a failure does not discard its siblings, and retry that file.
+const DEFAULT_ATTEMPTS = 5;
+const DEFAULT_BACKOFF_MS = 15_000;
+
+function uploadAsset({ release, file, repo }) {
+  const command = process.platform === "win32" ? "gh.exe" : "gh";
+  const result = spawnSync(
+    command,
+    ["release", "upload", release, file, "--clobber", "--repo", repo],
+    { shell: process.platform === "win32", stdio: "inherit" },
+  );
+
+  if (result.error) {
+    console.error(`Failed to start gh: ${result.error.message}`);
+    return 1;
+  }
+
+  return result.status ?? 1;
+}
+
+function wait(delayMs) {
+  return new Promise((resolve) => setTimeout(resolve, delayMs));
+}
+
+export function collectAssets(releaseDir, { readDir = readdirSync, stat = statSync } = {}) {
+  return readDir(releaseDir)
+    .map((name) => path.join(releaseDir, name))
+    .filter((file) => stat(file).isFile() && !file.endsWith(".yml"))
+    .sort();
+}
+
+export async function uploadWithRetry(
+  { release, files, repo },
+  {
+    attempts = DEFAULT_ATTEMPTS,
+    backoffMs = DEFAULT_BACKOFF_MS,
+    upload = uploadAsset,
+    sleep = wait,
+  } = {},
+) {
+  for (const file of files) {
+    let exitCode = 1;
+
+    for (let attempt = 1; attempt <= attempts; attempt += 1) {
+      exitCode = await upload({ release, file, repo });
+      if (exitCode === 0) break;
+
+      if (attempt < attempts) {
+        const delayMs = attempt * backoffMs;
+        console.warn(
+          `Uploading ${path.basename(file)} failed with exit code ${exitCode}; retrying in ${delayMs / 1000}s (${attempt + 1}/${attempts})`,
+        );
+        await sleep(delayMs);
+      }
+    }
+
+    if (exitCode !== 0) {
+      console.error(`::error::Giving up on ${path.basename(file)} after ${attempts} attempts`);
+      return exitCode;
+    }
+  }
+
+  return 0;
+}
+
+if (isMainModule(import.meta.url)) {
+  const [release, releaseDir, repo] = process.argv.slice(2);
+  if (!release || !releaseDir || !repo) {
+    console.error("Usage: node scripts/upload-release-assets.mjs <release> <dir> <repo>");
+    process.exitCode = 2;
+  } else {
+    const files = collectAssets(releaseDir);
+    if (files.length === 0) {
+      console.error(`::error::No release artifacts found in ${releaseDir}`);
+      process.exitCode = 1;
+    } else {
+      process.exitCode = await uploadWithRetry({ release, files, repo });
+    }
+  }
+}

scripts/upload-release-assets.test.mjs

@@ -0,0 +1,54 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { collectAssets, uploadWithRetry } from "./upload-release-assets.mjs";
+
+const noSleep = () => Promise.resolve();
+
+test("retries a failing asset and keeps its siblings", async () => {
+  const calls = [];
+  const exitCodes = { "a.dmg": [1, 1, 0], "b.zip": [0] };
+
+  const exitCode = await uploadWithRetry(
+    { release: "rel", files: ["a.dmg", "b.zip"], repo: "getpaseo/paseo" },
+    {
+      sleep: noSleep,
+      upload: ({ file }) => {
+        calls.push(file);
+        return exitCodes[file].shift();
+      },
+    },
+  );
+
+  assert.equal(exitCode, 0);
+  assert.deepEqual(calls, ["a.dmg", "a.dmg", "a.dmg", "b.zip"]);
+});
+
+test("gives up after the attempt limit and reports failure", async () => {
+  let calls = 0;
+
+  const exitCode = await uploadWithRetry(
+    { release: "rel", files: ["a.dmg", "b.zip"], repo: "getpaseo/paseo" },
+    {
+      attempts: 3,
+      sleep: noSleep,
+      upload: () => {
+        calls += 1;
+        return 1;
+      },
+    },
+  );
+
+  assert.equal(exitCode, 1);
+  // Stops at the first unrecoverable file rather than attempting b.zip.
+  assert.equal(calls, 3);
+});
+
+test("collects release files and skips manifests", () => {
+  const files = collectAssets("/release", {
+    readDir: () => ["Paseo.dmg", "beta-mac.yml", "Paseo.zip", "nested"],
+    stat: (file) => ({ isFile: () => !file.endsWith("nested") }),
+  });
+
+  assert.deepEqual(files, ["/release/Paseo.dmg", "/release/Paseo.zip"]);
+});