From 3cc4ae28664493b2f1ad75a6a6ed1d318c76e944 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Thu, 17 Sep 2026 22:50:13 +0200 Subject: [PATCH] ci(desktop): retry release asset uploads uploads.github.com returned HTTP 500 five times during the v0.9.0-beta.1 release, each costing a ~50 minute rebuild. The upload step passed every artifact to one gh invocation, so a single failed asset discarded the rest and failed the job. Upload each asset separately with bounded retries, shared by the macOS, Linux and Windows jobs. --- .github/workflows/ci.yml | 2 +- .github/workflows/desktop-release.yml | 30 +-------- scripts/upload-release-assets.mjs | 88 ++++++++++++++++++++++++++ scripts/upload-release-assets.test.mjs | 54 ++++++++++++++++ 4 files changed, 146 insertions(+), 28 deletions(-) create mode 100644 scripts/upload-release-assets.mjs create mode 100644 scripts/upload-release-assets.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f8d542370..3a1e8ed73c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: filters: .github/ci-paths.yml - name: Validate CI contracts - run: node --test scripts/ci-workflow.test.mjs scripts/daemon-launch-contract.test.mjs scripts/sync-fdroid-changelogs.test.mjs + run: node --test scripts/ci-workflow.test.mjs scripts/daemon-launch-contract.test.mjs scripts/sync-fdroid-changelogs.test.mjs scripts/upload-release-assets.test.mjs format: name: format diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 393cbc27e2..f6b07c8b37 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -191,16 +191,8 @@ jobs: run: | set -euo pipefail release_dir="${DESKTOP_PACKAGE_PATH}/release" - files=() - while IFS= read -r -d '' f; do - files+=("$f") - done < <(find "$release_dir" -maxdepth 1 -type f ! -name '*.yml' -print0 | sort -z) - if (( ${#files[@]} == 0 )); then - echo "::error::No release artifacts found in $release_dir" - exit 1 - fi release_lookup="$(node scripts/github-release.mjs --repo "${{ github.repository }}" --tag "$RELEASE_TAG")" - gh release upload "$release_lookup" "${files[@]}" --clobber --repo "${{ github.repository }}" + node scripts/upload-release-assets.mjs "$release_lookup" "$release_dir" "${{ github.repository }}" - name: Upload desktop artifacts to workflow if: env.SHOULD_PUBLISH != 'true' @@ -300,16 +292,8 @@ jobs: run: | set -euo pipefail release_dir="${DESKTOP_PACKAGE_PATH}/release" - files=() - while IFS= read -r -d '' f; do - files+=("$f") - done < <(find "$release_dir" -maxdepth 1 -type f ! -name '*.yml' -print0 | sort -z) - if (( ${#files[@]} == 0 )); then - echo "::error::No release artifacts found in $release_dir" - exit 1 - fi release_lookup="$(node scripts/github-release.mjs --repo "${{ github.repository }}" --tag "$RELEASE_TAG")" - gh release upload "$release_lookup" "${files[@]}" --clobber --repo "${{ github.repository }}" + node scripts/upload-release-assets.mjs "$release_lookup" "$release_dir" "${{ github.repository }}" - name: Upload desktop artifacts to workflow if: env.SHOULD_PUBLISH != 'true' @@ -406,16 +390,8 @@ jobs: run: | set -euo pipefail release_dir="${DESKTOP_PACKAGE_PATH}/release" - files=() - while IFS= read -r -d '' f; do - files+=("$f") - done < <(find "$release_dir" -maxdepth 1 -type f ! -name '*.yml' -print0 | sort -z) - if (( ${#files[@]} == 0 )); then - echo "::error::No release artifacts found in $release_dir" - exit 1 - fi release_lookup="$(node scripts/github-release.mjs --repo "${{ github.repository }}" --tag "$RELEASE_TAG")" - gh release upload "$release_lookup" "${files[@]}" --clobber --repo "${{ github.repository }}" + node scripts/upload-release-assets.mjs "$release_lookup" "$release_dir" "${{ github.repository }}" - name: Upload desktop artifacts to workflow if: env.SHOULD_PUBLISH != 'true' diff --git a/scripts/upload-release-assets.mjs b/scripts/upload-release-assets.mjs new file mode 100644 index 0000000000..d3393968ef --- /dev/null +++ b/scripts/upload-release-assets.mjs @@ -0,0 +1,88 @@ +import { spawnSync } from "node:child_process"; +import { readdirSync, statSync } from "node:fs"; +import path from "node:path"; + +import { isMainModule } from "./is-main-module.mjs"; + +// uploads.github.com returns HTTP 500 on large desktop binaries often enough that a +// release regularly loses a ~50 minute build to one failed asset. Upload each file on +// its own so a failure does not discard its siblings, and retry that file. +const DEFAULT_ATTEMPTS = 5; +const DEFAULT_BACKOFF_MS = 15_000; + +function uploadAsset({ release, file, repo }) { + const command = process.platform === "win32" ? "gh.exe" : "gh"; + const result = spawnSync( + command, + ["release", "upload", release, file, "--clobber", "--repo", repo], + { shell: process.platform === "win32", stdio: "inherit" }, + ); + + if (result.error) { + console.error(`Failed to start gh: ${result.error.message}`); + return 1; + } + + return result.status ?? 1; +} + +function wait(delayMs) { + return new Promise((resolve) => setTimeout(resolve, delayMs)); +} + +export function collectAssets(releaseDir, { readDir = readdirSync, stat = statSync } = {}) { + return readDir(releaseDir) + .map((name) => path.join(releaseDir, name)) + .filter((file) => stat(file).isFile() && !file.endsWith(".yml")) + .sort(); +} + +export async function uploadWithRetry( + { release, files, repo }, + { + attempts = DEFAULT_ATTEMPTS, + backoffMs = DEFAULT_BACKOFF_MS, + upload = uploadAsset, + sleep = wait, + } = {}, +) { + for (const file of files) { + let exitCode = 1; + + for (let attempt = 1; attempt <= attempts; attempt += 1) { + exitCode = await upload({ release, file, repo }); + if (exitCode === 0) break; + + if (attempt < attempts) { + const delayMs = attempt * backoffMs; + console.warn( + `Uploading ${path.basename(file)} failed with exit code ${exitCode}; retrying in ${delayMs / 1000}s (${attempt + 1}/${attempts})`, + ); + await sleep(delayMs); + } + } + + if (exitCode !== 0) { + console.error(`::error::Giving up on ${path.basename(file)} after ${attempts} attempts`); + return exitCode; + } + } + + return 0; +} + +if (isMainModule(import.meta.url)) { + const [release, releaseDir, repo] = process.argv.slice(2); + if (!release || !releaseDir || !repo) { + console.error("Usage: node scripts/upload-release-assets.mjs "); + process.exitCode = 2; + } else { + const files = collectAssets(releaseDir); + if (files.length === 0) { + console.error(`::error::No release artifacts found in ${releaseDir}`); + process.exitCode = 1; + } else { + process.exitCode = await uploadWithRetry({ release, files, repo }); + } + } +} diff --git a/scripts/upload-release-assets.test.mjs b/scripts/upload-release-assets.test.mjs new file mode 100644 index 0000000000..6f6deaabaf --- /dev/null +++ b/scripts/upload-release-assets.test.mjs @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { collectAssets, uploadWithRetry } from "./upload-release-assets.mjs"; + +const noSleep = () => Promise.resolve(); + +test("retries a failing asset and keeps its siblings", async () => { + const calls = []; + const exitCodes = { "a.dmg": [1, 1, 0], "b.zip": [0] }; + + const exitCode = await uploadWithRetry( + { release: "rel", files: ["a.dmg", "b.zip"], repo: "getpaseo/paseo" }, + { + sleep: noSleep, + upload: ({ file }) => { + calls.push(file); + return exitCodes[file].shift(); + }, + }, + ); + + assert.equal(exitCode, 0); + assert.deepEqual(calls, ["a.dmg", "a.dmg", "a.dmg", "b.zip"]); +}); + +test("gives up after the attempt limit and reports failure", async () => { + let calls = 0; + + const exitCode = await uploadWithRetry( + { release: "rel", files: ["a.dmg", "b.zip"], repo: "getpaseo/paseo" }, + { + attempts: 3, + sleep: noSleep, + upload: () => { + calls += 1; + return 1; + }, + }, + ); + + assert.equal(exitCode, 1); + // Stops at the first unrecoverable file rather than attempting b.zip. + assert.equal(calls, 3); +}); + +test("collects release files and skips manifests", () => { + const files = collectAssets("/release", { + readDir: () => ["Paseo.dmg", "beta-mac.yml", "Paseo.zip", "nested"], + stat: (file) => ({ isFile: () => !file.endsWith("nested") }), + }); + + assert.deepEqual(files, ["/release/Paseo.dmg", "/release/Paseo.zip"]); +});