Compare commits

..
Author SHA1 Message Date
Brent Vatne 9a436e8f72 Workflows: the launch decoder install retries a stalled apt-get and fails in minutes, not at the job's timeout
A package mirror stalled `apt-get update` on two of euxy's baseline runners for 19 minutes, with nothing printed, while their capture jobs waited (brentvatne/euxy#179). Each apt-get call now has network timeouts, a two-minute limit and a lock timeout. A stalled attempt is tried once more, then the step fails with LAUNCH_DECODER_INSTALL_FAILED, and the step itself stops after ten minutes.
2026-10-23 15:17:04 -07:00
3 changed files with 36 additions and 6 deletions

No files matched your search

+12 -2
View File
@@ -85,10 +85,20 @@ jobs:
- *install-explore
- name: Install the launch recording decoder
shell: bash
# Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it.
# Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it. A package
# mirror can stall without failing, so each apt-get call has network timeouts and a time limit, and a stalled
# attempt is tried once more: the step fails in minutes rather than holding the job to its timeout.
timeout-minutes: 10
run: |
set -euo pipefail
command -v ffmpeg > /dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg > /dev/null; }
if ! command -v ffmpeg > /dev/null; then
apt=(sudo apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120)
for attempt in 1 2; do
if timeout 120 "${apt[@]}" update && timeout 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi
if [ "$attempt" = 2 ]; then echo '::error title=LAUNCH_DECODER_INSTALL_FAILED::apt-get could not install ffmpeg; rerun the job.'; exit 1; fi
echo "::warning title=LAUNCH_DECODER_INSTALL_RETRY::apt-get did not install ffmpeg on attempt $attempt; trying again."
done
fi
ffmpeg -version | head -1
- name: Verify deterministic source planning
shell: bash
+12 -2
View File
@@ -122,10 +122,20 @@ jobs:
- *install-explore
- name: Install the launch recording decoder
shell: bash
# Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it.
# Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it. A package
# mirror can stall without failing, so each apt-get call has network timeouts and a time limit, and a stalled
# attempt is tried once more: the step fails in minutes rather than holding the job to its timeout.
timeout-minutes: 10
run: |
set -euo pipefail
command -v ffmpeg > /dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg > /dev/null; }
if ! command -v ffmpeg > /dev/null; then
apt=(sudo apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120)
for attempt in 1 2; do
if timeout 120 "${apt[@]}" update && timeout 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi
if [ "$attempt" = 2 ]; then echo '::error title=LAUNCH_DECODER_INSTALL_FAILED::apt-get could not install ffmpeg; rerun the job.'; exit 1; fi
echo "::warning title=LAUNCH_DECODER_INSTALL_RETRY::apt-get did not install ffmpeg on attempt $attempt; trying again."
done
fi
ffmpeg -version | head -1
- name: Verify deterministic source planning
env:
+12 -2
View File
@@ -111,10 +111,20 @@ jobs:
with: { ref: '${{ steps.pr.outputs.head }}', path: app-head, fetch-depth: 0, persist-credentials: false }
- name: Install the launch recording decoder
shell: bash
# Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it.
# Launch recordings (capture.launch) are decoded with ffmpeg; GitHub's Ubuntu runners do not ship it. A package
# mirror can stall without failing, so each apt-get call has network timeouts and a time limit, and a stalled
# attempt is tried once more: the step fails in minutes rather than holding the job to its timeout.
timeout-minutes: 10
run: |
set -euo pipefail
command -v ffmpeg > /dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg > /dev/null; }
if ! command -v ffmpeg > /dev/null; then
apt=(sudo apt-get -qq -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 -o DPkg::Lock::Timeout=120)
for attempt in 1 2; do
if timeout 120 "${apt[@]}" update && timeout 120 "${apt[@]}" install -y --no-install-recommends ffmpeg > /dev/null; then break; fi
if [ "$attempt" = 2 ]; then echo '::error title=LAUNCH_DECODER_INSTALL_FAILED::apt-get could not install ffmpeg; rerun the job.'; exit 1; fi
echo "::warning title=LAUNCH_DECODER_INSTALL_RETRY::apt-get did not install ffmpeg on attempt $attempt; trying again."
done
fi
ffmpeg -version | head -1
- name: Verify deterministic source planning
env: