mirror of
https://github.com/community-scripts/ProxmoxVE.git
synced 2026-10-10 21:02:09 +02:00
Immich: survive an interrupted update and a failing ML build (#17798)
* Immich: survive an interrupted update and a failing ML build An update cancelled midway leaves /opt/immich/app empty, and the next run died at once on cd /opt/immich/app/bin to enable maintenance mode (#17796). Maintenance mode is now only toggled when immich-admin exists. uv sync fetching the ml-models git dependency failed with "Could not resolve host" behind DNS filters such as AdGuard while DNS itself worked (#17797); running uv one download at a time got through. Retries now do that. If machine learning still cannot be built, the update no longer stops with maintenance mode on and every service down: it finishes, starts the web service, puts the previous version back into ~/.immich so the next update retries, and exits with an error. Updates are also announced as taking 5-15 minutes, since the first report came from cancelling one that looked stuck. * Immich: point immich-ml at the moved ml_start.sh and clear failed units The update moves ml_start.sh into app/machine-learning, but only rewrote immich-web's ExecStart, so older containers kept starting ML from /opt/immich/ml_start.sh and failed with 203/EXEC. Rewrite immich-ml the same way. A crash loop before the update can also leave both units rate-limited, which makes the final restart fail; reset them first.
This commit is contained in:
CanbiZ (MickLesk)
committed by
GitHub
GitHub
parent
d271571ff7
commit
227526cf55
+31
-10
@@ -115,7 +115,10 @@ EOF
|
||||
fi
|
||||
|
||||
if check_for_gh_release "Immich" "immich-app/immich" "${RELEASE}" "each release is tested individually before the version is updated. Please do not open issues for this"; then
|
||||
if [[ $(cat ~/.immich) > "2.5.1" ]]; then
|
||||
msg_warn "The update takes 5-15 minutes, do not interrupt it"
|
||||
PREV_IMMICH="$(cat ~/.immich)"
|
||||
# An interrupted update leaves app/ empty, so there may be no immich-admin to call.
|
||||
if [[ "$PREV_IMMICH" > "2.5.1" && -x /opt/immich/app/bin/immich-admin ]]; then
|
||||
msg_info "Enabling Maintenance Mode"
|
||||
cd /opt/immich/app/bin
|
||||
$STD ./immich-admin enable-maintenance-mode || true
|
||||
@@ -273,12 +276,14 @@ EOF
|
||||
msg_ok "Pre-installed Python ${ML_PYTHON}"
|
||||
msg_info "Updating Intel OpenVINO machine-learning"
|
||||
for attempt in $(seq 1 3); do
|
||||
$STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT -Pnu immich uv sync --extra openvino --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
|
||||
[[ $attempt -eq 3 ]] && { msg_error "uv sync failed three times, the machine-learning environment was not built"; exit 1; }
|
||||
msg_warn "uv sync attempt $attempt failed, retrying..." && sleep 10
|
||||
$STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT,UV_CONCURRENT_DOWNLOADS,UV_CONCURRENT_BUILDS,UV_CONCURRENT_INSTALLS -Pnu immich uv sync --extra openvino --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
|
||||
[[ $attempt -eq 3 ]] && { ML_FAILED=1; break; }
|
||||
# Parallel fetches can trip DNS rate limits (AdGuard, Pi-hole); retry one at a time.
|
||||
export UV_CONCURRENT_DOWNLOADS=1 UV_CONCURRENT_BUILDS=1 UV_CONCURRENT_INSTALLS=1
|
||||
msg_warn "uv sync attempt $attempt failed, retrying one download at a time..." && sleep 10
|
||||
done
|
||||
patchelf --clear-execstack "${VIRTUAL_ENV}/lib/python3.13/site-packages/onnxruntime/capi/onnxruntime_pybind11_state.cpython-313-$(arch_resolve "x86_64" "aarch64")-linux-gnu.so"
|
||||
msg_ok "Updated Intel OpenVINO machine-learning"
|
||||
[[ "${ML_FAILED:-0}" == 1 ]] || patchelf --clear-execstack "${VIRTUAL_ENV}/lib/python3.13/site-packages/onnxruntime/capi/onnxruntime_pybind11_state.cpython-313-$(arch_resolve "x86_64" "aarch64")-linux-gnu.so"
|
||||
[[ "${ML_FAILED:-0}" == 1 ]] || msg_ok "Updated Intel OpenVINO machine-learning"
|
||||
else
|
||||
ML_PYTHON="python3.13"
|
||||
msg_info "Pre-installing Python ${ML_PYTHON} for machine-learning"
|
||||
@@ -289,12 +294,15 @@ EOF
|
||||
msg_ok "Pre-installed Python ${ML_PYTHON}"
|
||||
msg_info "Updating machine-learning"
|
||||
for attempt in $(seq 1 3); do
|
||||
$STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT -Pnu immich uv sync --extra cpu --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
|
||||
[[ $attempt -eq 3 ]] && { msg_error "uv sync failed three times, the machine-learning environment was not built"; exit 1; }
|
||||
msg_warn "uv sync attempt $attempt failed, retrying..." && sleep 10
|
||||
$STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT,UV_CONCURRENT_DOWNLOADS,UV_CONCURRENT_BUILDS,UV_CONCURRENT_INSTALLS -Pnu immich uv sync --extra cpu --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
|
||||
[[ $attempt -eq 3 ]] && { ML_FAILED=1; break; }
|
||||
# Parallel fetches can trip DNS rate limits (AdGuard, Pi-hole); retry one at a time.
|
||||
export UV_CONCURRENT_DOWNLOADS=1 UV_CONCURRENT_BUILDS=1 UV_CONCURRENT_INSTALLS=1
|
||||
msg_warn "uv sync attempt $attempt failed, retrying one download at a time..." && sleep 10
|
||||
done
|
||||
msg_ok "Updated machine-learning"
|
||||
[[ "${ML_FAILED:-0}" == 1 ]] || msg_ok "Updated machine-learning"
|
||||
fi
|
||||
[[ "${ML_FAILED:-0}" == 1 ]] && msg_warn "uv sync failed three times, machine learning was not updated"
|
||||
cd "$SRC_DIR"
|
||||
cp -a machine-learning/{ann,immich_ml} "$ML_DIR"
|
||||
[[ -f "$INSTALL_DIR"/ml_start.sh ]] && mv "$INSTALL_DIR"/ml_start.sh "$ML_DIR"
|
||||
@@ -345,6 +353,10 @@ EOF
|
||||
sed -i "s|^ExecStart=.*|ExecStart=${APP_DIR}/bin/start.sh|" /etc/systemd/system/immich-web.service
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
if grep -q "^ExecStart=${INSTALL_DIR}/ml_start.sh" /etc/systemd/system/immich-ml.service; then
|
||||
sed -i "s|^ExecStart=.*|ExecStart=${ML_DIR}/ml_start.sh|" /etc/systemd/system/immich-ml.service
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
|
||||
# MickLesk temporary patch for HEIC thumbnail gen
|
||||
MEDIA_REPO_JS="/opt/immich/app/dist/repositories/media.repository.js"
|
||||
@@ -366,6 +378,15 @@ EOF
|
||||
$STD cd -
|
||||
msg_ok "Disabled Maintenance Mode"
|
||||
fi
|
||||
# A crash loop before the update leaves the units rate-limited, and restart would refuse.
|
||||
systemctl reset-failed immich-ml immich-web 2>/dev/null || true
|
||||
if [[ "${ML_FAILED:-0}" == 1 ]]; then
|
||||
systemctl restart immich-web || true
|
||||
[[ -f /etc/systemd/system/immich-proxy.service ]] && systemctl restart immich-proxy
|
||||
echo "$PREV_IMMICH" >~/.immich
|
||||
msg_error "Immich runs, but without machine learning (see the uv output above). Run update again to retry."
|
||||
exit 1
|
||||
fi
|
||||
systemctl restart immich-ml immich-web
|
||||
[[ -f /etc/systemd/system/immich-proxy.service ]] && systemctl restart immich-proxy
|
||||
msg_ok "Updated successfully!"
|
||||
|
||||
@@ -441,9 +441,11 @@ if [[ -f ~/.openvino ]]; then
|
||||
msg_ok "Pre-installed Python ${ML_PYTHON}"
|
||||
msg_info "Installing Intel OpenVINO machine-learning"
|
||||
for attempt in $(seq 1 3); do
|
||||
$STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT -Pnu immich uv sync --extra openvino --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
|
||||
$STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT,UV_CONCURRENT_DOWNLOADS,UV_CONCURRENT_BUILDS,UV_CONCURRENT_INSTALLS -Pnu immich uv sync --extra openvino --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
|
||||
[[ $attempt -eq 3 ]] && { msg_error "uv sync failed three times, the machine-learning environment was not built"; exit 1; }
|
||||
msg_warn "uv sync attempt $attempt failed, retrying..." && sleep 10
|
||||
# Parallel fetches can trip DNS rate limits (AdGuard, Pi-hole); retry one at a time.
|
||||
export UV_CONCURRENT_DOWNLOADS=1 UV_CONCURRENT_BUILDS=1 UV_CONCURRENT_INSTALLS=1
|
||||
msg_warn "uv sync attempt $attempt failed, retrying one download at a time..." && sleep 10
|
||||
done
|
||||
patchelf --clear-execstack "${VIRTUAL_ENV}/lib/python3.13/site-packages/onnxruntime/capi/onnxruntime_pybind11_state.cpython-313-$(arch_resolve "x86_64" "aarch64")-linux-gnu.so"
|
||||
msg_ok "Installed Intel OpenVINO machine-learning"
|
||||
@@ -457,9 +459,11 @@ else
|
||||
msg_ok "Pre-installed Python ${ML_PYTHON}"
|
||||
msg_info "Installing machine-learning"
|
||||
for attempt in $(seq 1 3); do
|
||||
$STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT -Pnu immich uv sync --extra cpu --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
|
||||
$STD sudo --preserve-env=VIRTUAL_ENV,UV_HTTP_TIMEOUT,UV_CONCURRENT_DOWNLOADS,UV_CONCURRENT_BUILDS,UV_CONCURRENT_INSTALLS -Pnu immich uv sync --extra cpu --no-dev --active --link-mode copy -n -p "${ML_PYTHON}" --managed-python && break
|
||||
[[ $attempt -eq 3 ]] && { msg_error "uv sync failed three times, the machine-learning environment was not built"; exit 1; }
|
||||
msg_warn "uv sync attempt $attempt failed, retrying..." && sleep 10
|
||||
# Parallel fetches can trip DNS rate limits (AdGuard, Pi-hole); retry one at a time.
|
||||
export UV_CONCURRENT_DOWNLOADS=1 UV_CONCURRENT_BUILDS=1 UV_CONCURRENT_INSTALLS=1
|
||||
msg_warn "uv sync attempt $attempt failed, retrying one download at a time..." && sleep 10
|
||||
done
|
||||
msg_ok "Installed machine-learning"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user