Compare commits

...

1 Commits

Author SHA1 Message Date
CanbiZ (MickLesk)
15779e4ee0 Retry reporting with fallback payloads
Enhance post_update_to_api to support a "force" mode and robust retry logic: add a 3rd-arg bypass to duplicate suppression, capture a short error summary, and perform up to three POST attempts (full payload, shortened error payload, minimal payload) with HTTP code checks and small backoffs. Mark POST_UPDATE_DONE on success (or after three attempts) to avoid infinite retries. Also invoke post_update_to_api with the "force" flag from cleanup paths in build.func and error_handler.func so a final status update is attempted after cleanup.
2026-02-13 13:45:31 +01:00
3 changed files with 91 additions and 6 deletions

View File

@@ -592,9 +592,12 @@ post_update_to_api() {
# Silent fail - telemetry should never break scripts # Silent fail - telemetry should never break scripts
command -v curl &>/dev/null || return 0 command -v curl &>/dev/null || return 0
# Prevent duplicate submissions # Support "force" mode (3rd arg) to bypass duplicate check for retries after cleanup
local force="${3:-}"
POST_UPDATE_DONE=${POST_UPDATE_DONE:-false} POST_UPDATE_DONE=${POST_UPDATE_DONE:-false}
[[ "$POST_UPDATE_DONE" == "true" ]] && return 0 if [[ "$POST_UPDATE_DONE" == "true" && "$force" != "force" ]]; then
return 0
fi
[[ "${DIAGNOSTICS:-no}" == "no" ]] && return 0 [[ "${DIAGNOSTICS:-no}" == "no" ]] && return 0
[[ -z "${RANDOM_UUID:-}" ]] && return 0 [[ -z "${RANDOM_UUID:-}" ]] && return 0
@@ -632,6 +635,7 @@ post_update_to_api() {
esac esac
# For failed/unknown status, resolve exit code and error description # For failed/unknown status, resolve exit code and error description
local short_error=""
if [[ "$pb_status" == "failed" ]] || [[ "$pb_status" == "unknown" ]]; then if [[ "$pb_status" == "failed" ]] || [[ "$pb_status" == "unknown" ]]; then
if [[ "$raw_exit_code" =~ ^[0-9]+$ ]]; then if [[ "$raw_exit_code" =~ ^[0-9]+$ ]]; then
exit_code="$raw_exit_code" exit_code="$raw_exit_code"
@@ -645,6 +649,7 @@ post_update_to_api() {
else else
error=$(json_escape "$(explain_exit_code "$exit_code")") error=$(json_escape "$(explain_exit_code "$exit_code")")
fi fi
short_error=$(json_escape "$(explain_exit_code "$exit_code")")
error_category=$(categorize_error "$exit_code") error_category=$(categorize_error "$exit_code")
[[ -z "$error" ]] && error="Unknown error" [[ -z "$error" ]] && error="Unknown error"
fi fi
@@ -661,8 +666,9 @@ post_update_to_api() {
pve_version=$(pveversion 2>/dev/null | awk -F'[/ ]' '{print $2}') || true pve_version=$(pveversion 2>/dev/null | awk -F'[/ ]' '{print $2}') || true
fi fi
# Full payload including all fields - allows record creation if initial call failed local http_code=""
# The Go service will find the record by random_id and PATCH, or create if not found
# ── Attempt 1: Full payload with complete error text ──
local JSON_PAYLOAD local JSON_PAYLOAD
JSON_PAYLOAD=$( JSON_PAYLOAD=$(
cat <<EOF cat <<EOF
@@ -694,11 +700,80 @@ post_update_to_api() {
EOF EOF
) )
# Fire-and-forget: never block, never fail http_code=$(curl -sS -w "%{http_code}" -m "${TELEMETRY_TIMEOUT}" -X POST "${TELEMETRY_URL}" \
-H "Content-Type: application/json" \
-d "$JSON_PAYLOAD" -o /dev/null 2>/dev/null) || http_code="000"
if [[ "$http_code" =~ ^2[0-9]{2}$ ]]; then
POST_UPDATE_DONE=true
return 0
fi
# ── Attempt 2: Short error text (no full log) ──
sleep 1
local RETRY_PAYLOAD
RETRY_PAYLOAD=$(
cat <<EOF
{
"random_id": "${RANDOM_UUID}",
"type": "${TELEMETRY_TYPE:-lxc}",
"nsapp": "${NSAPP:-unknown}",
"status": "${pb_status}",
"ct_type": ${CT_TYPE:-1},
"disk_size": ${DISK_SIZE:-0},
"core_count": ${CORE_COUNT:-0},
"ram_size": ${RAM_SIZE:-0},
"os_type": "${var_os:-}",
"os_version": "${var_version:-}",
"pve_version": "${pve_version}",
"method": "${METHOD:-default}",
"exit_code": ${exit_code},
"error": "${short_error}",
"error_category": "${error_category}",
"install_duration": ${duration},
"cpu_vendor": "${cpu_vendor}",
"cpu_model": "${cpu_model}",
"gpu_vendor": "${gpu_vendor}",
"gpu_model": "${gpu_model}",
"gpu_passthrough": "${gpu_passthrough}",
"ram_speed": "${ram_speed}",
"repo_source": "${REPO_SOURCE}"
}
EOF
)
http_code=$(curl -sS -w "%{http_code}" -m "${TELEMETRY_TIMEOUT}" -X POST "${TELEMETRY_URL}" \
-H "Content-Type: application/json" \
-d "$RETRY_PAYLOAD" -o /dev/null 2>/dev/null) || http_code="000"
if [[ "$http_code" =~ ^2[0-9]{2}$ ]]; then
POST_UPDATE_DONE=true
return 0
fi
# ── Attempt 3: Minimal payload (bare minimum to set status) ──
sleep 2
local MINIMAL_PAYLOAD
MINIMAL_PAYLOAD=$(
cat <<EOF
{
"random_id": "${RANDOM_UUID}",
"type": "${TELEMETRY_TYPE:-lxc}",
"nsapp": "${NSAPP:-unknown}",
"status": "${pb_status}",
"exit_code": ${exit_code},
"error": "${short_error}",
"error_category": "${error_category}",
"install_duration": ${duration}
}
EOF
)
curl -sS -w "%{http_code}" -m "${TELEMETRY_TIMEOUT}" -X POST "${TELEMETRY_URL}" \ curl -sS -w "%{http_code}" -m "${TELEMETRY_TIMEOUT}" -X POST "${TELEMETRY_URL}" \
-H "Content-Type: application/json" \ -H "Content-Type: application/json" \
-d "$JSON_PAYLOAD" -o /dev/null 2>&1 || true -d "$MINIMAL_PAYLOAD" -o /dev/null 2>/dev/null || true
# Tried 3 times - mark as done regardless to prevent infinite loops
POST_UPDATE_DONE=true POST_UPDATE_DONE=true
} }

View File

@@ -4130,6 +4130,10 @@ EOF'
echo -e "${BFR}${CM}${GN}Container ${CTID} removed${CL}" echo -e "${BFR}${CM}${GN}Container ${CTID} removed${CL}"
fi fi
# Force one final status update attempt after cleanup
# This ensures status is updated even if the first attempt failed (e.g., HTTP 400)
post_update_to_api "failed" "$install_exit_code" "force"
exit $install_exit_code exit $install_exit_code
fi fi
} }

View File

@@ -222,6 +222,12 @@ error_handler() {
pct destroy "$CTID" &>/dev/null || true pct destroy "$CTID" &>/dev/null || true
echo -e "${GN}${CL} Container ${CTID} removed" echo -e "${GN}${CL} Container ${CTID} removed"
fi fi
# Force one final status update attempt after cleanup
# This ensures status is updated even if the first attempt failed (e.g., HTTP 400)
if declare -f post_update_to_api &>/dev/null; then
post_update_to_api "failed" "$exit_code" "force"
fi
fi fi
fi fi
fi fi