Kima-Hub: use curl_with_retry for ML model downloads to fix timeout issues

ct/manyfold.sh

@@ -40,6 +40,8 @@ function update_script() {
     CURRENT_VERSION=$(grep -oP 'APP_VERSION=\K[^ ]+' /opt/manyfold/.env || echo "unknown")
     cp -r /opt/manyfold/app/storage /opt/manyfold_storage_backup 2>/dev/null || true
     cp -r /opt/manyfold/app/tmp /opt/manyfold_tmp_backup 2>/dev/null || true
+    cp /opt/manyfold/app/config/credentials.yml.enc /opt/manyfold_credentials.yml.enc 2>/dev/null || true
+    cp /opt/manyfold/app/config/master.key /opt/manyfold_master.key 2>/dev/null || true
     $STD tar -czf "/opt/manyfold_${CURRENT_VERSION}_backup.tar.gz" -C /opt/manyfold app
     msg_ok "Backed up Data"
 
@@ -55,12 +57,14 @@ function update_script() {
     RUBY_VERSION=${RUBY_INSTALL_VERSION} RUBY_INSTALL_RAILS="true" HOME=/home/manyfold setup_ruby
 
     msg_info "Restoring Data"
-    rm -rf /opt/manyfold/app/{storage,tmp}
+    rm -rf /opt/manyfold/app/{storage,tmp,config/credentials.yml.enc,config/master.key}
     cp -r /opt/manyfold_storage_backup /opt/manyfold/app/storage 2>/dev/null || true
     cp -r /opt/manyfold_tmp_backup /opt/manyfold/app/tmp 2>/dev/null || true
+    cp /opt/manyfold_credentials.yml.enc /opt/manyfold/app/config/credentials.yml.enc 2>/dev/null || true
+    cp /opt/manyfold_master.key /opt/manyfold/app/config/master.key 2>/dev/null || true
     chown -R manyfold:manyfold {/home/manyfold,/opt/manyfold}
     chown -R manyfold:manyfold /opt/manyfold/app/storage /opt/manyfold/app/tmp /opt/manyfold/app/config
-    rm -rf /opt/manyfold_storage_backup /opt/manyfold_tmp_backup
+    rm -rf /opt/manyfold_storage_backup /opt/manyfold_tmp_backup /opt/manyfold_credentials.yml.enc /opt/manyfold_master.key
     msg_ok "Restored Data"
 
     msg_info "Installing Manyfold"
@@ -76,8 +80,6 @@ function update_script() {
             bundle install
             corepack prepare '"$YARN_VERSION"' --activate
             corepack use '"$YARN_VERSION"'
-            rm -f config/credentials.yml.enc config/master.key
-            EDITOR=/bin/true bin/rails credentials:edit
             bin/rails db:migrate
             bin/rails assets:precompile
         '

install/kima-hub-install.sh

@@ -56,20 +56,20 @@ msg_ok "Installed Python Dependencies"
 msg_info "Downloading Essentia ML Models"
 mkdir -p /opt/kima-hub/models
 cd /opt/kima-hub/models
-curl -fsSL -o msd-musicnn-1.pb "https://essentia.upf.edu/models/autotagging/msd/msd-musicnn-1.pb"
-curl -fsSL -o mood_happy-msd-musicnn-1.pb "https://essentia.upf.edu/models/classification-heads/mood_happy/mood_happy-msd-musicnn-1.pb"
-curl -fsSL -o mood_sad-msd-musicnn-1.pb "https://essentia.upf.edu/models/classification-heads/mood_sad/mood_sad-msd-musicnn-1.pb"
-curl -fsSL -o mood_relaxed-msd-musicnn-1.pb "https://essentia.upf.edu/models/classification-heads/mood_relaxed/mood_relaxed-msd-musicnn-1.pb"
-curl -fsSL -o mood_aggressive-msd-musicnn-1.pb "https://essentia.upf.edu/models/classification-heads/mood_aggressive/mood_aggressive-msd-musicnn-1.pb"
-curl -fsSL -o mood_party-msd-musicnn-1.pb "https://essentia.upf.edu/models/classification-heads/mood_party/mood_party-msd-musicnn-1.pb"
-curl -fsSL -o mood_acoustic-msd-musicnn-1.pb "https://essentia.upf.edu/models/classification-heads/mood_acoustic/mood_acoustic-msd-musicnn-1.pb"
-curl -fsSL -o mood_electronic-msd-musicnn-1.pb "https://essentia.upf.edu/models/classification-heads/mood_electronic/mood_electronic-msd-musicnn-1.pb"
-curl -fsSL -o danceability-msd-musicnn-1.pb "https://essentia.upf.edu/models/classification-heads/danceability/danceability-msd-musicnn-1.pb"
-curl -fsSL -o voice_instrumental-msd-musicnn-1.pb "https://essentia.upf.edu/models/classification-heads/voice_instrumental/voice_instrumental-msd-musicnn-1.pb"
+curl_with_retry "https://essentia.upf.edu/models/autotagging/msd/msd-musicnn-1.pb" "msd-musicnn-1.pb"
+curl_with_retry "https://essentia.upf.edu/models/classification-heads/mood_happy/mood_happy-msd-musicnn-1.pb" "mood_happy-msd-musicnn-1.pb"
+curl_with_retry "https://essentia.upf.edu/models/classification-heads/mood_sad/mood_sad-msd-musicnn-1.pb" "mood_sad-msd-musicnn-1.pb"
+curl_with_retry "https://essentia.upf.edu/models/classification-heads/mood_relaxed/mood_relaxed-msd-musicnn-1.pb" "mood_relaxed-msd-musicnn-1.pb"
+curl_with_retry "https://essentia.upf.edu/models/classification-heads/mood_aggressive/mood_aggressive-msd-musicnn-1.pb" "mood_aggressive-msd-musicnn-1.pb"
+curl_with_retry "https://essentia.upf.edu/models/classification-heads/mood_party/mood_party-msd-musicnn-1.pb" "mood_party-msd-musicnn-1.pb"
+curl_with_retry "https://essentia.upf.edu/models/classification-heads/mood_acoustic/mood_acoustic-msd-musicnn-1.pb" "mood_acoustic-msd-musicnn-1.pb"
+curl_with_retry "https://essentia.upf.edu/models/classification-heads/mood_electronic/mood_electronic-msd-musicnn-1.pb" "mood_electronic-msd-musicnn-1.pb"
+curl_with_retry "https://essentia.upf.edu/models/classification-heads/danceability/danceability-msd-musicnn-1.pb" "danceability-msd-musicnn-1.pb"
+curl_with_retry "https://essentia.upf.edu/models/classification-heads/voice_instrumental/voice_instrumental-msd-musicnn-1.pb" "voice_instrumental-msd-musicnn-1.pb"
 msg_ok "Downloaded Essentia ML Models"
 
 msg_info "Downloading CLAP Model"
-curl -fsSL -o /opt/kima-hub/models/music_audioset_epoch_15_esc_90.14.pt "https://huggingface.co/lukewys/laion_clap/resolve/main/music_audioset_epoch_15_esc_90.14.pt"
+CURL_TIMEOUT=600 curl_with_retry "https://huggingface.co/lukewys/laion_clap/resolve/main/music_audioset_epoch_15_esc_90.14.pt" "/opt/kima-hub/models/music_audioset_epoch_15_esc_90.14.pt"
 msg_ok "Downloaded CLAP Model"
 
 msg_info "Building Backend"