proxmox-ve-install-scripts/install/huly-install.sh

#!/usr/bin/env bash

# Copyright (c) 2021-2025 community-scripts ORG
# Author: GitHub Copilot
# License: MIT | https://github.com/community-scripts/ProxmoxVE/raw/main/LICENSE
# Source: https://github.com/hcengineering/huly-selfhost

# Import Functions and Setup
source /dev/stdin <<<"$FUNCTIONS_FILE_PATH"
color
verb_ip6
catch_errors
setting_up_container
network_check
update_os

# Installing Dependencies
msg_info "Installing Dependencies"
$STD apt-get update
$STD apt-get install -y curl git ca-certificates gnupg nginx lsb-release python3 python3-pip python3-venv
msg_ok "Installed Dependencies"

# Installing Python and uv package manager
msg_info "Setting up Python environment and uv"
setup_uv
msg_ok "Python environment and uv ready"

msg_info "Installing MongoDB"
# Install MongoDB natively
$STD curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc | gpg --dearmor -o /etc/apt/keyrings/mongodb-server-7.0.gpg
$STD echo "deb [ arch=amd64,arm64 signed-by=/etc/apt/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/debian $(lsb_release -cs)/mongodb-org/7.0 main" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list >/dev/null
$STD apt-get update
$STD apt-get install -y mongodb-org
$STD systemctl enable --now mongod
msg_ok "Installed MongoDB"

msg_info "Installing Node.js and web-push"
# Install Node.js for running Huly services
NODE_VERSION=20 install_node_and_modules # This function is from tools.func
$STD npm install -g web-push
msg_ok "Installed Node.js and web-push"

# Use temporary Docker for extracting components
msg_info "Installing Docker temporarily (for Huly component extraction)"
$STD mkdir -p /etc/apt/keyrings
$STD curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
$STD echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list >/dev/null
$STD apt-get update
$STD apt-get install -y docker-ce docker-ce-cli containerd.io
$STD systemctl start docker
msg_ok "Installed Docker temporarily"

msg_info "Setting up Huly environment and configuration"
# Create necessary directories
$STD mkdir -p /opt/huly-selfhost
$STD mkdir -p /opt/huly/{front,account,transactor,collaborator,rekoni}

# Get server IP
SERVER_IP=$(hostname -I | awk '{print $1}')

# Generate VAPID keys for push notifications
VAPID_OUTPUT=$($STD web-push generate-vapid-keys)
VAPID_PUBLIC_KEY=$(echo "$VAPID_OUTPUT" | grep "Public Key:" | cut -d ":" -f2 | tr -d ' ')
VAPID_PRIVATE_KEY=$(echo "$VAPID_OUTPUT" | grep "Private Key:" | cut -d ":" -f2 | tr -d ' ')

# Create MongoDB database and user for Huly
# Ensure mongosh is available or use mongo if older version
MONGO_EXEC=$(command -v mongosh || command -v mongo)
if [ -z "$MONGO_EXEC" ]; then
    msg_error "Neither mongosh nor mongo command found. Cannot create MongoDB user for Huly."
    exit 1
fi

$STD $MONGO_EXEC --eval "
use huly;
db.createUser({
  user: 'huly',
  pwd: 'hulypassword123', // Consider making this configurable or randomly generated
  roles: [{role: 'readWrite', db: 'huly'}]
});
"
msg_ok "Created MongoDB user and database for Huly"

# Save configuration and VAPID keys
APP_SECRET_KEY=$(openssl rand -hex 32) # Generate a random secret key

cat <<EOF >/opt/huly-selfhost/native.conf
# Huly Native Configuration
SERVER_IP=$SERVER_IP
MONGO_URL=mongodb://huly:hulypassword123@localhost:27017/huly
MINIO_ENDPOINT=localhost:9000
MINIO_ACCESS_KEY=minioadmin # Consider making this configurable or randomly generated
MINIO_SECRET_KEY=minioadmin # Consider making this configurable or randomly generated
APP_SECRET_KEY=$APP_SECRET_KEY

# VAPID Keys for Push Notifications
VAPID_PUBLIC_KEY=$VAPID_PUBLIC_KEY
VAPID_PRIVATE_KEY=$VAPID_PRIVATE_KEY

# Service Ports (ensure these are unique and not conflicting)
FRONT_PORT=3000
ACCOUNT_PORT=3001
TRANSACTOR_PORT=3002
COLLABORATOR_PORT=3078
REKONI_PORT=4004
EOF

cat <<EOF >~/huly.creds
Huly Credentials (save this file securely!)
MongoDB URL: mongodb://huly:hulypassword123@localhost:27017/huly
MinIO Access: minioadmin / minioadmin (Console: http://$SERVER_IP:9001)
App Secret Key: $APP_SECRET_KEY
VAPID Public Key: $VAPID_PUBLIC_KEY
VAPID Private Key: $VAPID_PRIVATE_KEY
EOF
chmod 600 ~/huly.creds
msg_ok "Set up Huly environment and configuration in /opt/huly-selfhost/native.conf and ~/huly.creds"

msg_info "Extracting Huly applications from Docker images"
# Function to extract from container with fallback paths
extract_from_container() {
    local component_name="$1"
    local image_name="$2"
    local target_dir="$3"
    local container_name="huly-${component_name}-extract"

    msg_info "Pulling Docker image for $component_name: $image_name"
    if ! $STD docker pull "$image_name"; then
        msg_error "Failed to pull Docker image $image_name for $component_name."
        return 1
    fi

    msg_info "Extracting $component_name from $image_name to $target_dir"
    if ! $STD docker create --name "$container_name" "$image_name"; then
        msg_error "Failed to create Docker container for $component_name."
        return 1
    fi

    # Try common paths where applications might be located
    local extracted=false
    # Order matters: specific paths first, then broader ones
    for path_in_container in "/app/dist" "/app" "/usr/src/app/dist" "/usr/src/app" "/dist" "/opt/app" "/home/app"; do
        # Check if path exists in container before attempting to copy
        if $STD docker exec "$container_name" ls "${path_in_container}/." >/dev/null 2>&1; then
            if $STD docker cp "${container_name}:${path_in_container}/." "$target_dir/"; then
                extracted=true
                msg_ok "Extracted $component_name from ${path_in_container}"
                break
            fi
        fi
    done

    if ! $extracted; then
        msg_warn "Could not find standard app directory in $component_name container ($image_name). Attempting to copy entire root '/'. This might be slow and include unnecessary files."
        if $STD docker cp "${container_name}:/." "$target_dir/"; then
            msg_ok "Copied entire root for $component_name. Review $target_dir for correctness."
        else
            msg_error "Failed to copy any files for $component_name from $image_name."
            $STD docker rm "$container_name"
            return 1
        fi
    fi
    $STD docker rm "$container_name"
    msg_ok "Extraction process completed for $component_name"
}

# Extract Huly Components
extract_from_container "front" "hardcoreeng/front:latest" "/opt/huly/front"
extract_from_container "account" "hardcoreeng/account:latest" "/opt/huly/account"
extract_from_container "transactor" "hardcoreeng/transactor:latest" "/opt/huly/transactor"
extract_from_container "collaborator" "hardcoreeng/collaborator:latest" "/opt/huly/collaborator"
extract_from_container "rekoni" "hardcoreeng/rekoni:latest" "/opt/huly/rekoni"
msg_ok "Extracted all Huly applications"

msg_info "Installing MinIO for object storage"
# Fetch latest MinIO version dynamically if possible, or use a known good one
# For stability, using a fixed version. Update as needed.
MINIO_VERSION="RELEASE.2024-06-13T22-53-53Z"
$STD curl -fsSL "https://dl.min.io/server/minio/release/linux-amd64/archive/minio.${MINIO_VERSION}" -o /usr/local/bin/minio
$STD chmod +x /usr/local/bin/minio
useradd -r -s /bin/false minio 2>/dev/null || true # Create user if not exists, suppress error if it does
mkdir -p /opt/minio/data /etc/minio
chown -R minio:minio /opt/minio /etc/minio

# Source native.conf for MinIO keys to write into minio.conf
# This ensures consistency if keys were changed in native.conf
source /opt/huly-selfhost/native.conf

cat <<EOF >/etc/minio/minio.conf
MINIO_ROOT_USER=${MINIO_ACCESS_KEY}
MINIO_ROOT_PASSWORD=${MINIO_SECRET_KEY}
MINIO_VOLUMES="/opt/minio/data"
MINIO_OPTS="--console-address :9001"
EOF
msg_ok "Installed and configured MinIO"

msg_info "Setting up Rekoni Python environment"
REKONI_PYTHON_EXEC="/usr/bin/python3" # Default system Python

# Define Rekoni venv path
REKONI_VENV_PATH="/opt/huly-venv/rekoni"
$STD mkdir -p "$(dirname "$REKONI_VENV_PATH")"

if [ -f /opt/huly/rekoni/requirements.txt ]; then
    msg_info "Found requirements.txt for Rekoni. Creating virtual environment at $REKONI_VENV_PATH."
    if $STD /usr/local/bin/uv venv "$REKONI_VENV_PATH"; then
        REKONI_PYTHON_EXEC="$REKONI_VENV_PATH/bin/python3"
        msg_info "Installing Rekoni dependencies from requirements.txt using uv into venv."
        # Activate venv for pip install or ensure uv uses the correct python
        if $STD "$REKONI_VENV_PATH/bin/uv" pip install -r /opt/huly/rekoni/requirements.txt; then
            msg_ok "Rekoni dependencies installed into virtual environment."
        else
            msg_error "Failed to install Rekoni dependencies from requirements.txt using uv. Rekoni might not work."
            REKONI_PYTHON_EXEC="/usr/bin/python3" # Fallback to system Python
        fi
    else
        msg_warn "Failed to create virtual environment for Rekoni with uv. Will attempt system-wide Python package installation for Rekoni."
        # Attempt to install requirements.txt system-wide as a fallback
        msg_info "Attempting to install Rekoni dependencies from requirements.txt system-wide using uv."
        if $STD /usr/local/bin/uv pip install --system -r /opt/huly/rekoni/requirements.txt; then
            msg_ok "Installed Rekoni dependencies system-wide from requirements.txt."
        else
            msg_error "Failed to install Rekoni dependencies system-wide from requirements.txt. Rekoni might not work."
        fi
    fi
else
    msg_warn "requirements.txt not found for Rekoni in /opt/huly/rekoni/. Attempting to install a common set of Python packages system-wide."
    # List of common dependencies, keep this updated based on Rekoni's needs
    COMMON_REKONI_DEPS="fastapi uvicorn python-multipart Pillow Wand pdf2image pytesseract nltk spacy Faker requests beautifulsoup4 readability-lxml sentence-transformers pymongo minio pydantic tiktoken openai InstructorEmbedding PyMuPDF"
    msg_info "Installing common Python packages for Rekoni system-wide using uv: $COMMON_REKONI_DEPS"
    if $STD /usr/local/bin/uv pip install -v --system $COMMON_REKONI_DEPS; then
        msg_ok "Installed common Python packages system-wide for Rekoni."
    else
        msg_error "Failed to install common Python packages system-wide for Rekoni. Rekoni might not work."
    fi
fi

msg_info "Downloading NLTK and Spacy models for Rekoni (using $REKONI_PYTHON_EXEC)"
NLTK_MODELS="punkt stopwords wordnet omw-1.4" # Open Multilingual Wordnet
SPACY_MODEL="en_core_web_sm"                  # Small English model

# Ensure NLTK_DATA directory exists and is writable if needed, or use a user-writable path
# For system-wide, /usr/share/nltk_data is common. For venv, it might be within the venv.
# The -d flag for nltk.downloader specifies the download directory.
NLTK_DATA_DIR="/usr/share/nltk_data"
$STD mkdir -p "$NLTK_DATA_DIR"
# Grant write permission temporarily if needed, or run as root.
# For simplicity in script, assuming root or sudo for system-wide.
if ! $STD $REKONI_PYTHON_EXEC -m nltk.downloader -d "$NLTK_DATA_DIR" $NLTK_MODELS; then
    msg_warn "NLTK model download failed for some components ($NLTK_MODELS) using $REKONI_PYTHON_EXEC. This might affect Rekoni functionality."
fi
if ! $STD $REKONI_PYTHON_EXEC -m spacy download $SPACY_MODEL; then
    msg_warn "Spacy model $SPACY_MODEL download failed using $REKONI_PYTHON_EXEC. This might affect Rekoni functionality."
fi
msg_ok "Python setup for Rekoni completed. Rekoni will run using: $REKONI_PYTHON_EXEC"

msg_info "Removing Docker"
$STD systemctl stop docker
$STD apt-get remove -y docker-ce docker-ce-cli containerd.io
$STD apt-get purge -y docker-ce docker-ce-cli containerd.io --allow-remove-essential
$STD rm -rf /var/lib/docker /var/lib/containerd
$STD rm -f /etc/apt/sources.list.d/docker.list /etc/apt/keyrings/docker.gpg
msg_ok "Removed Docker"

msg_info "Configuring Nginx as reverse proxy"
$STD rm -f /etc/nginx/sites-enabled/default
cat <<EOF >/etc/nginx/sites-available/huly.conf
server {
    listen 80 default_server;
    listen [::]:80 default_server;
    server_name _; # Replace with your domain if you have one

    client_max_body_size 100M; # For file uploads via transactor

    # Source the config file to get port variables for nginx
    # Note: This is a bash-ism. Nginx doesn't directly execute this.
    # Ports must be hardcoded or managed by a script that generates this nginx config.
    # For simplicity, using fixed ports here, matching native.conf defaults.

    location / {
        proxy_pass http://127.0.0.1:3000; # Huly Frontend (FRONT_PORT)
        proxy_http_version 1.1;
        proxy_set_header Upgrade \$http_upgrade;
        proxy_set_header Connection "upgrade"; # For WebSockets if front uses them directly
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        proxy_read_timeout 300s; # Increase timeout for potentially long operations
        proxy_send_timeout 300s;
    }

    location /account/ {
        proxy_pass http://127.0.0.1:3001/; # Huly Account (ACCOUNT_PORT)
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
    }

    # Transactor handles general API and WebSocket for collaboration
    location /transactor/ {
        proxy_pass http://127.0.0.1:3002/; # Huly Transactor (TRANSACTOR_PORT)
        proxy_http_version 1.1;
        proxy_set_header Upgrade \$http_upgrade; # Essential for WebSocket
        proxy_set_header Connection "upgrade";   # Essential for WebSocket
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
    }
    
    # Specific path for uploads if front uses /upload directly to transactor
    # Ensure this matches how the frontend makes upload requests.
    # If uploads go via /transactor/upload, this specific block might not be needed.
    location /upload {
        proxy_pass http://127.0.0.1:3002/upload; # Huly Transactor Upload (TRANSACTOR_PORT)
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        client_max_body_size 100M; # Ensure this is set for uploads
    }

    location /collaborator/ {
        proxy_pass http://127.0.0.1:3078/; # Huly Collaborator (COLLABORATOR_PORT)
        proxy_http_version 1.1;
        proxy_set_header Upgrade \$http_upgrade; # Essential for WebSocket
        proxy_set_header Connection "upgrade";   # Essential for WebSocket
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
    }

    location /rekoni/ {
        proxy_pass http://127.0.0.1:4004/; # Huly Rekoni (REKONI_PORT)
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
    }
    
    # MinIO Console access (if needed through reverse proxy, otherwise direct access on :9001)
    # location /minio-console/ {
    #    proxy_pass http://127.0.0.1:9001/;
    #    proxy_set_header Host \$host;
    # }

    # MinIO API/S3 access (if needed through reverse proxy)
    # Usually, services access MinIO directly on localhost:9000
    # location /s3/ { # Example path, adjust as needed
    #    proxy_pass http://127.0.0.1:9000/;
    #    proxy_set_header Host \$host; # Important for MinIO to work correctly
    # }
}
EOF
$STD ln -sf /etc/nginx/sites-available/huly.conf /etc/nginx/sites-enabled/huly.conf
if nginx -t >/dev/null 2>&1; then
    $STD systemctl reload nginx
    msg_ok "Nginx configured and reloaded."
else
    msg_error "Nginx configuration test failed. Please check /etc/nginx/sites-available/huly.conf and nginx logs (journalctl -u nginx)."
    # Optionally, print nginx -t output for debugging
    nginx -t
fi

# Source the config file to get variables for systemd units
# This ensures systemd units use the values from native.conf
if [ -f /opt/huly-selfhost/native.conf ]; then
    source /opt/huly-selfhost/native.conf
else
    msg_error "Huly configuration file /opt/huly-selfhost/native.conf not found. Cannot create systemd services correctly."
    exit 1
fi

msg_info "Creating systemd services for Huly components and MinIO"

# MinIO service
cat <<EOF >/etc/systemd/system/minio.service
[Unit]
Description=MinIO Object Storage
Documentation=https://docs.min.io
Wants=network-online.target
After=network-online.target
AssertFileIsExecutable=/usr/local/bin/minio

[Service]
WorkingDirectory=/opt/minio
User=minio
Group=minio
EnvironmentFile=/etc/minio/minio.conf # Contains MINIO_ROOT_USER, MINIO_ROOT_PASSWORD, MINIO_VOLUMES, MINIO_OPTS
ExecStartPre=/bin/bash -c "if [ -z \\"\${MINIO_VOLUMES}\\" ]; then echo 'Variable MINIO_VOLUMES not set in /etc/minio/minio.conf'; exit 1; fi"
ExecStart=/usr/local/bin/minio server \$MINIO_OPTS \$MINIO_VOLUMES
Restart=always
LimitNOFILE=1048576
TasksMax=infinity
TimeoutStopSec=infinity # Or a reasonable timeout
SendSIGKILL=no

[Install]
WantedBy=multi-user.target
EOF

# Huly User for services (optional, but good practice)
# useradd -r -s /bin/false hulyuser 2>/dev/null || true

# Frontend service
# Check if the entrypoint is bundle.js or server.js or main.js - adjust ExecStart
# Common for Node.js apps: dist/bundle.js, dist/main.js, server.js, app.js
# Assuming /opt/huly/front/dist/bundle.js from previous observations
FRONT_EXEC_PATH="/opt/huly/front/dist/bundle.js" # Default, adjust if different
if [ ! -f "$FRONT_EXEC_PATH" ] && [ -f "/opt/huly/front/bundle.js" ]; then FRONT_EXEC_PATH="/opt/huly/front/bundle.js"; fi
if [ ! -f "$FRONT_EXEC_PATH" ] && [ -f "/opt/huly/front/main.js" ]; then FRONT_EXEC_PATH="/opt/huly/front/main.js"; fi
if [ ! -f "$FRONT_EXEC_PATH" ] && [ -f "/opt/huly/front/server.js" ]; then FRONT_EXEC_PATH="/opt/huly/front/server.js"; fi
# Add more checks if necessary, or make it configurable

cat <<EOF >/etc/systemd/system/huly-front.service
[Unit]
Description=Huly Frontend Service
After=network.target nginx.service huly-account.service huly-transactor.service huly-rekoni.service huly-collaborator.service
Wants=nginx.service huly-account.service huly-transactor.service huly-rekoni.service huly-collaborator.service

[Service]
Type=simple
WorkingDirectory=/opt/huly/front
ExecStart=/usr/bin/node $FRONT_EXEC_PATH
Restart=always
User=root # Consider 'hulyuser' or 'www-data' if created and permissions allow
EnvironmentFile=/opt/huly-selfhost/native.conf # Sources all needed env vars
# Explicitly set PORT if not in native.conf or to override
Environment="PORT=\${FRONT_PORT:-3000}" 
Environment="NODE_ENV=production"
# URLs should be sourced from native.conf or constructed if services are on different hosts
Environment="ACCOUNTS_URL=http://localhost:\${ACCOUNT_PORT:-3001}"
Environment="UPLOAD_URL=http://localhost:\${TRANSACTOR_PORT:-3002}/upload" # Ensure this path is correct
Environment="REKONI_URL=http://localhost:\${REKONI_PORT:-4004}"
Environment="COLLABORATOR_URL=ws://localhost:\${COLLABORATOR_PORT:-3078}" # Note: ws:// for collaborator
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target
EOF

# Account service
ACCOUNT_EXEC_PATH="/opt/huly/account/bundle.js" # Default
if [ ! -f "$ACCOUNT_EXEC_PATH" ] && [ -f "/opt/huly/account/main.js" ]; then ACCOUNT_EXEC_PATH="/opt/huly/account/main.js"; fi
# ... add other checks for account executable

cat <<EOF >/etc/systemd/system/huly-account.service
[Unit]
Description=Huly Account Service
After=network.target mongod.service
Requires=mongod.service

[Service]
Type=simple
WorkingDirectory=/opt/huly/account
ExecStart=/usr/bin/node $ACCOUNT_EXEC_PATH
Restart=always
User=root # Consider 'hulyuser'
EnvironmentFile=/opt/huly-selfhost/native.conf
Environment="PORT=\${ACCOUNT_PORT:-3001}"
Environment="TRANSACTOR_URL=ws://localhost:\${TRANSACTOR_PORT:-3002}" # Ensure ws or wss
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target
EOF

# Transactor service
TRANSACTOR_EXEC_PATH="/opt/huly/transactor/bundle.js" # Default
if [ ! -f "$TRANSACTOR_EXEC_PATH" ] && [ -f "/opt/huly/transactor/main.js" ]; then TRANSACTOR_EXEC_PATH="/opt/huly/transactor/main.js"; fi
# ... add other checks for transactor executable

cat <<EOF >/etc/systemd/system/huly-transactor.service
[Unit]
Description=Huly Transactor Service
After=network.target mongod.service minio.service huly-account.service
Requires=mongod.service minio.service huly-account.service

[Service]
Type=simple
WorkingDirectory=/opt/huly/transactor
ExecStart=/usr/bin/node $TRANSACTOR_EXEC_PATH
Restart=always
User=root # Consider 'hulyuser'
EnvironmentFile=/opt/huly-selfhost/native.conf
Environment="PORT=\${TRANSACTOR_PORT:-3002}"
# STORAGE_CONFIG needs careful construction based on native.conf values
Environment="STORAGE_CONFIG=minio|minio?accessKey=\${MINIO_ACCESS_KEY}&secretKey=\${MINIO_SECRET_KEY}&endpoint=\${MINIO_ENDPOINT}&secure=false" # Assuming http for minio
Environment="ACCOUNTS_URL=http://localhost:\${ACCOUNT_PORT:-3001}"
Environment="REKONI_URL=http://localhost:\${REKONI_PORT:-4004}"
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target
EOF

# Collaborator service
COLLABORATOR_EXEC_PATH="/opt/huly/collaborator/dist/bundle.js" # Default
if [ ! -f "$COLLABORATOR_EXEC_PATH" ] && [ -f "/opt/huly/collaborator/bundle.js" ]; then COLLABORATOR_EXEC_PATH="/opt/huly/collaborator/bundle.js"; fi
# ... add other checks for collaborator executable

cat <<EOF >/etc/systemd/system/huly-collaborator.service
[Unit]
Description=Huly Collaborator Service
After=network.target mongod.service
Requires=mongod.service

[Service]
Type=simple
WorkingDirectory=/opt/huly/collaborator
ExecStart=/usr/bin/node $COLLABORATOR_EXEC_PATH
Restart=always
User=root # Consider 'hulyuser'
EnvironmentFile=/opt/huly-selfhost/native.conf
Environment="PORT=\${COLLABORATOR_PORT:-3078}"
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target
EOF

# Rekoni service
# Rekoni is Python, ExecStart uses the determined $REKONI_PYTHON_EXEC
# Ensure main.py (or equivalent) is the entrypoint in /opt/huly/rekoni/
REKONI_ENTRYPOINT="main.py" # Default, ensure this file exists in /opt/huly/rekoni
if [ ! -f "/opt/huly/rekoni/$REKONI_ENTRYPOINT" ]; then
    msg_warn "Rekoni entrypoint $REKONI_ENTRYPOINT not found in /opt/huly/rekoni/. Service might fail."
    # Potentially look for app.py or other common Python entrypoints if main.py is missing
fi

cat <<EOF >/etc/systemd/system/huly-rekoni.service
[Unit]
Description=Huly Rekoni Service (File Processing)
After=network.target mongod.service minio.service
Requires=mongod.service minio.service

[Service]
Type=simple
WorkingDirectory=/opt/huly/rekoni
ExecStart=$REKONI_PYTHON_EXEC $REKONI_ENTRYPOINT
Restart=always
User=root # Consider 'hulyuser'
EnvironmentFile=/opt/huly-selfhost/native.conf
Environment="PORT=\${REKONI_PORT:-4004}"
Environment="NLTK_DATA=${NLTK_DATA_DIR}" # Pass NLTK data path
# Add other Rekoni specific ENV VARS if any, e.g. OPENAI_API_KEY if used and not in native.conf
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target
EOF

$STD systemctl daemon-reload
# Enable services (start them on boot)
$STD systemctl enable minio huly-front huly-account huly-transactor huly-collaborator huly-rekoni mongod nginx
# Start services now
$STD systemctl start minio mongod # Start dependencies first
# Brief pause to allow DB/MinIO to initialize before dependent services start
sleep 5
$STD systemctl start nginx huly-account # Account needs Mongo
sleep 2
$STD systemctl start huly-transactor huly-rekoni # These need Account, Mongo, Minio
sleep 2
$STD systemctl start huly-collaborator # Needs Mongo
sleep 2
$STD systemctl start huly-front # Front needs all backends

msg_ok "Created, enabled, and started Huly services and dependencies"

# Create version file to track installations and updates
echo "$(date '+%Y-%m-%d %H:%M:%S') - Initial Install of Huly Native" >/opt/huly-selfhost/version.txt

motd_ssh
customize

# Cleanup
msg_info "Cleaning up apt cache and unused packages"
$STD apt-get -y autoremove
$STD apt-get -y autoclean
msg_ok "Cleaned up"

echo -e "${INFO}${YW} Huly native installation completed successfully!${CL}"
echo -e "${INFO}${YW} Access URL: http://$SERVER_IP (or your configured domain)${CL}"
echo -e "${INFO}${YW} Services may take a few moments to fully initialize.${CL}"
echo -e ""
echo -e "${INFO}${YW} Service Status Commands:${CL}"
echo -e "${INFO}${YW} • Check all Huly services: systemctl status 'huly-*' minio mongod nginx ${CL}"
echo -e "${INFO}${YW} • Example: systemctl status huly-front ${CL}"
echo -e "${INFO}${YW} • View logs: journalctl -u <service_name> -f (e.g., journalctl -u huly-front -f)${CL}"
echo -e "${INFO}${YW} • MinIO Console: http://$SERVER_IP:9001 (User: ${MINIO_ACCESS_KEY}, Pass: ${MINIO_SECRET_KEY})${CL}"
echo -e ""
echo -e "${INFO}${YW} IMPORTANT: Review credentials in ~/huly.creds and store them securely.${CL}"
echo -e "${INFO}${YW} Configuration is in: /opt/huly-selfhost/native.conf${CL}"
echo -e "${INFO}${YW} Rekoni Python executable: $REKONI_PYTHON_EXEC ${CL}"