diff --git a/ct/firecrawl.sh b/ct/firecrawl.sh new file mode 100644 index 000000000..aa183bab8 --- /dev/null +++ b/ct/firecrawl.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +source <(curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxVE/main/misc/build.func) +# Copyright (c) 2021-2026 community-scripts ORG +# Author: esatbayhan +# License: MIT | https://github.com/community-scripts/ProxmoxVE/raw/main/LICENSE +# Source: https://www.firecrawl.dev/ + +APP="Firecrawl" +var_tags="${var_tags:-scraping;ai;crawler}" +var_cpu="${var_cpu:-4}" +var_ram="${var_ram:-4096}" +var_disk="${var_disk:-30}" +var_os="${var_os:-debian}" +var_version="${var_version:-13}" +var_arm64="${var_arm64:-no}" +var_unprivileged="${var_unprivileged:-1}" + +header_info "$APP" +variables +color +catch_errors + +function update_script() { + header_info + check_container_storage + check_container_resources + + if [[ ! -d /opt/firecrawl/apps/api ]]; then + msg_error "No ${APP} Installation Found!" + exit + fi + + if check_for_gh_release "firecrawl" "firecrawl/firecrawl"; then + msg_info "Stopping Services" + systemctl stop firecrawl firecrawl-playwright + msg_ok "Stopped Services" + + create_backup /opt/firecrawl/.env + + CLEAN_INSTALL=1 fetch_and_deploy_gh_release "firecrawl" "firecrawl/firecrawl" "tarball" "latest" "/opt/firecrawl" + + restore_backup + + FDB_VERSION="$(awk -F= '/^ARG FDB_VERSION=/{print $2; exit}' /opt/firecrawl/apps/api/Dockerfile)" + if [[ -z "$FDB_VERSION" ]]; then + msg_error "FDB_VERSION pin not found in upstream Dockerfile" + exit 1 + fi + if [[ "$(dpkg-query -W -f='${Version}' foundationdb-clients 2>/dev/null)" != "${FDB_VERSION}-"* ]]; then + FDB_ARCH="$(get_system_arch)" + [[ "$FDB_ARCH" == "arm64" ]] && FDB_ARCH="aarch64" + fetch_and_deploy_gh_release "foundationdb-clients" "apple/foundationdb" "binary" "$FDB_VERSION" "/opt/foundationdb-clients" "foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb" + fi + + msg_info "Building Go Library" + cd /opt/firecrawl/apps/api/sharedLibs/go-html-to-md + $STD go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go + msg_ok "Built Go Library" + + msg_info "Building Firecrawl API" + source "$HOME/.cargo/env" + cd /opt/firecrawl/apps/api + $STD pnpm install --frozen-lockfile + $STD pnpm build + CI=true $STD pnpm prune --prod --ignore-scripts + msg_ok "Built Firecrawl API" + + msg_info "Building Playwright Service" + cd /opt/firecrawl/apps/playwright-service-ts + $STD npm install + $STD npx playwright install chromium --with-deps + $STD npm run build + $STD npm prune --omit=dev + msg_ok "Built Playwright Service" + + msg_info "Starting Services" + systemctl start firecrawl-playwright firecrawl + msg_ok "Started Services" + msg_ok "Updated successfully!" + fi + exit +} + +start +build_container +description + +msg_ok "Completed Successfully!\n" +echo -e "${CREATING}${GN}${APP} setup has been successfully initialized!${CL}" +echo -e "${INFO}${YW}Access it using the following URL:${CL}" +echo -e "${GATEWAY}${BGN}http://${IP}:3002${CL}" diff --git a/ct/headers/firecrawl b/ct/headers/firecrawl new file mode 100644 index 000000000..050c01f42 --- /dev/null +++ b/ct/headers/firecrawl @@ -0,0 +1,6 @@ + _______ __ + / ____(_)_______ ______________ __ __/ / + / /_ / / ___/ _ \/ ___/ ___/ __ `/ | /| / / / + / __/ / / / / __/ /__/ / / /_/ /| |/ |/ / / +/_/ /_/_/ \___/\___/_/ \__,_/ |__/|__/_/ + diff --git a/install/firecrawl-install.sh b/install/firecrawl-install.sh new file mode 100644 index 000000000..f0cca03a5 --- /dev/null +++ b/install/firecrawl-install.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021-2026 community-scripts ORG +# Author: esatbayhan +# License: MIT | https://github.com/community-scripts/ProxmoxVE/raw/main/LICENSE +# Source: https://www.firecrawl.dev/ + +source /dev/stdin <<<"$FUNCTIONS_FILE_PATH" +color +verb_ip6 +catch_errors +setting_up_container +network_check +update_os + +msg_info "Installing Dependencies" +$STD apt install -y \ + cmake \ + git \ + nftables \ + pkg-config \ + procps \ + python3 \ + rabbitmq-server \ + redis-server +msg_ok "Installed Dependencies" + +NODE_VERSION="22" NODE_MODULE="pnpm@11.4.0" setup_nodejs +setup_go +RUST_PROFILE="minimal" setup_rust +PG_VERSION="17" PG_MODULES="cron" setup_postgresql + +fetch_and_deploy_gh_release "firecrawl" "firecrawl/firecrawl" "tarball" "latest" "/opt/firecrawl" + +msg_info "Configuring FDB" +FDB_VERSION="$(awk -F= '/^ARG FDB_VERSION=/{print $2; exit}' /opt/firecrawl/apps/api/Dockerfile)" +if [[ -z "$FDB_VERSION" ]]; then + msg_error "FDB_VERSION pin not found in upstream Dockerfile" + exit 1 +fi +FDB_ARCH="$(get_system_arch)" +[[ "$FDB_ARCH" == "arm64" ]] && FDB_ARCH="aarch64" +fetch_and_deploy_gh_release "foundationdb-clients" "apple/foundationdb" "binary" "$FDB_VERSION" "/opt/foundationdb-clients" "foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb" +msg_ok "Configured FDB" + + +PG_DB_NAME="firecrawl" PG_DB_USER="firecrawl" PG_DB_EXTENSIONS="pgcrypto,pg_cron" PG_DB_CREDS_FILE="/dev/null" setup_postgresql_db + +msg_info "Configuring pg_cron" +$STD runuser -u postgres -- psql -c "ALTER SYSTEM SET cron.use_background_workers = 'on';" +systemctl restart postgresql +until runuser -u postgres -- psql -c "SELECT 1;" &>/dev/null; do sleep 1; done +msg_ok "Configured pg_cron" + +msg_info "Configuring RabbitMQ" +systemctl enable -q --now rabbitmq-server +until rabbitmqctl status &>/dev/null; do sleep 1; done +RABBITMQ_PASSWORD="$(openssl rand -base64 32 | tr -dc 'a-zA-Z0-9' | head -c24)" +$STD rabbitmqctl add_user firecrawl "$RABBITMQ_PASSWORD" +$STD rabbitmqctl set_permissions -p / firecrawl ".*" ".*" ".*" +msg_ok "Configured RabbitMQ" + +systemctl enable -q --now redis-server + +msg_info "Importing NuQ Schema" +$STD runuser -u postgres -- psql -d firecrawl -f /opt/firecrawl/apps/nuq-postgres/nuq.sql +$STD runuser -u postgres -- psql -d firecrawl -c "GRANT USAGE ON SCHEMA nuq TO firecrawl;" +$STD runuser -u postgres -- psql -d firecrawl -c "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA nuq TO firecrawl;" +$STD runuser -u postgres -- psql -d firecrawl -c "GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA nuq TO firecrawl;" +$STD runuser -u postgres -- psql -d firecrawl -c "ALTER DEFAULT PRIVILEGES IN SCHEMA nuq GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO firecrawl;" +$STD runuser -u postgres -- psql -d firecrawl -c "ALTER DEFAULT PRIVILEGES IN SCHEMA nuq GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO firecrawl;" +msg_ok "Imported NuQ Schema" + +msg_info "Generating Configuration" +cat </opt/firecrawl/.env +ENV=local +HOST=0.0.0.0 +PORT=3002 +USE_DB_AUTHENTICATION=false +BULL_AUTH_KEY=$(openssl rand -hex 32) +JWT_SECRET=$(openssl rand -hex 32) + +REDIS_URL=redis://127.0.0.1:6379 +REDIS_RATE_LIMIT_URL=redis://127.0.0.1:6379 + +PLAYWRIGHT_MICROSERVICE_URL=http://127.0.0.1:3000/scrape + +POSTGRES_HOST=127.0.0.1 +POSTGRES_PORT=5432 +POSTGRES_USER=${PG_DB_USER} +POSTGRES_PASSWORD=${PG_DB_PASS} +POSTGRES_DB=${PG_DB_NAME} +NUQ_DATABASE_URL=postgresql://${PG_DB_USER}:${PG_DB_PASS}@127.0.0.1:5432/${PG_DB_NAME} +NUQ_DATABASE_URL_LISTEN=postgresql://${PG_DB_USER}:${PG_DB_PASS}@127.0.0.1:5432/${PG_DB_NAME} + +NUQ_BACKEND=pg +NUQ_RABBITMQ_URL=amqp://firecrawl:${RABBITMQ_PASSWORD}@127.0.0.1:5672/%2f + +WORKER_PORT=3005 +EXTRACT_WORKER_PORT=3004 +NUQ_WORKER_START_PORT=3006 +NUQ_WORKER_COUNT=5 +NUQ_PREFETCH_WORKER_PORT=3011 +NUQ_RECONCILER_WORKER_PORT=3012 +HARNESS_STARTUP_TIMEOUT_MS=60000 + +LOGGING_LEVEL=INFO +MAX_CONCURRENT_PAGES=10 +ALLOW_LOCAL_WEBHOOKS=false +BLOCK_MEDIA=false + +OPENAI_API_KEY= +OPENAI_BASE_URL= +OLLAMA_BASE_URL= +MODEL_NAME= +MODEL_EMBEDDING_NAME= +SEARXNG_ENDPOINT= +SEARXNG_ENGINES= +SEARXNG_CATEGORIES= +PROXY_SERVER= +PROXY_USERNAME= +PROXY_PASSWORD= +SELF_HOSTED_WEBHOOK_URL= +SELF_HOSTED_WEBHOOK_HMAC_SECRET= +EOF +chmod 600 /opt/firecrawl/.env +msg_ok "Generated Configuration" + +msg_info "Building Go Library" +cd /opt/firecrawl/apps/api/sharedLibs/go-html-to-md +$STD go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go +msg_ok "Built Go Library" + +msg_info "Building Firecrawl API" +cd /opt/firecrawl/apps/api +$STD pnpm install --frozen-lockfile +$STD pnpm build +CI=true $STD pnpm prune --prod --ignore-scripts +msg_ok "Built Firecrawl API" + +msg_info "Building Playwright Service" +cd /opt/firecrawl/apps/playwright-service-ts +$STD npm install +$STD npx playwright install chromium --with-deps +$STD npm run build +$STD npm prune --omit=dev +msg_ok "Built Playwright Service" + +msg_info "Creating Services" +cat </etc/systemd/system/firecrawl-playwright.service +[Unit] +Description=Firecrawl Playwright Service +After=network.target + +[Service] +Type=simple +User=root +WorkingDirectory=/opt/firecrawl/apps/playwright-service-ts +EnvironmentFile=/opt/firecrawl/.env +Environment=PORT=3000 +# /opt/firecrawl/.env also contains the API PORT=3002; force Playwright's private port here. +ExecStart=/usr/bin/env PORT=3000 /usr/bin/node /opt/firecrawl/apps/playwright-service-ts/dist/api.js +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +cat </etc/systemd/system/firecrawl.service +[Unit] +Description=Firecrawl API and Workers +After=network.target postgresql.service redis-server.service rabbitmq-server.service firecrawl-playwright.service +Requires=postgresql.service redis-server.service rabbitmq-server.service firecrawl-playwright.service + +[Service] +Type=simple +User=root +WorkingDirectory=/opt/firecrawl/apps/api +EnvironmentFile=/opt/firecrawl/.env +Environment=NODE_ENV=production +# Upstream uses --start-docker to skip install/build and start compiled node dist entrypoints. +ExecStart=/usr/bin/node /opt/firecrawl/apps/api/dist/src/harness.js --start-docker +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF +systemctl enable -q --now firecrawl-playwright +systemctl enable -q --now firecrawl +msg_ok "Created Services" + +msg_info "Configuring Firewall" +cat <<'EOF' >/etc/nftables.conf +#!/usr/sbin/nft -f + +flush ruleset + +table inet firecrawl_filter { + chain input { + type filter hook input priority 0; policy drop; + + iif "lo" accept + ct state established,related accept + + ip protocol icmp accept + ip6 nexthdr icmpv6 accept + udp sport 67 udp dport 68 accept + udp sport 547 udp dport 546 accept + + tcp dport { 22, 3002 } accept + } + + chain forward { + type filter hook forward priority 0; policy drop; + } + + chain output { + type filter hook output priority 0; policy accept; + } +} +EOF +systemctl enable -q nftables +systemctl restart nftables +msg_ok "Configured Firewall" + +motd_ssh +customize +cleanup_lxc