From d1c8711207a8d4744a061e4b9af5b04fbb8b35c3 Mon Sep 17 00:00:00 2001 From: "push-app-to-main[bot]" <203845782+push-app-to-main[bot]@users.noreply.github.com> Date: Sat, 20 Jun 2026 19:46:09 +1000 Subject: [PATCH] Apache-Airflow (#15228) * Add apache-airflow (ct) * Fix indentation for restore_backup function call --------- Co-authored-by: push-app-to-main[bot] <203845782+push-app-to-main[bot]@users.noreply.github.com> Co-authored-by: CanbiZ (MickLesk) <47820557+MickLesk@users.noreply.github.com> --- ct/apache-airflow.sh | 75 ++++++++++++++++ ct/headers/apache-airflow | 6 ++ install/apache-airflow-install.sh | 141 ++++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100644 ct/apache-airflow.sh create mode 100644 ct/headers/apache-airflow create mode 100644 install/apache-airflow-install.sh diff --git a/ct/apache-airflow.sh b/ct/apache-airflow.sh new file mode 100644 index 000000000..d6a331177 --- /dev/null +++ b/ct/apache-airflow.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +source <(curl -fsSL https://raw.githubusercontent.com/community-scripts/ProxmoxVE/main/misc/build.func) +# Copyright (c) 2021-2026 community-scripts ORG +# Author: MickLesk (CanbiZ) +# License: MIT | https://github.com/community-scripts/ProxmoxVE/raw/main/LICENSE +# Source: https://github.com/apache/airflow + +APP="Apache-Airflow" +var_tags="${var_tags:-workflow;scheduler;automation}" +var_cpu="${var_cpu:-2}" +var_ram="${var_ram:-4096}" +var_disk="${var_disk:-16}" +var_os="${var_os:-debian}" +var_version="${var_version:-13}" +var_arm64="${var_arm64:-no}" +var_unprivileged="${var_unprivileged:-1}" + +header_info "$APP" +variables +color +catch_errors + +function update_script() { + header_info + check_container_storage + check_container_resources + + if [[ ! -d /opt/airflow ]]; then + msg_error "No ${APP} Installation Found!" + exit + fi + + INSTALLED=$(cat ~/.airflow 2>/dev/null || echo "0") + LATEST=$(curl -fsSL "https://pypi.org/pypi/apache-airflow/json" | jq -r '.info.version') + + if [[ "$INSTALLED" == "$LATEST" ]]; then + msg_ok "Already on the latest version (${LATEST})" + exit + fi + + msg_info "Stopping Services" + systemctl stop airflow-api-server airflow-scheduler airflow-dag-processor airflow-triggerer + msg_ok "Stopped Services" + + create_backup /opt/airflow/.env + + msg_info "Updating Apache Airflow to ${LATEST}" + $STD uv pip install --python /opt/airflow/.venv/bin/python \ + "apache-airflow[postgres,fab]==${LATEST}" \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${LATEST}/constraints-3.12.txt" + echo "${LATEST}" >~/.airflow + msg_ok "Updated Apache Airflow to ${LATEST}" + + restore_backup + + msg_info "Running Database Migrations" + set -a && source /opt/airflow/.env && set +a + $STD /opt/airflow/.venv/bin/airflow db migrate + msg_ok "Ran Database Migrations" + + msg_info "Starting Services" + systemctl start airflow-api-server airflow-scheduler airflow-dag-processor airflow-triggerer + msg_ok "Started Services" + msg_ok "Updated successfully!" + exit +} + +start +build_container +description + +msg_ok "Completed Successfully!\n" +echo -e "${CREATING}${GN}${APP} setup has been successfully initialized!${CL}" +echo -e "${INFO}${YW} Access it using the following URL:${CL}" +echo -e "${TAB}${GATEWAY}${BGN}http://${IP}:8080${CL}" diff --git a/ct/headers/apache-airflow b/ct/headers/apache-airflow new file mode 100644 index 000000000..0c0912393 --- /dev/null +++ b/ct/headers/apache-airflow @@ -0,0 +1,6 @@ + ___ __ ___ _ ______ + / | ____ ____ ______/ /_ ___ / | (_)____/ __/ /___ _ __ + / /| | / __ \/ __ `/ ___/ __ \/ _ \______/ /| | / / ___/ /_/ / __ \ | /| / / + / ___ |/ /_/ / /_/ / /__/ / / / __/_____/ ___ |/ / / / __/ / /_/ / |/ |/ / +/_/ |_/ .___/\__,_/\___/_/ /_/\___/ /_/ |_/_/_/ /_/ /_/\____/|__/|__/ + /_/ diff --git a/install/apache-airflow-install.sh b/install/apache-airflow-install.sh new file mode 100644 index 000000000..89ec4835c --- /dev/null +++ b/install/apache-airflow-install.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021-2026 community-scripts ORG +# Author: MickLesk (CanbiZ) +# License: MIT | https://github.com/community-scripts/ProxmoxVE/raw/main/LICENSE +# Source: https://github.com/apache/airflow + +source /dev/stdin <<<"$FUNCTIONS_FILE_PATH" +color +verb_ip6 +catch_errors +setting_up_container +network_check +update_os + +msg_info "Installing Dependencies" +$STD apt install -y \ + build-essential \ + libpq-dev \ + libssl-dev \ + libffi-dev \ + python3-dev +msg_ok "Installed Dependencies" + +UV_PYTHON="3.12" setup_uv +PG_VERSION="16" setup_postgresql +PG_DB_NAME="airflow" PG_DB_USER="airflow" setup_postgresql_db + +msg_info "Installing Apache Airflow" +AIRFLOW_VERSION="3.2.1" +mkdir -p /opt/airflow/{dags,logs,plugins} +cd /opt/airflow +$STD uv venv --python 3.12 +$STD uv pip install --python /opt/airflow/.venv/bin/python \ + "apache-airflow[postgres,fab]==${AIRFLOW_VERSION}" \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.12.txt" +echo "${AIRFLOW_VERSION}" >~/.airflow +msg_ok "Installed Apache Airflow" + +msg_info "Configuring Application" +FERNET_KEY=$(/opt/airflow/.venv/bin/python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())") +SECRET_KEY=$(openssl rand -hex 32) +ADMIN_PASS=$(openssl rand -base64 12 | tr -dc 'a-zA-Z0-9' | cut -c1-12) +cat </opt/airflow/.env +AIRFLOW_HOME=/opt/airflow +AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://${PG_DB_USER}:${PG_DB_PASS}@localhost:5432/${PG_DB_NAME} +AIRFLOW__CORE__EXECUTOR=LocalExecutor +AIRFLOW__CORE__FERNET_KEY=${FERNET_KEY} +AIRFLOW__CORE__DAGS_FOLDER=/opt/airflow/dags +AIRFLOW__CORE__LOAD_EXAMPLES=false +AIRFLOW__CORE__AUTH_MANAGER=airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager +AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session +AIRFLOW__WEBSERVER__SECRET_KEY=${SECRET_KEY} +AIRFLOW__WEBSERVER__BASE_URL=http://${LOCAL_IP}:8080 +AIRFLOW_ADMIN_PASSWORD=${ADMIN_PASS} +EOF +set -a && source /opt/airflow/.env && set +a +$STD /opt/airflow/.venv/bin/airflow db migrate +$STD /opt/airflow/.venv/bin/airflow users create \ + --username admin \ + --firstname Admin \ + --lastname User \ + --role Admin \ + --email admin@example.com \ + --password "${ADMIN_PASS}" +msg_ok "Configured Application" + +msg_info "Creating Services" +cat </etc/systemd/system/airflow-api-server.service +[Unit] +Description=Apache Airflow API Server +After=network.target postgresql.service + +[Service] +Type=simple +User=root +EnvironmentFile=/opt/airflow/.env +ExecStart=/opt/airflow/.venv/bin/airflow api-server --port 8080 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +cat </etc/systemd/system/airflow-scheduler.service +[Unit] +Description=Apache Airflow Scheduler +After=network.target postgresql.service + +[Service] +Type=simple +User=root +EnvironmentFile=/opt/airflow/.env +ExecStart=/opt/airflow/.venv/bin/airflow scheduler +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +cat </etc/systemd/system/airflow-dag-processor.service +[Unit] +Description=Apache Airflow DAG Processor +After=network.target postgresql.service + +[Service] +Type=simple +User=root +EnvironmentFile=/opt/airflow/.env +ExecStart=/opt/airflow/.venv/bin/airflow dag-processor +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +cat </etc/systemd/system/airflow-triggerer.service +[Unit] +Description=Apache Airflow Triggerer +After=network.target postgresql.service + +[Service] +Type=simple +User=root +EnvironmentFile=/opt/airflow/.env +ExecStart=/opt/airflow/.venv/bin/airflow triggerer +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF +systemctl enable -q --now airflow-api-server airflow-scheduler airflow-dag-processor airflow-triggerer +msg_ok "Created Services" + +motd_ssh +customize +cleanup_lxc