commit c33afaf8f0a8e9d641b69ced8a450fcc89657e64 Author: Sean Robinson Date: Mon Jun 15 21:43:10 2026 -0400 init diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba678ac --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.env +.env diff --git a/docker/chroma/.env.example b/docker/chroma/.env.example new file mode 100644 index 0000000..7a3f4c9 --- /dev/null +++ b/docker/chroma/.env.example @@ -0,0 +1,82 @@ +# ============================================================================= +# Chroma Vector Database Configuration +# ============================================================================= +# Docs: https://docs.trychroma.com +# ============================================================================= + +# --------------------------------------------------------------------------- +# Image Version +# --------------------------------------------------------------------------- +# Pin a specific Chroma version (e.g., 0.6.3). Leave empty or set to 'latest' +# for the latest release. +CHROMA_VERSION=latest + +# --------------------------------------------------------------------------- +# Host Port +# --------------------------------------------------------------------------- +# The port Chroma's REST API will be accessible on from the host machine. +# The internal container port is fixed at 8000. +CHROMA_PORT=8000 + +# --------------------------------------------------------------------------- +# Persistence +# --------------------------------------------------------------------------- +# Persist data to disk (bind-mounted at /mnt/user/appdata/chroma/data). +IS_PERSISTENT=TRUE +PERSIST_DIRECTORY=/chroma/chroma/ + +# --------------------------------------------------------------------------- +# Server +# --------------------------------------------------------------------------- +# Comma-separated list of origins allowed for CORS (e.g., http://localhost:3000). +# Leave empty to allow all origins. +CHROMA_SERVER_CORS_ALLOW_ORIGINS= +# Thread pool size for the HTTP server. +CHROMA_SERVER_THREAD_POOL_SIZE=40 +# Allow resetting the database via the API. Set to 'true' with caution — this +# is destructive and will delete all data. +ALLOW_RESET=false + +# --------------------------------------------------------------------------- +# Authentication (optional) +# --------------------------------------------------------------------------- +# Uncomment and configure to enable authentication. The credentials value +# depends on the provider — for the basic token provider, use: +# CHROMA_SERVER_AUTHN_CREDENTIALS=your-auth-token-here +# +# For the multi-user token provider, supply a JSON mapping of tokens to users: +# CHROMA_SERVER_AUTHN_CREDENTIALS='{"test-token-1234":"admin-user"}' +# (sensitive) +# CHROMA_SERVER_AUTHN_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider +# CHROMA_SERVER_AUTHN_CREDENTIALS= +# CHROMA_AUTH_TOKEN_TRANSPORT_HEADER=X-Chroma-Token + +# --------------------------------------------------------------------------- +# Authorization (optional) +# --------------------------------------------------------------------------- +# Uncomment to enable authorization (requires authentication to be configured). +# (sensitive) +# CHROMA_SERVER_AUTHZ_PROVIDER=chromadb.auth.simple_rbac_authz.SimpleRBACAuthorizationProvider +# CHROMA_SERVER_AUTHZ_CONFIG= + +# --------------------------------------------------------------------------- +# Telemetry +# --------------------------------------------------------------------------- +# Disable anonymous telemetry. +ANONYMIZED_TELEMETRY=false +# OpenTelemetry endpoint (e.g., http://otel-collector:4318). +CHROMA_OPEN_TELEMETRY__ENDPOINT= +CHROMA_OPEN_TELEMETRY__SERVICE_NAME=chromadb +OTEL_EXPORTER_OTLP_HEADERS= + +# --------------------------------------------------------------------------- +# Migrations +# --------------------------------------------------------------------------- +# Migration behavior: 'apply' to run migrations on startup, 'none' to skip. +MIGRATIONS=apply + +# --------------------------------------------------------------------------- +# Memory / Performance +# --------------------------------------------------------------------------- +# Memory limit in bytes for Chroma's internal cache. 0 means unlimited. +CHROMA_MEMORY_LIMIT_BYTES=0 diff --git a/docker/chroma/compose.yaml b/docker/chroma/compose.yaml new file mode 100644 index 0000000..68785c3 --- /dev/null +++ b/docker/chroma/compose.yaml @@ -0,0 +1,94 @@ +name: chroma + +services: + # =========================================================================== + # Chroma Server (vector database) + # =========================================================================== + # Chroma is the open-source embedding database. It provides a REST API for + # storing and querying vector embeddings with metadata filtering, full-text + # search, and hybrid search capabilities. + # + # VoyageAI integration: The Chroma SERVER does not need VOYAGE_API_KEY. + # VoyageAI embeddings are configured on the CLIENT side when creating + # collections and adding documents. Set VOYAGE_API_KEY in the environment + # where client tools run (e.g., the agent host, not this container). + # + # Docs: https://docs.trychroma.com + # =========================================================================== + chroma: + image: ghcr.io/chroma-core/chroma:${CHROMA_VERSION:-latest} + restart: unless-stopped + volumes: + - ./data:/chroma/chroma/ + environment: + # Persistence — hardcoded to prevent accidental data loss from .env overrides + IS_PERSISTENT: TRUE + PERSIST_DIRECTORY: /chroma/chroma/ + + # Server + CHROMA_SERVER_CORS_ALLOW_ORIGINS: ${CHROMA_SERVER_CORS_ALLOW_ORIGINS:-} + CHROMA_SERVER_THREAD_POOL_SIZE: ${CHROMA_SERVER_THREAD_POOL_SIZE:-40} + CHROMA_SERVER_HTTP_PORT: ${CHROMA_SERVER_HTTP_PORT:-8000} + # WARNING: setting ALLOW_RESET=true in .env will wipe all data on restart + ALLOW_RESET: false + + # Authentication + CHROMA_SERVER_AUTHN_PROVIDER: ${CHROMA_SERVER_AUTHN_PROVIDER:-} + CHROMA_SERVER_AUTHN_CREDENTIALS: ${CHROMA_SERVER_AUTHN_CREDENTIALS:-} + CHROMA_SERVER_AUTHZ_PROVIDER: ${CHROMA_SERVER_AUTHZ_PROVIDER:-} + CHROMA_SERVER_AUTHZ_CONFIG: ${CHROMA_SERVER_AUTHZ_CONFIG:-} + CHROMA_AUTH_TOKEN_TRANSPORT_HEADER: ${CHROMA_AUTH_TOKEN_TRANSPORT_HEADER:-} + + # Telemetry + ANONYMIZED_TELEMETRY: ${ANONYMIZED_TELEMETRY:-false} + + # Migrations + MIGRATIONS: ${MIGRATIONS:-apply} + + # Memory / Performance — sensible default for homelab (~1 GB) + # Set to 0 for unlimited (Docker memory limit still applies) + CHROMA_MEMORY_LIMIT_BYTES: ${CHROMA_MEMORY_LIMIT_BYTES:-1073741824} + ports: + - "${CHROMA_PORT:-8000}:8000" + # healthcheck: + # test: + # [ + # "CMD", + # "python", + # "-c", + # "import sys, urllib.request; r = urllib.request.urlopen('http://localhost:8000/api/v2/heartbeat'); sys.exit(0 if r.status == 200 else 1)", + # ] + # interval: 30s + # timeout: 10s + # retries: 3 + # start_period: 15s + deploy: + resources: + limits: + memory: 8G + cpus: "4" + reservations: + memory: 2G + cpus: "1" + networks: + chroma: + aliases: + - chroma + pipeline: + aliases: + - chroma + +networks: + chroma: + name: chroma + driver: bridge + pipeline: + name: pipeline + external: true + # swag: + # name: swag + # external: true + +# Data is persisted via bind mount at ./data/ on the host filesystem. +# This survives docker compose down -v and container recreation. +volumes: {} diff --git a/docker/dify/.env.example b/docker/dify/.env.example new file mode 100644 index 0000000..b12b7b0 --- /dev/null +++ b/docker/dify/.env.example @@ -0,0 +1,142 @@ +# ============================================================================= +# Dify - Environment Configuration +# ============================================================================= +# Copy this file to .env and adjust values for your deployment. +# cp .env.example .env +# The actual .env is deployed by Dockhand — do NOT commit it. +# +# To start: +# docker compose up -d +# Public URL (behind SWAG): https://dify.ld50.xyz +# ============================================================================= + +# --------------------------------------------------------------------------- +# Database (PostgreSQL) +# --------------------------------------------------------------------------- +DB_USERNAME=dify +DB_PASSWORD=your-secure-password-here +DB_DATABASE=dify + +# --------------------------------------------------------------------------- +# Cache / Queue (Redis) +# --------------------------------------------------------------------------- +REDIS_PASSWORD=your-secure-password-here + +# --------------------------------------------------------------------------- +# Security +# --------------------------------------------------------------------------- +# Generate a strong SECRET_KEY: openssl rand -base64 42 +SECRET_KEY=change-me-to-a-random-generated-key +INIT_PASSWORD=change-me-on-first-login + +# --------------------------------------------------------------------------- +# LLM Provider (Venice.ai — OpenAI-compatible) +# --------------------------------------------------------------------------- +# Dify supports any OpenAI-compatible API. Configure your provider here. +# Venice.ai is used as the default. Set your Venice API key below. +VENICE_API_KEY=your-venice-api-key-here +VENICE_API_BASE=https://api.venice.ai/api/v1 + +# --------------------------------------------------------------------------- +# Embeddings Provider (Voyage AI for Weaviate) +# --------------------------------------------------------------------------- +# Weaviate's text2vec-voyageai module reads VOYAGEAI_APIKEY at container start. +# We map this from VOYAGEAI_API_KEY in docker-compose for readability. +VOYAGEAI_API_KEY=your-voyageai-api-key-here + +# --------------------------------------------------------------------------- +# Sandbox (secure code execution) +# --------------------------------------------------------------------------- +SANDBOX_API_KEY=your-sandbox-api-key-here +SANDBOX_GIN_MODE=release +SANDBOX_WORKER_TIMEOUT=15 +SANDBOX_ENABLE_NETWORK=true + +# --------------------------------------------------------------------------- +# Plugin Daemon +# --------------------------------------------------------------------------- +PLUGIN_DAEMON_KEY=change-me-to-a-random-generated-key +PLUGIN_DIFY_INNER_API_KEY=change-me-to-a-random-generated-key +PLUGIN_DAEMON_TIMEOUT=600.0 +PLUGIN_MAX_PACKAGE_SIZE=52428800 +PLUGIN_DEBUGGING_PORT=5003 +EXPOSE_PLUGIN_DEBUGGING_HOST=localhost +FORCE_VERIFYING_SIGNATURE=false + +# --------------------------------------------------------------------------- +# Exposed Ports +# --------------------------------------------------------------------------- +# Nginx (internal app proxy container) host ports. +# Keep non-80/443 because SWAG usually owns 80/443 on the host. +EXPOSE_NGINX_PORT=8089 +EXPOSE_NGINX_SSL_PORT=8443 + +# Plugin daemon debug port (only needed when debugging plugins externally) +EXPOSE_PLUGIN_DEBUGGING_PORT=5003 + +# --------------------------------------------------------------------------- +# URL Configuration +# --------------------------------------------------------------------------- +# Set these to match your public-facing URLs if behind a reverse proxy. +# These defaults work for direct LAN access. +CONSOLE_API_URL=https://dify.ld50.xyz +CONSOLE_WEB_URL=https://dify.ld50.xyz +SERVICE_API_URL=https://dify.ld50.xyz +APP_API_URL=https://dify.ld50.xyz +APP_WEB_URL=https://dify.ld50.xyz +FILES_URL=https://dify.ld50.xyz +# Internal Docker-network URL for service-to-service file fetches +INTERNAL_FILES_URL=http://dify-api:5001 +NEXT_PUBLIC_SOCKET_URL=wss://dify.ld50.xyz + +# --------------------------------------------------------------------------- +# Storage (upstream-aligned defaults) +# --------------------------------------------------------------------------- +# Keep these unless you have a reason to change storage backend behavior. +STORAGE_TYPE=opendal +STORAGE_LOCAL_PATH=storage +OPENDAL_SCHEME=fs +OPENDAL_ROOT=storage + +# Init-permissions container chowns bind-mounted storage before API/worker start. +# 1001:1001 matches upstream Dify container user defaults. +DIFY_STORAGE_UID=1001 +DIFY_STORAGE_GID=1001 + +# --------------------------------------------------------------------------- +# Vector Store (Weaviate) +# --------------------------------------------------------------------------- +VECTOR_STORE=weaviate +WEAVIATE_ENDPOINT=http://dify-weaviate:8080 +WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih + +# Weaviate service tuning/auth defaults (must match compose service config) +WEAVIATE_PERSISTENCE_DATA_PATH=/var/lib/weaviate +WEAVIATE_QUERY_DEFAULTS_LIMIT=25 +WEAVIATE_ENABLE_MODULES=text2vec-voyageai +WEAVIATE_DEFAULT_VECTORIZER_MODULE=text2vec-voyageai +WEAVIATE_CLUSTER_HOSTNAME=node1 +WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED=false +WEAVIATE_AUTHENTICATION_APIKEY_ENABLED=true +WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih +WEAVIATE_AUTHENTICATION_APIKEY_USERS=hello@dify.ai +WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED=true +WEAVIATE_AUTHORIZATION_ADMINLIST_USERS=hello@dify.ai +WEAVIATE_DISABLE_TELEMETRY=false +WEAVIATE_ENABLE_TOKENIZER_GSE=false +WEAVIATE_ENABLE_TOKENIZER_KAGOME_JA=false +WEAVIATE_ENABLE_TOKENIZER_KAGOME_KR=false + +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- +LOG_LEVEL=INFO +DEBUG=false +FLASK_DEBUG=false + +# --------------------------------------------------------------------------- +# External Network (SWAG / reverse proxy) +# --------------------------------------------------------------------------- +# Dify nginx joins this pre-existing Docker network so SWAG can route traffic +# to service name `dify-nginx` on port 80. +NETWORKS_EXTERNAL_NAME=swag diff --git a/docker/dify/dify-sandbox-conf/config.yaml b/docker/dify/dify-sandbox-conf/config.yaml new file mode 100644 index 0000000..90f0173 --- /dev/null +++ b/docker/dify/dify-sandbox-conf/config.yaml @@ -0,0 +1,13 @@ +app: + port: 8194 + debug: false + key: dify-sandbox +max_workers: 4 +max_requests: 50 +worker_timeout: 5 +python_path: /usr/local/bin/python3 +enable_network: true +allowed_syscalls: [] +proxy: + http: http://dify-ssrf-proxy:3128 + https: http://dify-ssrf-proxy:3128 diff --git a/docker/dify/dify-sandbox-dependencies/python-requirements.txt b/docker/dify/dify-sandbox-dependencies/python-requirements.txt new file mode 100644 index 0000000..1e9d726 --- /dev/null +++ b/docker/dify/dify-sandbox-dependencies/python-requirements.txt @@ -0,0 +1,2 @@ +# Python dependencies for Dify sandbox code execution +# Add custom packages here as needed diff --git a/docker/dify/docker-compose.yaml b/docker/dify/docker-compose.yaml new file mode 100644 index 0000000..07f06d7 --- /dev/null +++ b/docker/dify/docker-compose.yaml @@ -0,0 +1,490 @@ +name: dify + +services: + # =========================================================================== + # Database + # =========================================================================== + dify-db: + image: postgres:15-alpine + restart: unless-stopped + environment: + POSTGRES_USER: ${DB_USERNAME} + POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_DB: ${DB_DATABASE} + PGDATA: /var/lib/postgresql/data/pgdata + command: > + postgres + -c 'max_connections=${POSTGRES_MAX_CONNECTIONS:-100}' + -c 'shared_buffers=${POSTGRES_SHARED_BUFFERS:-128MB}' + -c 'work_mem=${POSTGRES_WORK_MEM:-4MB}' + -c 'maintenance_work_mem=${POSTGRES_MAINTENANCE_WORK_MEM:-64MB}' + -c 'effective_cache_size=${POSTGRES_EFFECTIVE_CACHE_SIZE:-4096MB}' + volumes: + - dify-db-data:/var/lib/postgresql/data + healthcheck: + test: + [ + "CMD", + "pg_isready", + "-h", + "dify-db", + "-U", + "${DB_USERNAME:-postgres}", + "-d", + "${DB_DATABASE:-dify}", + ] + interval: 5s + timeout: 3s + retries: 30 + networks: + - dify + + # =========================================================================== + # Cache / Queue + # =========================================================================== + dify-redis: + image: redis:7-alpine + restart: unless-stopped + command: redis-server --requirepass ${REDIS_PASSWORD} + volumes: + - dify-redis-data:/data + healthcheck: + test: + [ + "CMD-SHELL", + "redis-cli -a ${REDIS_PASSWORD:-difyai123456} ping | grep -q PONG", + ] + interval: 5s + timeout: 3s + retries: 10 + networks: + - dify + + # =========================================================================== + # SSRF Proxy (sandbox traffic goes through this to prevent SSRF attacks) + # =========================================================================== + dify-ssrf-proxy: + image: ubuntu/squid:latest + restart: unless-stopped + volumes: + - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template + - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint-mount.sh + entrypoint: + [ + "sh", + "-c", + "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh", + ] + environment: + HTTP_PORT: ${SSRF_HTTP_PORT:-3128} + COREDUMP_DIR: ${SSRF_COREDUMP_DIR:-/var/spool/squid} + REVERSE_PROXY_PORT: ${SSRF_REVERSE_PROXY_PORT:-8194} + SANDBOX_HOST: dify-sandbox + SANDBOX_PORT: ${SANDBOX_PORT:-8194} + networks: + - dify + + # =========================================================================== + # Sandbox (secure Python code execution) + # =========================================================================== + dify-sandbox: + image: langgenius/dify-sandbox:0.2.15 + restart: unless-stopped + environment: + API_KEY: ${SANDBOX_API_KEY} + GIN_MODE: ${SANDBOX_GIN_MODE:-release} + WORKER_TIMEOUT: ${SANDBOX_WORKER_TIMEOUT:-15} + ENABLE_NETWORK: ${SANDBOX_ENABLE_NETWORK:-true} + HTTP_PROXY: ${SANDBOX_HTTP_PROXY:-http://dify-ssrf-proxy:3128} + HTTPS_PROXY: ${SANDBOX_HTTPS_PROXY:-http://dify-ssrf-proxy:3128} + SANDBOX_PORT: ${SANDBOX_PORT:-8194} + volumes: + - ./dify-sandbox-dependencies:/dependencies + - ./dify-sandbox-conf:/conf + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8194/health"] + interval: 30s + timeout: 5s + retries: 3 + depends_on: + dify-ssrf-proxy: + condition: service_started + networks: + - dify + + # =========================================================================== + # Init permissions (upstream parity for bind-mounted storage) + # =========================================================================== + dify-init-permissions: + image: busybox:latest + restart: "no" + command: + - sh + - -c + - | + FLAG_FILE="/app/api/storage/.init_permissions" + if [ -f "$${FLAG_FILE}" ]; then + echo "Permissions already initialized. Exiting." + exit 0 + fi + echo "Initializing permissions for /app/api/storage" + chown -R ${DIFY_STORAGE_UID:-1001}:${DIFY_STORAGE_GID:-1001} /app/api/storage && touch "$${FLAG_FILE}" + echo "Permissions initialized. Exiting." + volumes: + - ./storage:/app/api/storage + + # =========================================================================== + # API (backend) + # =========================================================================== + dify-api: + image: langgenius/dify-api:1.14.2 + restart: unless-stopped + depends_on: + dify-init-permissions: + condition: service_completed_successfully + dify-db: + condition: service_healthy + dify-redis: + condition: service_healthy + dify-plugin-daemon: + condition: service_started + dify-weaviate: + condition: service_started + environment: + MODE: api + + # Core + SECRET_KEY: ${SECRET_KEY} + INIT_PASSWORD: ${INIT_PASSWORD} + DEPLOY_ENV: ${DEPLOY_ENV:-PRODUCTION} + MIGRATION_ENABLED: ${MIGRATION_ENABLED:-true} + + # Database + DB_HOST: dify-db + DB_PORT: 5432 + DB_USERNAME: ${DB_USERNAME} + DB_PASSWORD: ${DB_PASSWORD} + DB_DATABASE: ${DB_DATABASE} + + # Redis + REDIS_HOST: dify-redis + REDIS_PORT: 6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + REDIS_USE_SSL: ${REDIS_USE_SSL:-false} + + # Storage + STORAGE_TYPE: ${STORAGE_TYPE:-opendal} + STORAGE_LOCAL_PATH: ${STORAGE_LOCAL_PATH:-storage} + OPENDAL_SCHEME: ${OPENDAL_SCHEME:-fs} + OPENDAL_ROOT: ${OPENDAL_ROOT:-storage} + + # Vector Store + VECTOR_STORE: ${VECTOR_STORE:-weaviate} + WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://dify-weaviate:8080} + WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} + + # URLs + CONSOLE_API_URL: ${CONSOLE_API_URL:-} + CONSOLE_WEB_URL: ${CONSOLE_WEB_URL:-} + SERVICE_API_URL: ${SERVICE_API_URL:-} + APP_API_URL: ${APP_API_URL:-} + APP_WEB_URL: ${APP_WEB_URL:-} + FILES_URL: ${FILES_URL:-} + INTERNAL_FILES_URL: ${INTERNAL_FILES_URL:-} + + # LLM Provider (Venice.ai — OpenAI-compatible) + OPENAI_API_KEY: ${VENICE_API_KEY:-} + OPENAI_API_BASE: ${VENICE_API_BASE:-https://api.venice.ai/api/v1} + + # Logging + LOG_LEVEL: ${LOG_LEVEL:-INFO} + DEBUG: ${DEBUG:-false} + FLASK_DEBUG: ${FLASK_DEBUG:-false} + + # Plugins + PLUGIN_DAEMON_KEY: ${PLUGIN_DAEMON_KEY} + PLUGIN_DAEMON_URL: http://dify-plugin-daemon:5002 + PLUGIN_DAEMON_TIMEOUT: ${PLUGIN_DAEMON_TIMEOUT:-600.0} + PLUGIN_REMOTE_INSTALL_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost} + PLUGIN_REMOTE_INSTALL_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003} + PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PLUGIN_DIFY_INNER_API_KEY: ${PLUGIN_DIFY_INNER_API_KEY} + PLUGIN_DIFY_INNER_API_URL: http://dify-api:5001 + INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY} + FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-false} + volumes: + - ./storage:/app/api/storage + networks: + - dify + + # =========================================================================== + # Worker (Celery background worker) + # =========================================================================== + dify-worker: + image: langgenius/dify-api:1.14.2 + restart: unless-stopped + depends_on: + dify-init-permissions: + condition: service_completed_successfully + dify-db: + condition: service_healthy + dify-redis: + condition: service_healthy + dify-weaviate: + condition: service_started + environment: + MODE: worker + + # Core + SECRET_KEY: ${SECRET_KEY} + DEPLOY_ENV: ${DEPLOY_ENV:-PRODUCTION} + + # Database + DB_HOST: dify-db + DB_PORT: 5432 + DB_USERNAME: ${DB_USERNAME} + DB_PASSWORD: ${DB_PASSWORD} + DB_DATABASE: ${DB_DATABASE} + + # Redis + REDIS_HOST: dify-redis + REDIS_PORT: 6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + REDIS_USE_SSL: ${REDIS_USE_SSL:-false} + + # Storage + STORAGE_TYPE: ${STORAGE_TYPE:-opendal} + STORAGE_LOCAL_PATH: ${STORAGE_LOCAL_PATH:-storage} + OPENDAL_SCHEME: ${OPENDAL_SCHEME:-fs} + OPENDAL_ROOT: ${OPENDAL_ROOT:-storage} + + # Vector Store + VECTOR_STORE: ${VECTOR_STORE:-weaviate} + WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://dify-weaviate:8080} + WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} + + # LLM Provider (Venice.ai — OpenAI-compatible) + OPENAI_API_KEY: ${VENICE_API_KEY:-} + OPENAI_API_BASE: ${VENICE_API_BASE:-https://api.venice.ai/api/v1} + + # Logging + LOG_LEVEL: ${LOG_LEVEL:-INFO} + DEBUG: ${DEBUG:-false} + + # Plugins + PLUGIN_DAEMON_KEY: ${PLUGIN_DAEMON_KEY} + PLUGIN_DAEMON_URL: http://dify-plugin-daemon:5002 + PLUGIN_REMOTE_INSTALL_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost} + PLUGIN_REMOTE_INSTALL_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003} + PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PLUGIN_DIFY_INNER_API_KEY: ${PLUGIN_DIFY_INNER_API_KEY} + PLUGIN_DIFY_INNER_API_URL: http://dify-api:5001 + INNER_API_KEY_FOR_PLUGIN: ${PLUGIN_DIFY_INNER_API_KEY} + FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-false} + + # Celery + CELERY_BROKER_URL: redis://:${REDIS_PASSWORD}@dify-redis:6379/0 + CELERY_RESULT_BACKEND: redis://:${REDIS_PASSWORD}@dify-redis:6379/0 + volumes: + - ./storage:/app/api/storage + healthcheck: + test: + [ + "CMD-SHELL", + "celery -A celery_healthcheck.celery inspect ping", + ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + networks: + - dify + + # =========================================================================== + # Worker Beat (Celery periodic task scheduler) + # =========================================================================== + dify-worker-beat: + image: langgenius/dify-api:1.14.2 + restart: unless-stopped + depends_on: + dify-init-permissions: + condition: service_completed_successfully + dify-db: + condition: service_healthy + dify-redis: + condition: service_healthy + environment: + MODE: beat + + # Core + SECRET_KEY: ${SECRET_KEY} + DEPLOY_ENV: ${DEPLOY_ENV:-PRODUCTION} + + # Database + DB_HOST: dify-db + DB_PORT: 5432 + DB_USERNAME: ${DB_USERNAME} + DB_PASSWORD: ${DB_PASSWORD} + DB_DATABASE: ${DB_DATABASE} + + # Redis + REDIS_HOST: dify-redis + REDIS_PORT: 6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + REDIS_USE_SSL: ${REDIS_USE_SSL:-false} + + # Celery + CELERY_BROKER_URL: redis://:${REDIS_PASSWORD}@dify-redis:6379/0 + CELERY_RESULT_BACKEND: redis://:${REDIS_PASSWORD}@dify-redis:6379/0 + networks: + - dify + + # =========================================================================== + # Plugin Daemon + # =========================================================================== + dify-plugin-daemon: + image: langgenius/dify-plugin-daemon:0.6.1-local + restart: unless-stopped + depends_on: + dify-db: + condition: service_healthy + dify-redis: + condition: service_healthy + environment: + # Database + DB_HOST: dify-db + DB_PORT: 5432 + DB_USERNAME: ${DB_USERNAME} + DB_PASSWORD: ${DB_PASSWORD} + DB_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} + DB_SSL_MODE: ${DB_SSL_MODE:-disable} + + # Redis + REDIS_HOST: dify-redis + REDIS_PORT: 6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + + # Plugin Daemon + SERVER_PORT: ${PLUGIN_DAEMON_PORT:-5002} + SERVER_KEY: ${PLUGIN_DAEMON_KEY} + MAX_PLUGIN_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} + PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false} + + # Dify API connection + DIFY_INNER_API_KEY: ${PLUGIN_DIFY_INNER_API_KEY} + DIFY_INNER_API_URL: http://dify-api:5001 + + # Debugging + PLUGIN_REMOTE_INSTALLING_HOST: ${PLUGIN_DEBUGGING_HOST:-0.0.0.0} + PLUGIN_REMOTE_INSTALLING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003} + + # Storage + PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} + PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} + PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} + PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} + PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} + PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} + + # Python + PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} + PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} + + # Misc + FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} + + # Plugin stdio + PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024} + PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880} + PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} + volumes: + - ./storage-plugin-daemon:/app/storage + ports: + - ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003}:${PLUGIN_DEBUGGING_PORT:-5003} + networks: + - dify + + # =========================================================================== + # Web (Next.js frontend) + # =========================================================================== + dify-web: + image: langgenius/dify-web:1.14.2 + restart: unless-stopped + environment: + CONSOLE_API_URL: ${CONSOLE_API_URL:-} + SERVER_CONSOLE_API_URL: http://dify-api:5001 + APP_API_URL: ${APP_API_URL:-} + NEXT_PUBLIC_SOCKET_URL: ${NEXT_PUBLIC_SOCKET_URL:-} + NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-1} + TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} + CSP_WHITELIST: ${CSP_WHITELIST:-} + ALLOW_EMBED: ${ALLOW_EMBED:-false} + MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} + MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai} + depends_on: + - dify-api + networks: + - dify + + # =========================================================================== + # Nginx (reverse proxy) + # =========================================================================== + dify-nginx: + image: nginx:alpine + restart: unless-stopped + depends_on: + - dify-api + - dify-web + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf + - ./nginx/proxy.conf:/etc/nginx/proxy.conf + - ./nginx/conf.d:/etc/nginx/conf.d + - ./storage:/app/storage:ro + ports: + - ${EXPOSE_NGINX_PORT:-80}:80 + - ${EXPOSE_NGINX_SSL_PORT:-443}:443 + networks: + - dify + - external_network + + # =========================================================================== + # Vector Store (Weaviate) + # =========================================================================== + dify-weaviate: + image: semitechnologies/weaviate:1.27.0 + restart: unless-stopped + volumes: + - ./dify-weaviate-data:/var/lib/weaviate + environment: + PERSISTENCE_DATA_PATH: ${WEAVIATE_PERSISTENCE_DATA_PATH:-/var/lib/weaviate} + QUERY_DEFAULTS_LIMIT: ${WEAVIATE_QUERY_DEFAULTS_LIMIT:-25} + ENABLE_MODULES: ${WEAVIATE_ENABLE_MODULES:-text2vec-voyageai} + DEFAULT_VECTORIZER_MODULE: ${WEAVIATE_DEFAULT_VECTORIZER_MODULE:-text2vec-voyageai} + VOYAGEAI_APIKEY: ${VOYAGEAI_API_KEY:-} + CLUSTER_HOSTNAME: ${WEAVIATE_CLUSTER_HOSTNAME:-node1} + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: ${WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED:-false} + AUTHENTICATION_APIKEY_ENABLED: ${WEAVIATE_AUTHENTICATION_APIKEY_ENABLED:-true} + AUTHENTICATION_APIKEY_ALLOWED_KEYS: ${WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} + AUTHENTICATION_APIKEY_USERS: ${WEAVIATE_AUTHENTICATION_APIKEY_USERS:-hello@dify.ai} + AUTHORIZATION_ADMINLIST_ENABLED: ${WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED:-true} + AUTHORIZATION_ADMINLIST_USERS: ${WEAVIATE_AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai} + DISABLE_TELEMETRY: ${WEAVIATE_DISABLE_TELEMETRY:-false} + ENABLE_TOKENIZER_GSE: ${WEAVIATE_ENABLE_TOKENIZER_GSE:-false} + ENABLE_TOKENIZER_KAGOME_JA: ${WEAVIATE_ENABLE_TOKENIZER_KAGOME_JA:-false} + ENABLE_TOKENIZER_KAGOME_KR: ${WEAVIATE_ENABLE_TOKENIZER_KAGOME_KR:-false} + networks: + - dify + +networks: + dify: + name: dify + # driver: bridge + external_network: + name: ${NETWORKS_EXTERNAL_NAME:-swag} + external: true + +volumes: + dify-db-data: + driver: local + dify-redis-data: + driver: local diff --git a/docker/dify/nginx/conf.d/default.conf b/docker/dify/nginx/conf.d/default.conf new file mode 100644 index 0000000..72c1c1c --- /dev/null +++ b/docker/dify/nginx/conf.d/default.conf @@ -0,0 +1,63 @@ +server { + listen 80; + server_name _; + + # API routes + location /console/api { + proxy_pass http://dify-api:5001; + include /etc/nginx/proxy.conf; + } + + location /api { + proxy_pass http://dify-api:5001; + include /etc/nginx/proxy.conf; + } + + location /v1 { + proxy_pass http://dify-api:5001; + include /etc/nginx/proxy.conf; + } + + location /files { + proxy_pass http://dify-api:5001; + include /etc/nginx/proxy.conf; + } + + location /mcp { + proxy_pass http://dify-api:5001; + include /etc/nginx/proxy.conf; + } + + location /triggers { + proxy_pass http://dify-api:5001; + include /etc/nginx/proxy.conf; + } + + # WebSocket + location /socket.io/ { + proxy_pass http://dify-api:5001; + include /etc/nginx/proxy.conf; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_cache_bypass $http_upgrade; + } + + # Explore / shared apps + location /explore { + proxy_pass http://dify-web:3000; + include /etc/nginx/proxy.conf; + } + + # Plugin endpoints + location /e/ { + proxy_pass http://dify-plugin-daemon:5002; + proxy_set_header Dify-Hook-Url $scheme://$host$request_uri; + include /etc/nginx/proxy.conf; + } + + # Web frontend (Next.js) + location / { + proxy_pass http://dify-web:3000; + include /etc/nginx/proxy.conf; + } +} diff --git a/docker/dify/nginx/nginx.conf b/docker/dify/nginx/nginx.conf new file mode 100644 index 0000000..e59247d --- /dev/null +++ b/docker/dify/nginx/nginx.conf @@ -0,0 +1,26 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + keepalive_timeout 65; + client_max_body_size 100M; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/docker/dify/nginx/proxy.conf b/docker/dify/nginx/proxy.conf new file mode 100644 index 0000000..c7a0f7a --- /dev/null +++ b/docker/dify/nginx/proxy.conf @@ -0,0 +1,9 @@ +proxy_set_header Host $host; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; +proxy_set_header X-Forwarded-Port $server_port; +proxy_http_version 1.1; +proxy_set_header Connection ""; +proxy_buffering off; +proxy_read_timeout 3600s; +proxy_send_timeout 3600s; diff --git a/docker/dify/ssrf_proxy/docker-entrypoint.sh b/docker/dify/ssrf_proxy/docker-entrypoint.sh new file mode 100755 index 0000000..613897b --- /dev/null +++ b/docker/dify/ssrf_proxy/docker-entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Modified based on Squid OCI image entrypoint + +# This entrypoint aims to forward the squid logs to stdout to assist users of +# common container related tooling (e.g., kubernetes, docker-compose, etc) to +# access the service logs. + +# Moreover, it invokes the squid binary, leaving all the desired parameters to +# be provided by the "command" passed to the spawned container. If no command +# is provided by the user, the default behavior (as per the CMD statement in +# the Dockerfile) will be to use Ubuntu's default configuration [1] and run +# squid with the "-NYC" options to mimic the behavior of the Ubuntu provided +# systemd unit. + +# [1] The default configuration is changed in the Dockerfile to allow local +# network connections. See the Dockerfile for further information. + +echo "[ENTRYPOINT] re-create snakeoil self-signed certificate removed in the build process" +if [ ! -f /etc/ssl/private/ssl-cert-snakeoil.key ]; then + /usr/sbin/make-ssl-cert generate-default-snakeoil --force-overwrite > /dev/null 2>&1 +fi + +tail -F /var/log/squid/access.log 2>/dev/null & +tail -F /var/log/squid/error.log 2>/dev/null & +tail -F /var/log/squid/store.log 2>/dev/null & +tail -F /var/log/squid/cache.log 2>/dev/null & + +# Replace environment variables in the template and output to the squid.conf +echo "[ENTRYPOINT] replacing environment variables in the template" +awk '{ + while(match($0, /\${[A-Za-z_][A-Za-z_0-9]*}/)) { + var = substr($0, RSTART+2, RLENGTH-3) + val = ENVIRON[var] + $0 = substr($0, 1, RSTART-1) val substr($0, RSTART+RLENGTH) + } + print +}' /etc/squid/squid.conf.template > /etc/squid/squid.conf + +/usr/sbin/squid -Nz +echo "[ENTRYPOINT] starting squid" +/usr/sbin/squid -f /etc/squid/squid.conf -NYC 1 diff --git a/docker/dify/ssrf_proxy/squid.conf.template b/docker/dify/ssrf_proxy/squid.conf.template new file mode 100644 index 0000000..fbe9ebc --- /dev/null +++ b/docker/dify/ssrf_proxy/squid.conf.template @@ -0,0 +1,106 @@ +acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) +acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) +acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) +acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) +acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines +acl SSL_ports port 443 +# acl SSL_ports port 1025-65535 # Enable the configuration to resolve this issue: https://github.com/langgenius/dify/issues/12792 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT +acl allowed_domains dstdomain .marketplace.dify.ai +http_access allow allowed_domains +http_access deny !Safe_ports +http_access deny CONNECT !SSL_ports +http_access allow localhost manager +http_access deny manager +http_access allow localhost +include /etc/squid/conf.d/*.conf +http_access deny all +tcp_outgoing_address 0.0.0.0 + +################################## Proxy Server ################################ +http_port ${HTTP_PORT} +coredump_dir ${COREDUMP_DIR} +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims +refresh_pattern \/InRelease$ 0 0% 0 refresh-ims +refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern . 0 20% 4320 + + +# cache_dir ufs /var/spool/squid 100 16 256 +# upstream proxy, set to your own upstream proxy IP to avoid SSRF attacks +# cache_peer 172.1.1.1 parent 3128 0 no-query no-digest no-netdb-exchange default + +################################## Reverse Proxy To Sandbox ################################ +http_port ${REVERSE_PROXY_PORT} accel vhost +cache_peer ${SANDBOX_HOST} parent ${SANDBOX_PORT} 0 no-query originserver +acl src_all src all +http_access allow src_all + +# Unless the option's size is increased, an error will occur when uploading more than two files. +client_request_buffer_max_size 100 MB + +################################## Performance & Concurrency ############################### +# Increase file descriptor limit for high concurrency +max_filedescriptors 65536 + +# Timeout configurations for image requests +connect_timeout 30 seconds +request_timeout 2 minutes +read_timeout 2 minutes +client_lifetime 5 minutes +shutdown_lifetime 30 seconds + +# Persistent connections - improve performance for multiple requests +server_persistent_connections on +client_persistent_connections on +persistent_request_timeout 30 seconds +pconn_timeout 1 minute + +# Connection pool and concurrency limits +client_db on +server_idle_pconn_timeout 2 minutes +client_idle_pconn_timeout 2 minutes + +# Quick abort settings - don't abort requests that are mostly done +quick_abort_min 16 KB +quick_abort_max 16 MB +quick_abort_pct 95 + +# Memory and cache optimization +memory_cache_mode disk +cache_mem 256 MB +maximum_object_size_in_memory 512 KB + +# DNS resolver settings for better performance +dns_timeout 30 seconds +dns_retransmit_interval 5 seconds +# By default, Squid uses the system's configured DNS resolvers. +# If you need to override them, set dns_nameservers to appropriate servers +# for your environment (for example, internal/corporate DNS). The following +# is an example using public DNS and SHOULD be customized before use: +# dns_nameservers 8.8.8.8 8.8.4.4 + +# Logging format for better debugging +logformat dify_log %ts.%03tu %6tr %>a %Ss/%03>Hs %/dev/null && exec headroom proxy "$0" "$@"', + ] + restart: unless-stopped + env_file: + - .env + environment: + HOME: /tmp/headroom-home + HEADROOM_HOST: 0.0.0.0 + HEADROOM_WORKSPACE_DIR: /tmp/headroom-home/.headroom + HEADROOM_CONFIG_DIR: /tmp/headroom-home/.headroom/config + HEADROOM_TELEMETRY: ${HEADROOM_TELEMETRY:-off} + HEADROOM_LOG_LEVEL: ${HEADROOM_LOG_LEVEL:-INFO} + # Langfuse / OpenTelemetry tracing + # Keys (LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY) come from .env via env_file + LANGFUSE_HOST: http://langfuse-web:3000 + LANGFUSE_PUBLIC_KEY: ${LANGFUSE_PUBLIC_KEY} + LANGFUSE_SECRET_KEY: ${LANGFUSE_SECRET_KEY} + OTEL_SERVICE_NAME: headroom-proxy + OTEL_EXPORTER_OTLP_ENDPOINT: http://langfuse-web:3000/api/public/otel/v1 + ports: + - "${HEADROOM_PORT:-8787}:${HEADROOM_PORT:-8787}" + volumes: + # Headroom persistent data (memories, config, cache, learn artifacts) + - ${HEADROOM_HOST_HOME:?set HEADROOM_HOST_HOME}/.headroom:/tmp/headroom-home/.headroom + # Agent-specific directories are NOT mounted here — they're for the + # CLI / wrap mode. The proxy only needs the shared .headroom dir. + command: + - "--host" + - "0.0.0.0" + - "--port" + - "${HEADROOM_PORT:-8787}" + - "--openai-api-url" + - "${OPENAI_TARGET_API_URL:-https://api.venice.ai/v1}" + - "--memory" + - "--memory-db-path" + - "/tmp/headroom-home/.headroom/memory.db" + # Traffic learning — writes AGENTS.md / MEMORY.md with failure + # patterns mined from agent sessions + - "--learn" + # healthcheck: + # test: ["CMD", "headroom", "--help"] + # interval: 30s + # timeout: 10s + # retries: 3 + # start_period: 15s + deploy: + resources: + limits: + memory: 8G + cpus: "8" + reservations: + memory: 4G + cpus: "2" + networks: + headroom: {} + langfuse: + aliases: + - headroom-proxy + pipeline: + aliases: + - headroom-proxy + +networks: + headroom: + name: headroom + driver: bridge + langfuse: + name: langfuse_langfuse + external: true + pipeline: + name: pipeline + external: true diff --git a/docker/lgtm/.env.example b/docker/lgtm/.env.example new file mode 100644 index 0000000..7b7d354 --- /dev/null +++ b/docker/lgtm/.env.example @@ -0,0 +1,65 @@ +# ============================================================================= +# otel-lgtm - OpenTelemetry Backend (Grafana, Prometheus, Tempo, Loki, Pyroscope) +# ============================================================================= +# Image: grafana/otel-lgtm +# Docs: https://github.com/grafana/docker-otel-lgtm +# ============================================================================= + +# -- Image Tag ----------------------------------------------------------------- +OTEL_LGTM_VERSION=latest + +# -- Grafana Admin Credentials ------------------------------------------------- +GF_ADMIN_USER=admin +GF_ADMIN_PASSWORD=admin + +# -- Exposed Ports ------------------------------------------------------------- +# Grafana dashboard +EXPOSE_GRAFANA_PORT=3000 +# OpenTelemetry Collector - OTLP gRPC (used by instrumented apps/services) +EXPOSE_OTLP_GRPC_PORT=4317 +# OpenTelemetry Collector - OTLP HTTP (used by instrumented apps/services) +EXPOSE_OTLP_HTTP_PORT=4318 +# Prometheus metrics UI +EXPOSE_PROMETHEUS_PORT=9090 +# Tempo HTTP query endpoint +EXPOSE_TEMPO_HTTP_PORT=3200 +# Pyroscope continuous profiling UI +EXPOSE_PYROSCOPE_PORT=4040 + +# -- Verbose Container Logging ------------------------------------------------- +# Set any of these to "true" to enable verbose logs for that component. +# These are component-internal logs (not the application telemetry data). +ENABLE_LOGS_GRAFANA=false +ENABLE_LOGS_LOKI=false +ENABLE_LOGS_PROMETHEUS=false +ENABLE_LOGS_TEMPO=false +ENABLE_LOGS_PYROSCOPE=false +ENABLE_LOGS_OTELCOL=false +# Override: enable ALL component logging at once +ENABLE_LOGS_ALL=false + +# -- eBPF Auto-Instrumentation (OBI) ------------------------------------------ +# Requires Linux kernel 5.8+ with BTF support. Also requires privileged mode +# and host PID namespace. If enabled, uncomment the `cap_add`, `network_mode`, +# and `pid` lines in docker-compose.yaml. +ENABLE_OBI=false + +# -- External OTLP Forwarding -------------------------------------------------- +# Forward telemetry to an external OTLP-compatible backend (e.g., Grafana Cloud). +# Leave blank to send data only to the local LGTM stack. +OTEL_EXPORTER_OTLP_ENDPOINT= +OTEL_EXPORTER_OTLP_HEADERS= + +# -- Tempo MCP Server --------------------------------------------------------- +# Enables the Tempo MCP server for AI tool integration (e.g., Cline, Claude Code). +# Set to "--query-frontend.mcp-server.enabled=true" to enable. +TEMPO_EXTRA_ARGS= + +# -- Per-Backend Extra CLI Args ----------------------------------------------- +# Additional command-line flags for individual backends. +# Example: PROMETHEUS_EXTRA_ARGS="--storage.tsdb.retention.time=90d" +PROMETHEUS_EXTRA_ARGS= +LOKI_EXTRA_ARGS= +TEMPO_EXTRA_ARGS_V2= +PYROSCOPE_EXTRA_ARGS= +OTELCOL_EXTRA_ARGS= diff --git a/docker/lgtm/docker-compose.yaml b/docker/lgtm/docker-compose.yaml new file mode 100644 index 0000000..7f4acb5 --- /dev/null +++ b/docker/lgtm/docker-compose.yaml @@ -0,0 +1,94 @@ +services: + lgtm: + image: grafana/otel-lgtm:${OTEL_LGTM_VERSION:-latest} + restart: unless-stopped + environment: + # Grafana admin credentials + GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASSWORD:-admin} + + # Enable verbose container logging for specific components + ENABLE_LOGS_GRAFANA: ${ENABLE_LOGS_GRAFANA:-false} + ENABLE_LOGS_LOKI: ${ENABLE_LOGS_LOKI:-false} + ENABLE_LOGS_PROMETHEUS: ${ENABLE_LOGS_PROMETHEUS:-false} + ENABLE_LOGS_TEMPO: ${ENABLE_LOGS_TEMPO:-false} + ENABLE_LOGS_PYROSCOPE: ${ENABLE_LOGS_PYROSCOPE:-false} + ENABLE_LOGS_OTELCOL: ${ENABLE_LOGS_OTELCOL:-false} + ENABLE_LOGS_ALL: ${ENABLE_LOGS_ALL:-false} + + # eBPF auto-instrumentation (OBI) + # Requires Linux kernel 5.8+ with BTF support + ENABLE_OBI: ${ENABLE_OBI:-false} + + # Forward telemetry to an external OTLP endpoint (e.g., Grafana Cloud) + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} + OTEL_EXPORTER_OTLP_HEADERS: ${OTEL_EXPORTER_OTLP_HEADERS:-} + + # Enable Tempo MCP server for AI tool integration + TEMPO_EXTRA_ARGS: ${TEMPO_EXTRA_ARGS:-} + + # Extra CLI args for individual backends (optional) + PROMETHEUS_EXTRA_ARGS: ${PROMETHEUS_EXTRA_ARGS:-} + LOKI_EXTRA_ARGS: ${LOKI_EXTRA_ARGS:-} + TEMPO_EXTRA_ARGS_V2: ${TEMPO_EXTRA_ARGS_V2:-} + PYROSCOPE_EXTRA_ARGS: ${PYROSCOPE_EXTRA_ARGS:-} + OTELCOL_EXTRA_ARGS: ${OTELCOL_EXTRA_ARGS:-} + + ports: + # Grafana dashboard + - ${EXPOSE_GRAFANA_PORT:-3000}:3000 + # OpenTelemetry Collector - OTLP gRPC + - ${EXPOSE_OTLP_GRPC_PORT:-4317}:4317 + # OpenTelemetry Collector - OTLP HTTP + - ${EXPOSE_OTLP_HTTP_PORT:-4318}:4318 + # Prometheus + - ${EXPOSE_PROMETHEUS_PORT:-9090}:9090 + # Tempo HTTP query endpoint + - ${EXPOSE_TEMPO_HTTP_PORT:-3200}:3200 + # Pyroscope + - ${EXPOSE_PYROSCOPE_PORT:-4040}:4040 + + volumes: + # Persistent storage for all backend data + - ./lgtm-data:/data + + # If OBI (eBPF) is enabled, these capabilities are needed + # Uncomment the cap_add and network_mode when ENABLE_OBI=true + # cap_add: + # - SYS_ADMIN + # - BPF + # network_mode: host + # pid: host + + networks: + - lgtm + - pipeline + - swag + + # healthcheck: + # test: + # [ + # "CMD", + # "wget", + # "--no-verbose", + # "--tries=1", + # "--spider", + # "http://localhost:3000/api/health", + # ] + # interval: 15s + # timeout: 5s + # retries: 10 + # start_period: 30s + +networks: + lgtm: + name: lgtm + driver: bridge + pipeline: + name: pipeline + external: true + swag: + name: swag + external: true + +volumes: {} diff --git a/docker/lgtm/swag/lgtm.subdomain.conf b/docker/lgtm/swag/lgtm.subdomain.conf new file mode 100644 index 0000000..a590c2e --- /dev/null +++ b/docker/lgtm/swag/lgtm.subdomain.conf @@ -0,0 +1,40 @@ +## ----------------------------------------------------------------------------- +## SWAG proxy config for otel-lgtm +## Domain: lgtm.ld50.xyz +## Upstream: lgtm:3000 (shared Docker network: ${NETWORKS_EXTERNAL_NAME:-swag}) +## +## Install: +## 1) Copy this file into SWAG: /config/nginx/proxy-confs/lgtm.subdomain.conf +## 2) Ensure both stacks share the same external Docker network (e.g. `swag`). +## 3) In curated_compose/lgtm/docker-compose.yaml, uncomment external_network. +## 4) Reload SWAG. +## ----------------------------------------------------------------------------- + +server { + listen 443 ssl; + listen [::]:443 ssl; + + server_name lgtm.*; + + include /config/nginx/ssl.conf; + + # Grafana dashboards can embed large panels / JSON + client_max_body_size 20M; + + location / { + include /config/nginx/proxy.conf; + + set $upstream_app lgtm; + set $upstream_port 3000; + set $upstream_proto http; + + proxy_pass $upstream_proto://$upstream_app:$upstream_port; + + # Required for Grafana live queries and real-time dashboard updates + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + } +} diff --git a/docker/n8n/.env.example b/docker/n8n/.env.example new file mode 100644 index 0000000..c91cc06 --- /dev/null +++ b/docker/n8n/.env.example @@ -0,0 +1,52 @@ +# =========================================================================== +# n8n stack environment +# Domain target: https://n8n.ld50.xyz +# =========================================================================== + +# n8n image tag (use 'stable' for production) +N8N_VERSION=stable + +# Host port for n8n main UI/API +EXPOSE_N8N_PORT=5678 + +# Runtime +NODE_ENV=production +GENERIC_TIMEZONE=America/Chicago +N8N_LOG_LEVEL=info + +# Public URL config +N8N_HOST=n8n.ld50.xyz +N8N_PROTOCOL=https +N8N_PORT=5678 +N8N_EDITOR_BASE_URL=https://n8n.ld50.xyz +WEBHOOK_URL=https://n8n.ld50.xyz/ + +# Security: REQUIRED +# Use a long random value and keep it identical across main + workers. +N8N_ENCRYPTION_KEY=replace-with-a-long-random-string + +# Queue mode (recommended for production) +EXECUTIONS_MODE=queue +N8N_WORKER_CONCURRENCY=10 +QUEUE_HEALTH_CHECK_ACTIVE=true + +# PostgreSQL +DB_HOST=n8n-db +DB_PORT=5432 +DB_DATABASE=n8n +DB_USERNAME=n8n +DB_PASSWORD=change-me-to-a-strong-db-password + +# Redis +QUEUE_BULL_REDIS_HOST=n8n-redis +QUEUE_BULL_REDIS_PORT=6379 +REDIS_PASSWORD=change-me-to-a-strong-redis-password + +# Optional hardening / noise reduction +N8N_DIAGNOSTICS_ENABLED=false +N8N_VERSION_NOTIFICATIONS_ENABLED=false +N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=true + +# Optional shared external Docker network (for SWAG) +# Keep compose network stanza commented out until needed. +NETWORKS_EXTERNAL_NAME=swag diff --git a/docker/n8n/.gitignore b/docker/n8n/.gitignore new file mode 100644 index 0000000..80a78a5 --- /dev/null +++ b/docker/n8n/.gitignore @@ -0,0 +1,8 @@ +# Runtime/persistent data +n8n-db-data/ +n8n-redis-data/ +n8n-data/ +n8n-files/ + +# Local secrets +.env diff --git a/docker/n8n/docker-compose.yaml b/docker/n8n/docker-compose.yaml new file mode 100644 index 0000000..da51e2e --- /dev/null +++ b/docker/n8n/docker-compose.yaml @@ -0,0 +1,160 @@ +name: n8n + +services: + # =========================================================================== + # n8n Postgres (required for reliable production deployments) + # =========================================================================== + n8n-db: + image: postgres:15-alpine + restart: unless-stopped + environment: + POSTGRES_USER: ${DB_USERNAME} + POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_DB: ${DB_DATABASE} + volumes: + - ./n8n-db-data:/var/lib/postgresql/data + healthcheck: + test: + [ + "CMD", + "pg_isready", + "-h", + "n8n-db", + "-U", + "${DB_USERNAME:-n8n}", + "-d", + "${DB_DATABASE:-n8n}", + ] + interval: 5s + timeout: 3s + retries: 30 + networks: + - n8n + + # =========================================================================== + # n8n Redis (queue broker) + # =========================================================================== + n8n-redis: + image: redis:7-alpine + restart: unless-stopped + command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD} + volumes: + - ./n8n-redis-data:/data + healthcheck: + test: + [ + "CMD-SHELL", + "redis-cli -a ${REDIS_PASSWORD:-change-me} ping | grep -q PONG", + ] + interval: 5s + timeout: 3s + retries: 30 + networks: + - n8n + + # =========================================================================== + # n8n main process (UI + API + scheduler) + # =========================================================================== + n8n-main: + image: docker.n8n.io/n8nio/n8n:${N8N_VERSION:-stable} + restart: unless-stopped + depends_on: + n8n-db: + condition: service_healthy + n8n-redis: + condition: service_healthy + ports: + - ${EXPOSE_N8N_PORT:-5678}:5678 + environment: + NODE_ENV: ${NODE_ENV:-production} + TZ: ${GENERIC_TIMEZONE:-UTC} + GENERIC_TIMEZONE: ${GENERIC_TIMEZONE:-UTC} + N8N_LOG_LEVEL: ${N8N_LOG_LEVEL:-info} + N8N_DIAGNOSTICS_ENABLED: ${N8N_DIAGNOSTICS_ENABLED:-false} + N8N_VERSION_NOTIFICATIONS_ENABLED: ${N8N_VERSION_NOTIFICATIONS_ENABLED:-false} + N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS: ${N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS:-true} + N8N_ENCRYPTION_KEY: ${N8N_ENCRYPTION_KEY} + N8N_HOST: ${N8N_HOST:-n8n.ld50.xyz} + N8N_PROTOCOL: ${N8N_PROTOCOL:-https} + N8N_PORT: ${N8N_PORT:-5678} + N8N_EDITOR_BASE_URL: ${N8N_EDITOR_BASE_URL:-https://n8n.ld50.xyz} + WEBHOOK_URL: ${WEBHOOK_URL:-https://n8n.ld50.xyz/} + EXECUTIONS_MODE: ${EXECUTIONS_MODE:-queue} + DB_TYPE: postgresdb + DB_POSTGRESDB_HOST: ${DB_HOST:-n8n-db} + DB_POSTGRESDB_PORT: ${DB_PORT:-5432} + DB_POSTGRESDB_DATABASE: ${DB_DATABASE:-n8n} + DB_POSTGRESDB_USER: ${DB_USERNAME:-n8n} + DB_POSTGRESDB_PASSWORD: ${DB_PASSWORD} + QUEUE_BULL_REDIS_HOST: ${QUEUE_BULL_REDIS_HOST:-n8n-redis} + QUEUE_BULL_REDIS_PORT: ${QUEUE_BULL_REDIS_PORT:-6379} + QUEUE_BULL_REDIS_PASSWORD: ${REDIS_PASSWORD} + QUEUE_HEALTH_CHECK_ACTIVE: ${QUEUE_HEALTH_CHECK_ACTIVE:-true} + + # OpenTelemetry — export traces to the otel-lgtm pipeline + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://lgtm:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-n8n-main} + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment=production} + volumes: + - ./n8n-data:/home/node/.n8n + - ./n8n-files:/files + networks: + - n8n + # - external_network + - pipeline + + # =========================================================================== + # n8n worker (executes queued jobs) + # Scale this service out with: docker compose up -d --scale n8n-worker=3 + # =========================================================================== + n8n-worker: + image: docker.n8n.io/n8nio/n8n:${N8N_VERSION:-stable} + restart: unless-stopped + command: worker --concurrency=${N8N_WORKER_CONCURRENCY:-10} + depends_on: + n8n-db: + condition: service_healthy + n8n-redis: + condition: service_healthy + environment: + NODE_ENV: ${NODE_ENV:-production} + TZ: ${GENERIC_TIMEZONE:-UTC} + GENERIC_TIMEZONE: ${GENERIC_TIMEZONE:-UTC} + N8N_LOG_LEVEL: ${N8N_LOG_LEVEL:-info} + N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS: ${N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS:-true} + N8N_ENCRYPTION_KEY: ${N8N_ENCRYPTION_KEY} + EXECUTIONS_MODE: ${EXECUTIONS_MODE:-queue} + DB_TYPE: postgresdb + DB_POSTGRESDB_HOST: ${DB_HOST:-n8n-db} + DB_POSTGRESDB_PORT: ${DB_PORT:-5432} + DB_POSTGRESDB_DATABASE: ${DB_DATABASE:-n8n} + DB_POSTGRESDB_USER: ${DB_USERNAME:-n8n} + DB_POSTGRESDB_PASSWORD: ${DB_PASSWORD} + QUEUE_BULL_REDIS_HOST: ${QUEUE_BULL_REDIS_HOST:-n8n-redis} + QUEUE_BULL_REDIS_PORT: ${QUEUE_BULL_REDIS_PORT:-6379} + QUEUE_BULL_REDIS_PASSWORD: ${REDIS_PASSWORD} + QUEUE_HEALTH_CHECK_ACTIVE: ${QUEUE_HEALTH_CHECK_ACTIVE:-true} + + # OpenTelemetry — export traces to the otel-lgtm pipeline + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://lgtm:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-n8n-worker} + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment=production} + volumes: + - ./n8n-data:/home/node/.n8n + - ./n8n-files:/files + networks: + - n8n + - pipeline + +networks: + n8n: + name: n8n + driver: bridge + # external_network: + # name: ${NETWORKS_EXTERNAL_NAME:-swag} + # external: true + pipeline: + name: pipeline + external: true diff --git a/docker/n8n/swag/n8n.subdomain.conf b/docker/n8n/swag/n8n.subdomain.conf new file mode 100644 index 0000000..0ba34c3 --- /dev/null +++ b/docker/n8n/swag/n8n.subdomain.conf @@ -0,0 +1,40 @@ +## ----------------------------------------------------------------------------- +## SWAG proxy config for n8n +## Domain: n8n.ld50.xyz +## Upstream: n8n-main:5678 (shared Docker network: ${NETWORKS_EXTERNAL_NAME:-swag}) +## +## Install: +## 1) Copy this file into SWAG: /config/nginx/proxy-confs/n8n.subdomain.conf +## 2) Ensure both stacks share the same external Docker network (e.g. `swag`). +## 3) In curated_compose/n8n/docker-compose.yaml, uncomment external_network. +## 4) Reload SWAG. +## ----------------------------------------------------------------------------- + +server { + listen 443 ssl; + listen [::]:443 ssl; + + server_name n8n.ld50.xyz; + + include /config/nginx/ssl.conf; + + # n8n imports/exports can be large + client_max_body_size 100M; + + location / { + include /config/nginx/proxy.conf; + + set $upstream_app n8n-main; + set $upstream_port 5678; + set $upstream_proto http; + + proxy_pass $upstream_proto://$upstream_app:$upstream_port; + + # Required for SSE / websocket-like upgrades used by parts of n8n + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + } +}