diff --git a/.gitignore b/.gitignore index 4f509e5..6209c77 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -*.env \ No newline at end of file +prod/**/*.env +.claude \ No newline at end of file diff --git a/Taskfile.yml b/Taskfile.yml index 4724264..54f2758 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -21,33 +21,24 @@ tasks: run-consumer: cmd: go run ./cmd/consumer {{.CLI_ARGS}} dotenv: - - config/app/.consumer.env - - config/app/.mongodb.env + - config/app/consumer.env + - config/app/mongodb.env run-feedgen-az: cmd: go run ./cmd/feedgen/az {{.CLI_ARGS}} dotenv: - - config/app/feedgen/.az.env - - config/app/.mongodb.env + - config/app/feedgen/az.env + - config/app/mongodb.env run-api: cmd: go run ./cmd/api {{.CLI_ARGS}} dotenv: - - config/app/.api.env - - config/app/.mongodb.env + - config/app/api.env + - config/app/mongodb.env run-manager: cmd: go run cmd/manager/main.go {{.CLI_ARGS}} - generate-env: - desc: Generate env files from templates - cmds: - - cp config/app/consumer.env.example config/app/.consumer.env - - cp config/app/api.env.example config/app/.api.env - - cp config/app/mongodb.env.example config/app/.mongodb.env - - cp config/app/feedgen/az.env.example config/app/feedgen/.az.env - - cp config/mongodb/env.example config/mongodb/.env - docker-publish-all: desc: Publish docker images for all services cmds: diff --git a/config/app/api.env.example b/config/app/api.env similarity index 100% rename from config/app/api.env.example rename to config/app/api.env diff --git a/config/app/consumer.env b/config/app/consumer.env new file mode 100644 index 0000000..d1aab33 --- /dev/null +++ b/config/app/consumer.env @@ -0,0 +1,3 @@ +POST_MAX_DATE=720h # Save only posts created in the last month +POST_COLLECTION_CUTOFF_CRON_DELAY=30m # 30 minutes +POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=1000000 # Delete post documents after 1 million diff --git a/config/app/feedgen/az.env.example b/config/app/feedgen/az.env similarity index 100% rename from config/app/feedgen/az.env.example rename to config/app/feedgen/az.env diff --git a/config/app/mongodb.env.example b/config/app/mongodb.env similarity index 100% rename from config/app/mongodb.env.example rename to config/app/mongodb.env diff --git a/config/mongodb/env.example b/config/mongodb/env similarity index 100% rename from config/mongodb/env.example rename to config/mongodb/env diff --git a/prod/Makefile b/prod/Makefile new file mode 100644 index 0000000..a990dfc --- /dev/null +++ b/prod/Makefile @@ -0,0 +1,26 @@ +.PHONY: configure + +configure: + @cp config/app/api.env.example config/app/.api.env + @cp config/app/consumer.env.example config/app/.consumer.env + @cp config/app/mongodb.env.example config/app/.mongodb.env + @cp config/app/feedgen/az.env.example config/app/feedgen/.az.env + @cp config/caddy/env.example config/caddy/.env + @cp config/mongodb/env.example config/mongodb/.env + + @read -p "Enter MongoDB username: " mongodb_user; \ + printf "Enter MongoDB password: "; \ + read mongodb_pass; \ + sed -i "s/MONGO_INITDB_ROOT_USERNAME=.*/MONGO_INITDB_ROOT_USERNAME=$$mongodb_user/" config/mongodb/.env; \ + sed -i "s/MONGO_INITDB_ROOT_PASSWORD=.*/MONGO_INITDB_ROOT_PASSWORD=$$mongodb_pass/" config/mongodb/.env; \ + sed -i "s/MONGODB_USERNAME=.*/MONGODB_USERNAME=$$mongodb_user/" config/app/.mongodb.env; \ + sed -i "s/MONGODB_PASSWORD=.*/MONGODB_PASSWORD=$$mongodb_pass/" config/app/.mongodb.env + + @read -p "Enter domain name (e.g., feeds.bsky.example.com): " domain; \ + read -p "Enter your AT Protocol DID: " publisher_did; \ + sed -i "s/DOMAIN=.*/DOMAIN=$$domain/" config/caddy/.env; \ + sed -i "s|FEEDGEN_HOSTNAME=.*|FEEDGEN_HOSTNAME=https://$$domain|" config/app/.api.env; \ + sed -i "s/FEEDGEN_PUBLISHER_DID=.*/FEEDGEN_PUBLISHER_DID=$$publisher_did/" config/app/.api.env + + @echo + @echo "Configuration complete! You can now run 'docker compose up -d'" diff --git a/prod/README.md b/prod/README.md new file mode 100644 index 0000000..f110863 --- /dev/null +++ b/prod/README.md @@ -0,0 +1,76 @@ +# Example Production Deployment + +This is an example of a production deployment for the Feed Generator. + +## Architecture + +The production setup includes the following services: + +- **MongoDB**: Database for storing posts and feed data +- **Consumer**: Service that consumes AT Protocol firehose data +- **Feed Generator (AZ)**: Generates feeds for Azerbaijan-related content +- **API**: REST API service for serving feeds +- **Caddy**: Reverse proxy + +## Quick Start + +1. **Configure the environment**: + ```bash + make configure + ``` + This will: + - Copy all example configuration files + - Prompt for MongoDB credentials + - Prompt for domain name and AT Protocol DID + - Update configuration files with your values + +2. **Start the services**: + ```bash + docker compose up -d + ``` + +3. **Check service status**: + ```bash + docker compose ps + docker compose logs + ``` + +## Configuration Files + +### Application Configuration +- `config/app/.api.env` - API service configuration +- `config/app/.consumer.env` - Consumer service configuration +- `config/app/.mongodb.env` - MongoDB connection settings +- `config/app/feedgen/.az.env` - Azerbaijan feed generator settings + +### Infrastructure Configuration +- `config/caddy/.env` - Caddy reverse proxy settings +- `config/caddy/Caddyfile` - Caddy server configuration +- `config/mongodb/.env` - MongoDB initialization settings + +## Environment Variables + +### API Service +- `FEEDGEN_HOSTNAME` - Public hostname for the feed generator +- `FEEDGEN_PUBLISHER_DID` - Your AT Protocol DID +- `API_PORT` - Port for the API service (default: 8421) + +### Consumer Service +- `POST_MAX_DATE` - Maximum age of posts to store (default: 720h/30 days) +- `POST_COLLECTION_CUTOFF_CRON_DELAY` - Cleanup interval (default: 30m) +- `POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT` - Max documents before cleanup (default: 1M) + +### AZ Feed Generator +- `FEED_AZ_GENERATER_CRON_DELAY` - Feed generation interval (default: 1m) +- `FEED_AZ_COLLECTION_CUTOFF_CRON_DELAY` - Cleanup interval (default: 30m) +- `FEED_AZ_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT` - Max documents before cleanup (default: 500K) + +### MongoDB +- `MONGODB_HOST` - MongoDB hostname (default: mongodb) +- `MONGODB_PORT` - MongoDB port (default: 27017) +- `MONGODB_USERNAME` - Database username +- `MONGODB_PASSWORD` - Database password + +### Caddy +- `DOMAIN` - Your domain name +- `API_HOST` - Internal API service URL (default: http://api:8421) diff --git a/prod/config/app/api.env.example b/prod/config/app/api.env.example new file mode 100644 index 0000000..4925ba0 --- /dev/null +++ b/prod/config/app/api.env.example @@ -0,0 +1,3 @@ +FEEDGEN_HOSTNAME=https://feeds.bsky.example.com +FEEDGEN_PUBLISHER_DID=did:plc:qwertyuiopp +API_PORT=8421 \ No newline at end of file diff --git a/config/app/consumer.env.example b/prod/config/app/consumer.env.example similarity index 55% rename from config/app/consumer.env.example rename to prod/config/app/consumer.env.example index efa0e72..d1aab33 100644 --- a/config/app/consumer.env.example +++ b/prod/config/app/consumer.env.example @@ -1,3 +1,3 @@ POST_MAX_DATE=720h # Save only posts created in the last month POST_COLLECTION_CUTOFF_CRON_DELAY=30m # 30 minutes -POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=10000000 # Delete post documents after 10 million +POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=1000000 # Delete post documents after 1 million diff --git a/prod/config/app/feedgen/az.env.example b/prod/config/app/feedgen/az.env.example new file mode 100644 index 0000000..c19581c --- /dev/null +++ b/prod/config/app/feedgen/az.env.example @@ -0,0 +1,3 @@ +FEED_AZ_GENERATER_CRON_DELAY=1m # 1 minute +FEED_AZ_COLLECTION_CUTOFF_CRON_DELAY=30m # 30 minutes +FEED_AZ_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=500000 # Delete post documents after 500 thousand diff --git a/prod/config/app/mongodb.env.example b/prod/config/app/mongodb.env.example new file mode 100644 index 0000000..92f820b --- /dev/null +++ b/prod/config/app/mongodb.env.example @@ -0,0 +1,4 @@ +MONGODB_HOST=mongodb +MONGODB_PORT=27017 +MONGODB_USERNAME=root +MONGODB_PASSWORD=toor diff --git a/prod/config/caddy/Caddyfile b/prod/config/caddy/Caddyfile new file mode 100644 index 0000000..08c5e2e --- /dev/null +++ b/prod/config/caddy/Caddyfile @@ -0,0 +1,11 @@ +{ + admin off +} + +{$DOMAIN} { + request_body { + max_size 8MB + } + + reverse_proxy {$API_HOST} +} diff --git a/prod/config/caddy/env.example b/prod/config/caddy/env.example new file mode 100644 index 0000000..0d4b23e --- /dev/null +++ b/prod/config/caddy/env.example @@ -0,0 +1,2 @@ +DOMAIN=feeds.bsky.example.com +API_HOST=http://api:8421 diff --git a/prod/config/mongodb/env.example b/prod/config/mongodb/env.example new file mode 100644 index 0000000..c0fb643 --- /dev/null +++ b/prod/config/mongodb/env.example @@ -0,0 +1,2 @@ +MONGO_INITDB_ROOT_USERNAME=root +MONGO_INITDB_ROOT_PASSWORD=toor diff --git a/prod/docker-compose.yml b/prod/docker-compose.yml new file mode 100644 index 0000000..f62ce86 --- /dev/null +++ b/prod/docker-compose.yml @@ -0,0 +1,63 @@ +services: + mongodb: + image: mongo:8.0.9-noble + restart: unless-stopped + # ports: + # - 27017:27017 + env_file: ./config/mongodb/.env + volumes: + - mongodb_data:/data/db + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongosh --quiet + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + + consumer: + image: git.aykhans.me/bsky/feedgen-consumer:latest + restart: unless-stopped + env_file: + - ./config/app/.mongodb.env + - ./config/app/.consumer.env + depends_on: + mongodb: + condition: service_healthy + + feedgen_az: + image: git.aykhans.me/bsky/feedgen-generator-az:latest + restart: unless-stopped + env_file: + - ./config/app/.mongodb.env + - ./config/app/feedgen/.az.env + depends_on: + mongodb: + condition: service_healthy + + api: + image: git.aykhans.me/bsky/feedgen-api:latest + restart: unless-stopped + ports: + - 8421:8421 + env_file: + - ./config/app/.mongodb.env + - ./config/app/.api.env + depends_on: + mongodb: + condition: service_healthy + + caddy: + image: caddy:2.10.0-alpine + restart: unless-stopped + ports: + - 80:80 + - 443:443 + - 443:443/udp + env_file: ./config/caddy/.env + volumes: + - ./config/caddy/Caddyfile:/etc/caddy/Caddyfile + - caddy_data:/data + +volumes: + mongodb_data: + caddy_data: