aykhans/bsky-feedgen
1
0
Fork 0
mirror of https://github.com/aykhans/bsky-feedgen.git synced 2025-05-30 02:30:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add example for production deployment

Browse Source
This commit is contained in:
Aykhan Shahsuvarov 2025-05-28 00:33:01 +04:00
parent dbf892535b
commit 7b89230cae
17 changed files with 202 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1 +1,2 @@
*.env
prod/**/*.env
.claude

21
Taskfile.yml
View File

@ -21,33 +21,24 @@ tasks:
run-consumer:
cmd: go run ./cmd/consumer {{.CLI_ARGS}}
dotenv:
- config/app/.consumer.env
- config/app/.mongodb.env
- config/app/consumer.env
- config/app/mongodb.env
run-feedgen-az:
cmd: go run ./cmd/feedgen/az {{.CLI_ARGS}}
dotenv:
- config/app/feedgen/.az.env
- config/app/.mongodb.env
- config/app/feedgen/az.env
- config/app/mongodb.env
run-api:
cmd: go run ./cmd/api {{.CLI_ARGS}}
dotenv:
- config/app/.api.env
- config/app/.mongodb.env
- config/app/api.env
- config/app/mongodb.env
run-manager:
cmd: go run cmd/manager/main.go {{.CLI_ARGS}}
generate-env:
desc: Generate env files from templates
cmds:
- cp config/app/consumer.env.example config/app/.consumer.env
- cp config/app/api.env.example config/app/.api.env
- cp config/app/mongodb.env.example config/app/.mongodb.env
- cp config/app/feedgen/az.env.example config/app/feedgen/.az.env
- cp config/mongodb/env.example config/mongodb/.env
docker-publish-all:
desc: Publish docker images for all services
cmds:

0
config/app/api.env.example → config/app/api.env
View File

3
config/app/consumer.env Normal file
View File

@ -0,0 +1,3 @@
POST_MAX_DATE=720h # Save only posts created in the last month
POST_COLLECTION_CUTOFF_CRON_DELAY=30m # 30 minutes
POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=1000000 # Delete post documents after 1 million

0
config/app/feedgen/az.env.example → config/app/feedgen/az.env
View File

0
config/app/mongodb.env.example → config/app/mongodb.env
View File

0
config/mongodb/env.example → config/mongodb/env
View File

26
prod/Makefile Normal file
View File

@ -0,0 +1,26 @@
.PHONY: configure
configure:
@cp config/app/api.env.example config/app/.api.env
@cp config/app/consumer.env.example config/app/.consumer.env
@cp config/app/mongodb.env.example config/app/.mongodb.env
@cp config/app/feedgen/az.env.example config/app/feedgen/.az.env
@cp config/caddy/env.example config/caddy/.env
@cp config/mongodb/env.example config/mongodb/.env
@read -p "Enter MongoDB username: " mongodb_user; \
printf "Enter MongoDB password: "; \
read mongodb_pass; \
sed -i "s/MONGO_INITDB_ROOT_USERNAME=.*/MONGO_INITDB_ROOT_USERNAME=$$mongodb_user/" config/mongodb/.env; \
sed -i "s/MONGO_INITDB_ROOT_PASSWORD=.*/MONGO_INITDB_ROOT_PASSWORD=$$mongodb_pass/" config/mongodb/.env; \
sed -i "s/MONGODB_USERNAME=.*/MONGODB_USERNAME=$$mongodb_user/" config/app/.mongodb.env; \
sed -i "s/MONGODB_PASSWORD=.*/MONGODB_PASSWORD=$$mongodb_pass/" config/app/.mongodb.env
@read -p "Enter domain name (e.g., feeds.bsky.example.com): " domain; \
read -p "Enter your AT Protocol DID: " publisher_did; \
sed -i "s/DOMAIN=.*/DOMAIN=$$domain/" config/caddy/.env; \
sed -i "s|FEEDGEN_HOSTNAME=.*|FEEDGEN_HOSTNAME=https://$$domain|" config/app/.api.env; \
sed -i "s/FEEDGEN_PUBLISHER_DID=.*/FEEDGEN_PUBLISHER_DID=$$publisher_did/" config/app/.api.env
@echo
@echo "Configuration complete! You can now run 'docker compose up -d'"

76
prod/README.md Normal file
View File

@ -0,0 +1,76 @@
# Example Production Deployment
This is an example of a production deployment for the Feed Generator.
## Architecture
The production setup includes the following services:
- **MongoDB**: Database for storing posts and feed data
- **Consumer**: Service that consumes AT Protocol firehose data
- **Feed Generator (AZ)**: Generates feeds for Azerbaijan-related content
- **API**: REST API service for serving feeds
- **Caddy**: Reverse proxy
## Quick Start
1. **Configure the environment**:
```bash
make configure
```
This will:
- Copy all example configuration files
- Prompt for MongoDB credentials
- Prompt for domain name and AT Protocol DID
- Update configuration files with your values
2. **Start the services**:
```bash
docker compose up -d
```
3. **Check service status**:
```bash
docker compose ps
docker compose logs
```
## Configuration Files
### Application Configuration
- `config/app/.api.env` - API service configuration
- `config/app/.consumer.env` - Consumer service configuration
- `config/app/.mongodb.env` - MongoDB connection settings
- `config/app/feedgen/.az.env` - Azerbaijan feed generator settings
### Infrastructure Configuration
- `config/caddy/.env` - Caddy reverse proxy settings
- `config/caddy/Caddyfile` - Caddy server configuration
- `config/mongodb/.env` - MongoDB initialization settings
## Environment Variables
### API Service
- `FEEDGEN_HOSTNAME` - Public hostname for the feed generator
- `FEEDGEN_PUBLISHER_DID` - Your AT Protocol DID
- `API_PORT` - Port for the API service (default: 8421)
### Consumer Service
- `POST_MAX_DATE` - Maximum age of posts to store (default: 720h/30 days)
- `POST_COLLECTION_CUTOFF_CRON_DELAY` - Cleanup interval (default: 30m)
- `POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT` - Max documents before cleanup (default: 1M)
### AZ Feed Generator
- `FEED_AZ_GENERATER_CRON_DELAY` - Feed generation interval (default: 1m)
- `FEED_AZ_COLLECTION_CUTOFF_CRON_DELAY` - Cleanup interval (default: 30m)
- `FEED_AZ_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT` - Max documents before cleanup (default: 500K)
### MongoDB
- `MONGODB_HOST` - MongoDB hostname (default: mongodb)
- `MONGODB_PORT` - MongoDB port (default: 27017)
- `MONGODB_USERNAME` - Database username
- `MONGODB_PASSWORD` - Database password
### Caddy
- `DOMAIN` - Your domain name
- `API_HOST` - Internal API service URL (default: http://api:8421)

3
prod/config/app/api.env.example Normal file
View File

@ -0,0 +1,3 @@
FEEDGEN_HOSTNAME=https://feeds.bsky.example.com
FEEDGEN_PUBLISHER_DID=did:plc:qwertyuiopp
API_PORT=8421

2
config/app/consumer.env.example → prod/config/app/consumer.env.example
View File

@ -1,3 +1,3 @@
POST_MAX_DATE=720h # Save only posts created in the last month
POST_COLLECTION_CUTOFF_CRON_DELAY=30m # 30 minutes
POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=10000000 # Delete post documents after 10 million
POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=1000000 # Delete post documents after 1 million

3
prod/config/app/feedgen/az.env.example Normal file
View File

@ -0,0 +1,3 @@
FEED_AZ_GENERATER_CRON_DELAY=1m # 1 minute
FEED_AZ_COLLECTION_CUTOFF_CRON_DELAY=30m # 30 minutes
FEED_AZ_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=500000 # Delete post documents after 500 thousand

4
prod/config/app/mongodb.env.example Normal file
View File

@ -0,0 +1,4 @@
MONGODB_HOST=mongodb
MONGODB_PORT=27017
MONGODB_USERNAME=root
MONGODB_PASSWORD=toor

11
prod/config/caddy/Caddyfile Normal file
View File

@ -0,0 +1,11 @@
{
admin off
}
{$DOMAIN} {
request_body {
max_size 8MB
}
reverse_proxy {$API_HOST}
}

2
prod/config/caddy/env.example Normal file
View File

@ -0,0 +1,2 @@
DOMAIN=feeds.bsky.example.com
API_HOST=http://api:8421

2
prod/config/mongodb/env.example Normal file
View File

@ -0,0 +1,2 @@
MONGO_INITDB_ROOT_USERNAME=root
MONGO_INITDB_ROOT_PASSWORD=toor

63
prod/docker-compose.yml Normal file
View File

@ -0,0 +1,63 @@
services:
mongodb:
image: mongo:8.0.9-noble
restart: unless-stopped
# ports:
# - 27017:27017
env_file: ./config/mongodb/.env
volumes:
- mongodb_data:/data/db
healthcheck:
test: echo 'db.runCommand("ping").ok' | mongosh --quiet
interval: 10s
timeout: 5s
retries: 5
start_period: 20s
consumer:
image: git.aykhans.me/bsky/feedgen-consumer:latest
restart: unless-stopped
env_file:
- ./config/app/.mongodb.env
- ./config/app/.consumer.env
depends_on:
mongodb:
condition: service_healthy
feedgen_az:
image: git.aykhans.me/bsky/feedgen-generator-az:latest
restart: unless-stopped
env_file:
- ./config/app/.mongodb.env
- ./config/app/feedgen/.az.env
depends_on:
mongodb:
condition: service_healthy
api:
image: git.aykhans.me/bsky/feedgen-api:latest
restart: unless-stopped
ports:
- 8421:8421
env_file:
- ./config/app/.mongodb.env
- ./config/app/.api.env
depends_on:
mongodb:
condition: service_healthy
caddy:
image: caddy:2.10.0-alpine
restart: unless-stopped
ports:
- 80:80
- 443:443
- 443:443/udp
env_file: ./config/caddy/.env
volumes:
- ./config/caddy/Caddyfile:/etc/caddy/Caddyfile
- caddy_data:/data
volumes:
mongodb_data:
caddy_data: