Merge pull request #13 from aykhans/dependabot/go_modules/go.mongodb.org/mongo-driver-1.17.4

Bump go.mongodb.org/mongo-driver from 1.17.3 to 1.17.4

.github/dependabot.yaml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "gomod" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"

.github/workflows/lint.yaml

@@ -0,0 +1,23 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-go@v6
+        with:
+          go-version: stable
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v8
+        with:
+          version: v2.5.0

.github/workflows/publish-docker.yaml

@@ -0,0 +1,99 @@
+name: Publish Docker Images
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'cmd/*/version.go'
+      - 'cmd/feedgen/*/version.go'
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      api: ${{ steps.filter.outputs.api }}
+      consumer: ${{ steps.filter.outputs.consumer }}
+      feedgen-az: ${{ steps.filter.outputs.feedgen-az }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            api:
+              - 'cmd/api/version.go'
+            consumer:
+              - 'cmd/consumer/version.go'
+            feedgen-az:
+              - 'cmd/feedgen/az/version.go'
+
+  publish-api:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.api == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.aykhans.me
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Install Task
+        uses: arduino/setup-task@v2
+
+      - name: Build and publish API
+        run: task docker-publish-api
+
+  publish-consumer:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.consumer == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.aykhans.me
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Install Task
+        uses: arduino/setup-task@v2
+
+      - name: Build and publish Consumer
+        run: task docker-publish-consumer
+
+  publish-feedgen-az:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.feedgen-az == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.aykhans.me
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Install Task
+        uses: arduino/setup-task@v2
+
+      - name: Build and publish Feedgen AZ
+        run: task docker-publish-feedgen-az

.gitignore

@@ -1 +1,2 @@
-*.env
+prod/**/*.env
+.claude

Taskfile.yml

@@ -3,7 +3,7 @@
 version: "3"
 
 vars:
-    DOCKER_REGISTRY: "git.aykhans.me/bsky/"
+  DOCKER_REGISTRY: "git.aykhans.me"
 
 tasks:
   ftl:
@@ -21,33 +21,24 @@ tasks:
   run-consumer:
     cmd: go run ./cmd/consumer {{.CLI_ARGS}}
     dotenv:
-            - config/app/.consumer.env
-            - config/app/.mongodb.env
+      - config/app/consumer.env
+      - config/app/mongodb.env
 
   run-feedgen-az:
     cmd: go run ./cmd/feedgen/az {{.CLI_ARGS}}
     dotenv:
-            - config/app/feedgen/.az.env
-            - config/app/.mongodb.env
+      - config/app/feedgen/az.env
+      - config/app/mongodb.env
 
   run-api:
     cmd: go run ./cmd/api {{.CLI_ARGS}}
     dotenv:
-            - config/app/.api.env
-            - config/app/.mongodb.env
+      - config/app/api.env
+      - config/app/mongodb.env
 
   run-manager:
     cmd: go run cmd/manager/main.go {{.CLI_ARGS}}
 
-    generate-env:
-        desc: Generate env files from templates
-        cmds:
-            - cp config/app/consumer.env.example config/app/.consumer.env
-            - cp config/app/api.env.example config/app/.api.env
-            - cp config/app/mongodb.env.example config/app/.mongodb.env
-            - cp config/app/feedgen/az.env.example config/app/feedgen/.az.env
-            - cp config/mongodb/env.example config/mongodb/.env
-
   docker-publish-all:
     desc: Publish docker images for all services
     cmds:
@@ -60,7 +51,7 @@ tasks:
     desc: Publish docker image for api service
     vars:
       GO_VERSION_FILE: ./cmd/api/version.go
-            IMAGE_NAME: feedgen-api
+      IMAGE_NAME: /bsky/feedgen-api
       VERSION:
         sh: grep -o 'const version = "[^"]*"' {{.GO_VERSION_FILE}} | grep -o '"[^"]*"' | tr -d '"'
       VERSIONED_IMAGE: "{{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:{{.VERSION}}"
@@ -80,7 +71,7 @@ tasks:
     desc: Publish docker image for consumer service
     vars:
       GO_VERSION_FILE: ./cmd/consumer/version.go
-            IMAGE_NAME: feedgen-consumer
+      IMAGE_NAME: /bsky/feedgen-consumer
       VERSION:
         sh: grep -o 'const version = "[^"]*"' {{.GO_VERSION_FILE}} | grep -o '"[^"]*"' | tr -d '"'
       VERSIONED_IMAGE: "{{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:{{.VERSION}}"
@@ -96,7 +87,7 @@ tasks:
     desc: Publish docker image for feedgen-az service
     vars:
       GO_VERSION_FILE: ./cmd/feedgen/az/version.go
-            IMAGE_NAME: feedgen-generator-az
+      IMAGE_NAME: /bsky/feedgen-generator-az
       VERSION:
         sh: grep -o 'const version = "[^"]*"' {{.GO_VERSION_FILE}} | grep -o '"[^"]*"' | tr -d '"'
       VERSIONED_IMAGE: "{{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:{{.VERSION}}"
@@ -110,6 +101,9 @@ tasks:
 
   docker-publish-manager:
     desc: Publish docker image for manager service
+    vars:
+      GO_VERSION_FILE: ./cmd/manager/version.go
+      IMAGE_NAME: /bsky/feedgen-manager
     cmds:
-            - docker build -t git.aykhans.me/bsky/feedgen-manager:latest -f ./cmd/manager/Dockerfile .
-            - docker push git.aykhans.me/bsky/feedgen-manager:latest
+      - docker build -t {{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:latest -f ./cmd/manager/Dockerfile .
+      - docker push {{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:latest

cmd/api/version.go

@@ -1,3 +1,3 @@
 package main
 
-const version = "0.1.0"
+const version = "0.2.208"

cmd/feedgen/az/version.go

@@ -1,3 +1,3 @@
 package main
 
-const version = "0.1.0"
+const version = "0.1.112"

config/app/consumer.env

@@ -0,0 +1,3 @@
+POST_MAX_DATE=720h # Save only posts created in the last month
+POST_COLLECTION_CUTOFF_CRON_DELAY=30m # 30 minutes
+POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=1000000 # Delete post documents after 1 million

go.mod

@@ -1,11 +1,11 @@
 module github.com/aykhans/bsky-feedgen
 
-go 1.24
+go 1.24.0
 
 require (
 	github.com/bluesky-social/indigo v0.0.0-20250516010818-f8de501bd6a0
-	github.com/gorilla/websocket v1.5.1
-	go.mongodb.org/mongo-driver v1.17.3
+	github.com/gorilla/websocket v1.5.3
+	go.mongodb.org/mongo-driver v1.17.4
 	go.uber.org/automaxprocs v1.6.0
 )
 
@@ -21,7 +21,7 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gocql/gocql v1.7.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.2
+	github.com/golang-jwt/jwt/v5 v5.3.0
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.4.0 // indirect
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
@@ -85,8 +85,8 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/cobra v1.9.1
-	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/spf13/cobra v1.10.1
+	github.com/spf13/pflag v1.0.9 // indirect
 	github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e // indirect
 	github.com/whyrusleeping/go-did v0.0.0-20240828165449-bcaa7ae21371
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
@@ -103,10 +103,9 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.38.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sync v0.14.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/term v0.32.0
+	golang.org/x/sys v0.37.0 // indirect
+	golang.org/x/term v0.36.0
 	golang.org/x/text v0.25.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect

go.sum

@@ -46,8 +46,8 @@ github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus=
 github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
-github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
+github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -60,8 +60,8 @@ github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
-github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@@ -276,10 +276,10 @@ github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hg
 github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
-github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
-github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
-github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
+github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
+github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -320,8 +320,8 @@ gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRyS
 gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8=
 gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q=
 gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I=
-go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ=
-go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
+go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw=
+go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
 go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
@@ -402,15 +402,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
+golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
-golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
-golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
+golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
+golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=

pkg/api/base.go

@@ -23,6 +23,7 @@ func Run(
 		return err
 	}
 	feedHandler := handler.NewFeedHandler(feeds, apiConfig.FeedgenPublisherDID)
+	generatorHandler := handler.NewGeneratorHandler()
 
 	authMiddleware := middleware.NewAuth(apiConfig.ServiceDID)
 
@@ -34,6 +35,9 @@ func Run(
 		"GET /xrpc/app.bsky.feed.getFeedSkeleton",
 		authMiddleware.JWTAuthMiddleware(http.HandlerFunc(feedHandler.GetFeedSkeleton)),
 	)
+	mux.HandleFunc("GET /{feed}/users", generatorHandler.GetAllUsers)
+	mux.HandleFunc("GET /{feed}/users/valid/", generatorHandler.GetValidUsers)
+	mux.HandleFunc("GET /{feed}/users/invalid/", generatorHandler.GetInvalidUsers)
 
 	httpServer := &http.Server{
 		Addr:    fmt.Sprintf(":%d", apiConfig.APIPort),

pkg/api/handler/generator.go

@@ -0,0 +1,57 @@
+package handler
+
+import (
+	"net/http"
+
+	"github.com/aykhans/bsky-feedgen/pkg/api/response"
+	generatorAz "github.com/aykhans/bsky-feedgen/pkg/generator/az"
+)
+
+type GeneratorHandler struct{}
+
+func NewGeneratorHandler() *GeneratorHandler {
+	return &GeneratorHandler{}
+}
+
+func (handler *GeneratorHandler) GetValidUsers(w http.ResponseWriter, r *http.Request) {
+	feed := r.PathValue("feed")
+
+	validUsers := make([]string, 0)
+	switch feed {
+	case "AzPulse":
+		validUsers = generatorAz.Users.GetValidUsers()
+	}
+
+	response.JSON(w, 200, response.M{
+		"feed":  feed,
+		"users": validUsers,
+	})
+}
+
+func (handler *GeneratorHandler) GetInvalidUsers(w http.ResponseWriter, r *http.Request) {
+	feed := r.PathValue("feed")
+
+	invalidUsers := make([]string, 0)
+	switch feed {
+	case "AzPulse":
+		invalidUsers = generatorAz.Users.GetInvalidUsers()
+	}
+
+	response.JSON(w, 200, response.M{
+		"feed":  feed,
+		"users": invalidUsers,
+	})
+}
+
+func (handler *GeneratorHandler) GetAllUsers(w http.ResponseWriter, r *http.Request) {
+	feed := r.PathValue("feed")
+
+	responseData := response.M{"feed": feed}
+	switch feed {
+	case "AzPulse":
+		responseData["valid_users"] = generatorAz.Users.GetValidUsers()
+		responseData["invalid_users"] = generatorAz.Users.GetInvalidUsers()
+	}
+
+	response.JSON(w, 200, responseData)
+}

pkg/feed/az.go

@@ -39,7 +39,7 @@ func (f *FeedAz) Describe(_ context.Context) bsky.FeedDescribeFeedGenerator_Feed
 
 func (f *FeedAz) GetPage(
 	ctx context.Context,
-	_ string,
+	_ string, // user did
 	limit int64,
 	cursor string,
 ) ([]*bsky.FeedDefs_SkeletonFeedPost, *string, error) {

pkg/generator/az/base.go

@@ -110,16 +110,24 @@ func (generator *Generator) Start(ctx context.Context, cursorOption types.Genera
 }
 
 func (generator *Generator) IsValid(post *collections.Post) bool {
+	// Skip posts that are deep replies (not direct replies to original posts)
 	if post.Reply != nil && post.Reply.RootURI != post.Reply.ParentURI {
 		return false
 	}
 
-	if isValidUser := users.IsValid(post.DID); isValidUser != nil {
+	// Check if the user who created this post is in our pre-defined list
+	// This allows for explicit inclusion/exclusion of specific users
+	if isValidUser := Users.IsValid(post.DID); isValidUser != nil {
 		return *isValidUser
 	}
 
-	if (slices.Contains(post.Langs, "az") && len(post.Langs) < 3) || // Posts in Azerbaijani language with fewer than 3 languages
-		generator.textRegex.MatchString(post.Text) { // Posts containing Azerbaijan-related keywords
+	// A post is considered valid if it meets either of the following criteria:
+	// 1. It's primarily in Azerbaijani (language code "az") with less than 3 detected languages
+	//    (to filter out multi-language spam)
+	// 2. It contains Azerbaijan-related keywords in the text AND has at least one valid language
+	//    from our approved language list
+	if (slices.Contains(post.Langs, "az") && len(post.Langs) < 3) ||
+		(generator.textRegex.MatchString(post.Text) && Langs.IsExistsAny(post.Langs)) {
 		return true
 	}
 

pkg/generator/az/lists.go

@@ -2,7 +2,7 @@ package az
 
 import "github.com/aykhans/bsky-feedgen/pkg/generator"
 
-var users = generator.Users{
+var Users = generator.Users{
 	// Invalid
 	"did:plc:5zww7zorx2ajw7hqrhuix3ba": false,
 	"did:plc:c4vhz47h566t2ntgd7gtawen": false,
@@ -14,6 +14,32 @@ var users = generator.Users{
 	"did:plc:5vwjnzaibnwscbbcvkzhy57v": false,
 	"did:plc:6mfp3coadoobuvlg6w2avw6x": false,
 	"did:plc:lm2uhaoqoe6yo76oeihndfyi": false,
+	"did:plc:vizwdor43adw3277u2kkrssd": false,
+	"did:plc:oqatvbgbhvqbjl2w2o63ehgi": false,
+	"did:plc:gy7yilnydusx5hy2z3dltynp": false,
+	"did:plc:xk7cs24wk6njv42azm2yd7dv": false,
+	"did:plc:ijmt7f4p3dcfqtg3j3zshimn": false,
+	"did:plc:2q5dx6whenn7pnsrfn3jpd6h": false,
+	"did:plc:s2waw3gkmn7h2nn6od44apng": false,
+	"did:plc:4hm6gb7dzobynqrpypif3dck": false,
+	"did:plc:odvarii7w7soygxet3xvzop7": false,
+	"did:plc:5cbkdchsxjvz5fog2oo7m4le": false,
+	"did:plc:ooeuisen5rtr4rojmz7gkbrh": false,
+	"did:plc:6bvhdvgeqkj7nol2zodtqmww": false,
+	"did:plc:k6sxlkd5ssq2uaylzisap2tw": false,
+	"did:plc:uxljnh22mmfzmr4i3oien6mx": false,
+	"did:plc:w5gg2zgwcyfevphehdcmavev": false,
+	"did:plc:ckawbibgmrwg3lbskfppwtlw": false,
+	"did:plc:43fdk46qa5gsokzygzildsaq": false,
+	"did:plc:3szm5t3tknphjtj73twqfonw": false,
+	"did:plc:4ukvsogndgp67sv6f6ohse3y": false,
+	"did:plc:cdplzvv63u5jxb4fxm4vpfgm": false,
+	"did:plc:namifrcorf6hzy45phd4shvt": false,
+	"did:plc:ltvtwjps77bqgm2knhlbswyk": false,
+	"did:plc:acglo4ret2f2wc5duqtispsa": false,
+	"did:plc:zibx3delbo24mdsccz6s7qa4": false,
+	"did:plc:dbpnhjiyq5e7pe3a4mt3jyhx": false,
+	"did:plc:ilvqavldtvn4ytagkvjafq6k": false,
 
 	// Valid
 	"did:plc:jbt4qi6psd7rutwzedtecsq7": true,
@@ -22,6 +48,16 @@ var users = generator.Users{
 	"did:plc:phtq2rhgbwipyx5ie3apw44j": true,
 	"did:plc:jfdvklrs5n5qv7f25v6swc5h": true,
 	"did:plc:u5ez5w6qslh6advti4wyddba": true,
-	"did:plc:cs2cbzojm6hmx5lfxiuft3mq": true,
 	"did:plc:x7alwnnjygt2aqcwblhazko7": true,
+	"did:plc:mgciyhgfn65z7iazxuar6o6a": true,
+	"did:plc:ay2f5go4lxq2hspiaqohegac": true,
+	"did:plc:ftoopigdpuzqt2kpeyqxsofx": true,
+	"did:plc:cs2cbzojm6hmx5lfxiuft3mq": true,
+}
+
+var Langs = generator.Langs{
+	"az": true,
+	"en": true,
+	"tr": true,
+	"ru": true,
 }

pkg/generator/base.go

@@ -4,6 +4,15 @@ import "github.com/aykhans/bsky-feedgen/pkg/utils"
 
 type Users map[string]bool
 
+// IsValid checks if a given DID exists in the Users map and returns its validity status.
+//
+// Parameters:
+//
+//	did: The Decentralized Identifier string to check
+//
+// Returns:
+//   - *bool: A pointer to the validity status if the DID exists in the map
+//   - nil: If the DID does not exist in the map
 func (u Users) IsValid(did string) *bool {
 	isValid, ok := u[did]
 	if ok == false {
@@ -12,3 +21,69 @@ func (u Users) IsValid(did string) *bool {
 
 	return utils.ToPtr(isValid)
 }
+
+// GetValidUsers returns a slice of DIDs that are marked as valid in the Users map.
+//
+// Returns:
+//   - []string: A slice of valid DIDs, limited by the specified parameters
+func (u Users) GetValidUsers() []string {
+	validUsers := make([]string, 0)
+
+	for did, isValid := range u {
+		if isValid {
+			validUsers = append(validUsers, did)
+		}
+	}
+
+	return validUsers
+}
+
+// GetInvalidUsers returns a slice of DIDs that are marked as invalid in the Users map.
+//
+// Returns:
+//   - []string: A slice of invalid DIDs, limited by the specified parameters
+func (u Users) GetInvalidUsers() []string {
+	invalidUsers := make([]string, 0)
+
+	for did, isValid := range u {
+		if !isValid {
+			invalidUsers = append(invalidUsers, did)
+		}
+	}
+
+	return invalidUsers
+}
+
+// GetAll returns a slice of all DIDs in the Users map, regardless of validity status.
+//
+// Returns:
+//   - []string: A slice containing all DIDs in the map
+func (u Users) GetAll() []string {
+	allUsers := make([]string, 0, len(u))
+
+	for did := range u {
+		allUsers = append(allUsers, did)
+	}
+
+	return allUsers
+}
+
+type Langs map[string]bool
+
+// IsExistsAny checks if any of the given language codes exist in the Langs map.
+//
+// Parameters:
+//   - langs: A slice of language code strings to check for existence
+//
+// Returns:
+//   - bool: true if at least one language code from the input slice exists in the map,
+//     false if none of the provided language codes exist
+func (l Langs) IsExistsAny(langs []string) bool {
+	for _, lang := range langs {
+		if _, ok := l[lang]; ok {
+			return true
+		}
+	}
+
+	return false
+}

prod/Makefile

@@ -0,0 +1,26 @@
+.PHONY: configure
+
+configure:
+	@cp config/app/api.env.example config/app/.api.env
+	@cp config/app/consumer.env.example config/app/.consumer.env
+	@cp config/app/mongodb.env.example config/app/.mongodb.env
+	@cp config/app/feedgen/az.env.example config/app/feedgen/.az.env
+	@cp config/caddy/env.example config/caddy/.env
+	@cp config/mongodb/env.example config/mongodb/.env
+
+	@read -p "Enter MongoDB username: " mongodb_user; \
+	printf "Enter MongoDB password: "; \
+	read mongodb_pass; \
+	sed -i "s/MONGO_INITDB_ROOT_USERNAME=.*/MONGO_INITDB_ROOT_USERNAME=$$mongodb_user/" config/mongodb/.env; \
+	sed -i "s/MONGO_INITDB_ROOT_PASSWORD=.*/MONGO_INITDB_ROOT_PASSWORD=$$mongodb_pass/" config/mongodb/.env; \
+	sed -i "s/MONGODB_USERNAME=.*/MONGODB_USERNAME=$$mongodb_user/" config/app/.mongodb.env; \
+	sed -i "s/MONGODB_PASSWORD=.*/MONGODB_PASSWORD=$$mongodb_pass/" config/app/.mongodb.env
+
+	@read -p "Enter domain name (e.g., feeds.bsky.example.com): " domain; \
+	read -p "Enter your AT Protocol DID: " publisher_did; \
+	sed -i "s/DOMAIN=.*/DOMAIN=$$domain/" config/caddy/.env; \
+	sed -i "s|FEEDGEN_HOSTNAME=.*|FEEDGEN_HOSTNAME=https://$$domain|" config/app/.api.env; \
+	sed -i "s/FEEDGEN_PUBLISHER_DID=.*/FEEDGEN_PUBLISHER_DID=$$publisher_did/" config/app/.api.env
+
+	@echo
+	@echo "Configuration complete! You can now run 'docker compose up -d'"

prod/README.md

@@ -0,0 +1,76 @@
+# Example Production Deployment
+
+This is an example of a production deployment for the Feed Generator.
+
+## Architecture
+
+The production setup includes the following services:
+
+- **MongoDB**: Database for storing posts and feed data
+- **Consumer**: Service that consumes AT Protocol firehose data
+- **Feed Generator (AZ)**: Generates feeds for Azerbaijan-related content
+- **API**: REST API service for serving feeds
+- **Caddy**: Reverse proxy
+
+## Quick Start
+
+1. **Configure the environment**:
+   ```bash
+   make configure
+   ```
+   This will:
+   - Copy all example configuration files
+   - Prompt for MongoDB credentials
+   - Prompt for domain name and AT Protocol DID
+   - Update configuration files with your values
+
+2. **Start the services**:
+   ```bash
+   docker compose up -d
+   ```
+
+3. **Check service status**:
+   ```bash
+   docker compose ps
+   docker compose logs
+   ```
+
+## Configuration Files
+
+### Application Configuration
+- `config/app/.api.env` - API service configuration
+- `config/app/.consumer.env` - Consumer service configuration  
+- `config/app/.mongodb.env` - MongoDB connection settings
+- `config/app/feedgen/.az.env` - Azerbaijan feed generator settings
+
+### Infrastructure Configuration
+- `config/caddy/.env` - Caddy reverse proxy settings
+- `config/caddy/Caddyfile` - Caddy server configuration
+- `config/mongodb/.env` - MongoDB initialization settings
+
+## Environment Variables
+
+### API Service
+- `FEEDGEN_HOSTNAME` - Public hostname for the feed generator
+- `FEEDGEN_PUBLISHER_DID` - Your AT Protocol DID
+- `API_PORT` - Port for the API service (default: 8421)
+
+### Consumer Service
+- `POST_MAX_DATE` - Maximum age of posts to store (default: 720h/30 days)
+- `POST_COLLECTION_CUTOFF_CRON_DELAY` - Cleanup interval (default: 30m)
+- `POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT` - Max documents before cleanup (default: 1M)
+
+### AZ Feed Generator
+- `FEED_AZ_GENERATER_CRON_DELAY` - Feed generation interval (default: 1m)
+- `FEED_AZ_COLLECTION_CUTOFF_CRON_DELAY` - Cleanup interval (default: 30m)
+- `FEED_AZ_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT` - Max documents before cleanup (default: 500K)
+
+### MongoDB
+- `MONGODB_HOST` - MongoDB hostname (default: mongodb)
+- `MONGODB_PORT` - MongoDB port (default: 27017)
+- `MONGODB_USERNAME` - Database username
+- `MONGODB_PASSWORD` - Database password
+
+### Caddy
+- `DOMAIN` - Your domain name
+- `API_HOST` - Internal API service URL (default: http://api:8421)

prod/config/app/api.env.example

@@ -0,0 +1,3 @@
+FEEDGEN_HOSTNAME=https://feeds.bsky.example.com
+FEEDGEN_PUBLISHER_DID=did:plc:qwertyuiopp
+API_PORT=8421

prod/config/app/consumer.env.example

@@ -1,3 +1,3 @@
 POST_MAX_DATE=720h # Save only posts created in the last month
 POST_COLLECTION_CUTOFF_CRON_DELAY=30m # 30 minutes
-POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=10000000 # Delete post documents after 10 million
+POST_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=1000000 # Delete post documents after 1 million

prod/config/app/feedgen/az.env.example

@@ -0,0 +1,3 @@
+FEED_AZ_GENERATER_CRON_DELAY=1m # 1 minute
+FEED_AZ_COLLECTION_CUTOFF_CRON_DELAY=30m # 30 minutes
+FEED_AZ_COLLECTION_CUTOFF_CRON_MAX_DOCUMENT=500000 # Delete post documents after 500 thousand

prod/config/app/mongodb.env.example

@@ -0,0 +1,4 @@
+MONGODB_HOST=mongodb
+MONGODB_PORT=27017
+MONGODB_USERNAME=root
+MONGODB_PASSWORD=toor

prod/config/caddy/Caddyfile

@@ -0,0 +1,11 @@
+{
+    admin off
+}
+
+{$DOMAIN} {
+    request_body {
+        max_size 8MB
+    }
+
+    reverse_proxy {$API_HOST}
+}

prod/config/caddy/env.example

@@ -0,0 +1,2 @@
+DOMAIN=feeds.bsky.example.com
+API_HOST=http://api:8421

prod/config/mongodb/env.example

@@ -0,0 +1,2 @@
+MONGO_INITDB_ROOT_USERNAME=root
+MONGO_INITDB_ROOT_PASSWORD=toor

prod/docker-compose.yml

@@ -0,0 +1,63 @@
+services:
+    mongodb:
+        image: mongo:8.0.9-noble
+        restart: unless-stopped
+        # ports:
+        #     - 27017:27017
+        env_file: ./config/mongodb/.env
+        volumes:
+            - mongodb_data:/data/db
+        healthcheck:
+            test: echo 'db.runCommand("ping").ok' | mongosh --quiet
+            interval: 10s
+            timeout: 5s
+            retries: 5
+            start_period: 20s
+
+    consumer:
+        image: git.aykhans.me/bsky/feedgen-consumer:latest
+        restart: unless-stopped
+        env_file:
+            - ./config/app/.mongodb.env
+            - ./config/app/.consumer.env
+        depends_on:
+            mongodb:
+                condition: service_healthy
+
+    feedgen_az:
+        image: git.aykhans.me/bsky/feedgen-generator-az:latest
+        restart: unless-stopped
+        env_file:
+            - ./config/app/.mongodb.env
+            - ./config/app/feedgen/.az.env
+        depends_on:
+            mongodb:
+                condition: service_healthy
+
+    api:
+        image: git.aykhans.me/bsky/feedgen-api:latest
+        restart: unless-stopped
+        ports:
+            - 8421:8421
+        env_file:
+            - ./config/app/.mongodb.env
+            - ./config/app/.api.env
+        depends_on:
+            mongodb:
+                condition: service_healthy
+
+    caddy:
+        image: caddy:2.10.0-alpine
+        restart: unless-stopped
+        ports:
+            - 80:80
+            - 443:443
+            - 443:443/udp
+        env_file: ./config/caddy/.env
+        volumes:
+            - ./config/caddy/Caddyfile:/etc/caddy/Caddyfile
+            - caddy_data:/data
+
+volumes:
+    mongodb_data:
+    caddy_data: