This commit is contained in:
2025-05-19 01:49:56 +04:00
commit e6fec752f9
52 changed files with 4337 additions and 0 deletions

17
cmd/api/Dockerfile Normal file
View File

@@ -0,0 +1,17 @@
FROM golang:1.24-alpine AS builder
WORKDIR /src
COPY go.mod go.sum ./
COPY ../../pkg ./pkg
COPY ../../cmd/api ./cmd/api
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o api ./cmd/api/main.go
FROM gcr.io/distroless/static-debian12:latest
WORKDIR /app
COPY --from=builder /src/api .
ENTRYPOINT ["/app/api"]

30
cmd/api/README.md Normal file
View File

@@ -0,0 +1,30 @@
# API Service
## Overview
The API service is responsible for serving custom Bluesky feeds to clients. It implements the necessary endpoints required by the Bluesky protocol to deliver feed content.
**Pre-Built Docker Image**: `git.aykhans.me/bsky/feedgen-api:latest`
## API Endpoints
- `GET /.well-known/did.json`: DID configuration
- `GET /xrpc/app.bsky.feed.describeFeedGenerator`: Describe the feed generator
- `GET /xrpc/app.bsky.feed.getFeedSkeleton`: Main feed endpoint
## Running the Service
### Docker
```bash
docker build -f cmd/api/Dockerfile -t bsky-feedgen-api .
docker run --env-file config/app/.api.env --env-file config/app/.mongodb.env -p 8421:8421 bsky-feedgen-api
```
### Local Development
```bash
task run-api
# or
make run-api
```

60
cmd/api/main.go Normal file
View File

@@ -0,0 +1,60 @@
package main
import (
"context"
"os"
"os/signal"
"syscall"
"github.com/aykhans/bsky-feedgen/pkg/api"
"github.com/aykhans/bsky-feedgen/pkg/config"
"github.com/aykhans/bsky-feedgen/pkg/feed"
"github.com/aykhans/bsky-feedgen/pkg/logger"
"github.com/aykhans/bsky-feedgen/pkg/storage/mongodb"
"github.com/aykhans/bsky-feedgen/pkg/storage/mongodb/collections"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go listenForTermination(func() { cancel() })
apiConfig, errMap := config.NewAPIConfig()
if errMap != nil {
logger.Log.Error("API ENV error", "error", errMap.ToStringMap())
os.Exit(1)
}
mongoDBConfig, errMap := config.NewMongoDBConfig()
if errMap != nil {
logger.Log.Error("mongodb ENV error", "error", errMap.ToStringMap())
os.Exit(1)
}
client, err := mongodb.NewDB(ctx, mongoDBConfig)
if err != nil {
logger.Log.Error("mongodb connection error", "error", err)
os.Exit(1)
}
feedAzCollection, err := collections.NewFeedAzCollection(client)
if err != nil {
logger.Log.Error(err.Error())
os.Exit(1)
}
feeds := []feed.Feed{
feed.NewFeedAz("AzPulse", apiConfig.FeedgenPublisherDID, feedAzCollection),
}
if err := api.Run(ctx, apiConfig, feeds); err != nil {
logger.Log.Error("API error", "error", err)
}
}
func listenForTermination(do func()) {
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
<-sigChan
do()
}

17
cmd/consumer/Dockerfile Normal file
View File

@@ -0,0 +1,17 @@
FROM golang:1.24-alpine AS builder
WORKDIR /src
COPY go.mod go.sum ./
COPY ../../pkg ./pkg
COPY ../../cmd/consumer ./cmd/consumer
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o consumer ./cmd/consumer/main.go
FROM gcr.io/distroless/static-debian12:latest
WORKDIR /app
COPY --from=builder /src/consumer .
ENTRYPOINT ["/app/consumer"]

40
cmd/consumer/README.md Normal file
View File

@@ -0,0 +1,40 @@
# Consumer Service
## Overview
The Consumer service is responsible for connecting to the Bluesky firehose, processing incoming posts, and storing them in MongoDB for later use by the feed generator.
**Pre-Built Docker Image**: `git.aykhans.me/bsky/feedgen-consumer:latest`
## Features
- Connects to the Bluesky firehose websocket
- Processes and filters incoming posts
- Stores relevant post data in MongoDB
- Includes data management via cron jobs
- Implements collection size limits
- Prunes older data to prevent storage issues
## Command Line Options
- `-cursor`: Specify the starting point for data consumption
- `last-consumed`: Resume from the last processed data (default)
- `first-stream`: Start from the beginning of the firehose
- `current-stream`: Start from the current position in the firehose
## Running the Service
### Docker
```bash
docker build -f cmd/consumer/Dockerfile -t bsky-feedgen-consumer .
docker --env-file config/app/.consumer.env --env-file config/app/.mongodb.env run bsky-feedgen-consumer
```
### Local Development
```bash
task run-consumer
# or
make run-consumer
```

122
cmd/consumer/main.go Normal file
View File

@@ -0,0 +1,122 @@
package main
import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/aykhans/bsky-feedgen/pkg/consumer"
"github.com/aykhans/bsky-feedgen/pkg/types"
"github.com/aykhans/bsky-feedgen/pkg/config"
"github.com/aykhans/bsky-feedgen/pkg/logger"
"github.com/aykhans/bsky-feedgen/pkg/storage/mongodb"
"github.com/aykhans/bsky-feedgen/pkg/storage/mongodb/collections"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go listenForTermination(func() { cancel() })
flag.Usage = func() {
fmt.Println(
`Usage:
consumer [flags]
Flags:
-h, -help Display this help message
-cursor string Specify the starting point for data consumption (default: last-consumed)
Options:
last-consumed: Resume from the last processed data in storage
first-stream: Start from the beginning of the firehose
current-stream: Start from the current position in the firehose stream`)
}
var cursorOption types.ConsumerCursor
flag.Var(&cursorOption, "cursor", "")
flag.Parse()
if args := flag.Args(); len(args) > 0 {
if len(args) == 1 {
fmt.Printf("unexpected argument: %s\n\n", args[0])
} else {
fmt.Printf("unexpected arguments: %v\n\n", strings.Join(args, ", "))
}
flag.CommandLine.Usage()
os.Exit(1)
}
if cursorOption == "" {
_ = cursorOption.Set("")
}
consumerConfig, errMap := config.NewConsumerConfig()
if errMap != nil {
logger.Log.Error("consumer ENV error", "error", errMap.ToStringMap())
os.Exit(1)
}
mongoDBConfig, errMap := config.NewMongoDBConfig()
if errMap != nil {
logger.Log.Error("mongodb ENV error", "error", errMap.ToStringMap())
os.Exit(1)
}
client, err := mongodb.NewDB(ctx, mongoDBConfig)
if err != nil {
logger.Log.Error("mongodb connection error", "error", err)
os.Exit(1)
}
postCollection, err := collections.NewPostCollection(client)
if err != nil {
logger.Log.Error(err.Error())
os.Exit(1)
}
startCrons(ctx, consumerConfig, postCollection)
logger.Log.Info("Cron jobs started")
err = consumer.ConsumeAndSaveToMongoDB(
ctx,
postCollection,
"wss://bsky.network",
cursorOption,
consumerConfig.PostMaxDate, // Save only posts created before PostMaxDate
10*time.Second, // Save consumed data to MongoDB every 10 seconds
)
if err != nil {
logger.Log.Error(err.Error())
}
}
func startCrons(ctx context.Context, consumerConfig *config.ConsumerConfig, postCollection *collections.PostCollection) {
// Post collection cutoff
go func() {
for {
startTime := time.Now()
deleteCount, err := postCollection.CutoffByCount(ctx, consumerConfig.PostCollectionCutoffCronMaxDocument)
if err != nil {
logger.Log.Error("Post collection cutoff cron error", "error", err)
}
elapsedTime := time.Since(startTime)
logger.Log.Info("Post collection cutoff cron completed", "count", deleteCount, "time", elapsedTime)
time.Sleep(consumerConfig.PostCollectionCutoffCronDelay)
}
}()
}
func listenForTermination(do func()) {
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
<-sigChan
do()
}

17
cmd/feedgen/az/Dockerfile Normal file
View File

@@ -0,0 +1,17 @@
FROM golang:1.24-alpine AS builder
WORKDIR /src
COPY go.mod go.sum ./
COPY ../../pkg ./pkg
COPY ../../cmd/feedgen/az ./cmd/feedgen/az
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o feedgen ./cmd/feedgen/az/main.go
FROM gcr.io/distroless/static-debian12:latest
WORKDIR /app
COPY --from=builder /src/feedgen .
ENTRYPOINT ["/app/feedgen"]

38
cmd/feedgen/az/README.md Normal file
View File

@@ -0,0 +1,38 @@
# AzPulse Feed Generator Service
## Overview
The AzPulse Feed Generator service processes posts stored by the Consumer service and generates feed content that will be served by the API service. It implements the logic for the "AzPulse" feed, which showcases selected content from the Bluesky network.
**Pre-Built Docker Image**: `git.aykhans.me/bsky/feedgen-generator-az:latest`
## Features
- Processes posts from MongoDB
- Applies custom feed generation logic for the AzPulse feed
- Stores feed results in MongoDB for API service to access
- Manages feed data lifecycle with automatic pruning
- Runs as a background service with cron jobs
## Command Line Options
- `-cursor`: Specify the starting point for feed data generation
- `last-generated`: Resume from the last generated data (default)
- `first-post`: Start from the beginning of the posts collection
## Running the Service
### Docker
```bash
docker build -f cmd/feedgen/az/Dockerfile -t bsky-feedgen-az .
docker --env-file config/app/feedgen/.az.env --enf-file config/app/.mongodb.env run bsky-feedgen-az
```
### Local Development
```bash
task run-feedgen-az
# or
make run-feedgen-az
```

140
cmd/feedgen/az/main.go Normal file
View File

@@ -0,0 +1,140 @@
package main
import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/aykhans/bsky-feedgen/pkg/generator"
"github.com/aykhans/bsky-feedgen/pkg/types"
"github.com/aykhans/bsky-feedgen/pkg/config"
"github.com/aykhans/bsky-feedgen/pkg/logger"
"github.com/aykhans/bsky-feedgen/pkg/storage/mongodb"
"github.com/aykhans/bsky-feedgen/pkg/storage/mongodb/collections"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go listenForTermination(func() { cancel() })
flag.Usage = func() {
fmt.Println(
`Usage:
feedgen-az [flags]
Flags:
-h, -help Display this help message
-cursor string Specify the starting point for feed data generation (default: last-generated)
Options:
last-generated: Resume from the last generated data in storage
first-post: Start from the beginning of the posts`)
}
var cursorOption types.GeneratorCursor
flag.Var(&cursorOption, "cursor", "")
flag.Parse()
if args := flag.Args(); len(args) > 0 {
if len(args) == 1 {
fmt.Printf("unexpected argument: %s\n\n", args[0])
} else {
fmt.Printf("unexpected arguments: %v\n\n", strings.Join(args, ", "))
}
flag.CommandLine.Usage()
os.Exit(1)
}
if cursorOption == "" {
_ = cursorOption.Set("")
}
feedGenAzConfig, errMap := config.NewFeedGenAzConfig()
if errMap != nil {
logger.Log.Error("feedGenAzConfig ENV error", "error", errMap.ToStringMap())
os.Exit(1)
}
mongoDBConfig, errMap := config.NewMongoDBConfig()
if errMap != nil {
logger.Log.Error("mongodb ENV error", "error", errMap.ToStringMap())
os.Exit(1)
}
client, err := mongodb.NewDB(ctx, mongoDBConfig)
if err != nil {
logger.Log.Error("mongodb connection error", "error", err)
os.Exit(1)
}
postCollection, err := collections.NewPostCollection(client)
if err != nil {
logger.Log.Error(err.Error())
os.Exit(1)
}
feedAzCollection, err := collections.NewFeedAzCollection(client)
if err != nil {
logger.Log.Error(err.Error())
os.Exit(1)
}
feedGeneratorAz := generator.NewFeedGeneratorAz(postCollection, feedAzCollection)
startCrons(ctx, feedGenAzConfig, feedGeneratorAz, feedAzCollection, cursorOption)
logger.Log.Info("Cron jobs started")
<-ctx.Done()
}
func startCrons(
ctx context.Context,
feedGenAzConfig *config.FeedGenAzConfig,
feedGeneratorAz *generator.FeedGeneratorAz,
feedAzCollection *collections.FeedAzCollection,
cursorOption types.GeneratorCursor,
) {
// Feed az generator
go func() {
for {
startTime := time.Now()
err := feedGeneratorAz.Start(ctx, cursorOption, 1)
if err != nil {
logger.Log.Error("Feed az generator cron error", "error", err)
}
elapsedTime := time.Since(startTime)
logger.Log.Info("Feed az generator cron completed", "time", elapsedTime)
time.Sleep(feedGenAzConfig.GeneratorCronDelay)
}
}()
// feed_az collection cutoff
go func() {
for {
startTime := time.Now()
deleteCount, err := feedAzCollection.CutoffByCount(ctx, feedGenAzConfig.CollectionMaxDocument)
if err != nil {
logger.Log.Error("feed_az collection cutoff cron error", "error", err)
}
elapsedTime := time.Since(startTime)
logger.Log.Info("feed_az collection cutoff cron completed", "count", deleteCount, "time", elapsedTime)
time.Sleep(feedGenAzConfig.CutoffCronDelay)
}
}()
}
func listenForTermination(do func()) {
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
<-sigChan
do()
}

18
cmd/manager/Dockerfile Normal file
View File

@@ -0,0 +1,18 @@
FROM golang:1.24-alpine AS builder
WORKDIR /src
COPY go.mod go.sum ./
COPY ../../pkg ./pkg
COPY ../../cmd/manager ./cmd/manager
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o manager ./cmd/manager/main.go
FROM gcr.io/distroless/static-debian12:latest
WORKDIR /app
COPY --from=builder /src/manager .
ENTRYPOINT ["/app/manager"]
CMD ["help"]

47
cmd/manager/README.md Normal file
View File

@@ -0,0 +1,47 @@
# Feed Manager CLI Tool
## Overview
The Feed Manager is a command-line interface (CLI) tool that allows users to create, update, and delete feed generator records on the Bluesky network.
**Pre-Built Docker Image**: `git.aykhans.me/bsky/feedgen-manager:latest`
## Commands
### Create a Feed Generator
```bash
task run-manager create
# or
make run-manager create
```
This will prompt for:
- Your Bluesky handle
- Your Bluesky password
- Feed generator hostname
- Record short name (for the URL)
- Display name
- Description (optional)
- Avatar image path (optional)
### Update a Feed Generator
```bash
task run-manager update
# or
make run-manager update
```
Allows updating the properties of an existing feed generator record.
### Delete a Feed Generator
```bash
task run-manager delete
# or
make run-manager delete
```
Permenantly removes a feed generator record from the Bluesky network.

268
cmd/manager/main.go Normal file
View File

@@ -0,0 +1,268 @@
package main
import (
"bufio"
"context"
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"github.com/aykhans/bsky-feedgen/pkg/manage"
"github.com/spf13/cobra"
"golang.org/x/term"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go listenForTermination(func() {
cancel()
fmt.Println()
os.Exit(130)
})
var rootCmd = &cobra.Command{
Use: "feedgen-manager",
Short: "BlueSkye Feed Generator client CLI",
Long: "A command-line tool for managing feed generators on Bluesky",
CompletionOptions: cobra.CompletionOptions{HiddenDefaultCmd: true},
}
var createCmd = &cobra.Command{
Use: "create",
Short: "Create a feed generator record",
Run: func(cmd *cobra.Command, args []string) {
handle := prompt("Enter your Bluesky handle", "")
if handle == "" {
fmt.Println("\nError: handle is required")
os.Exit(1)
}
password := promptPassword("Enter your Bluesky password (preferably an App Password)")
if handle == "" {
fmt.Println("\nError: password is required")
os.Exit(1)
}
service := prompt("Optionally, enter a custom PDS service to sign in with", manage.DefaultPDSHost)
feedgenHostname := prompt("Enter the feed generator hostname (e.g. 'feeds.bsky.example.com')", "")
if feedgenHostname == "" {
fmt.Println("\nError: hostname is required")
os.Exit(1)
}
recordKey := prompt("Enter a short name for the record. This will be shown in the feed's URL and should be unique.", "")
if recordKey == "" {
fmt.Println("\nError: short name is required")
os.Exit(1)
}
displayName := prompt("Enter a display name for your feed", "")
if displayName == "" {
fmt.Println("\nError: display name is required")
os.Exit(1)
}
description := prompt("Optionally, enter a brief description of your feed", "")
avatar := prompt("Optionally, enter a local path to an avatar that will be used for the feed", "")
client, err := manage.NewClientWithAuth(ctx, manage.NewClient(&service), handle, password)
if err != nil {
fmt.Printf("\nAuthentication failed: %v", err)
os.Exit(1)
}
err = manage.CreateFeedGenerator(
ctx,
client,
displayName,
toPtr(description),
toPtr(avatar),
"did:web:"+feedgenHostname,
recordKey,
)
if err != nil {
fmt.Printf("\nFailed to create feed generator record: %v\n", err)
os.Exit(1)
}
fmt.Println("\nFeed generator created successfully! 🎉")
},
}
var updateCmd = &cobra.Command{
Use: "update",
Short: "Update a feed generator record",
Run: func(cmd *cobra.Command, args []string) {
handle := prompt("Enter your Bluesky handle", "")
if handle == "" {
fmt.Println("\nError: handle is required")
os.Exit(1)
}
password := promptPassword("Enter your Bluesky password (preferably an App Password)")
if handle == "" {
fmt.Println("\nError: password is required")
os.Exit(1)
}
service := prompt("Optionally, enter a custom PDS service to sign in with", manage.DefaultPDSHost)
feedgenHostname := prompt("Optionally, enter the feed generator hostname (e.g. 'feeds.bsky.example.com')", "")
recordKey := prompt("Enter short name of the record", "")
if recordKey == "" {
fmt.Println("\nError: short name is required")
os.Exit(1)
}
displayName := prompt("Optionally, enter a display name for your feed", "")
description := prompt("Optionally, enter a brief description of your feed", "")
avatar := prompt("Optionally, enter a local path to an avatar that will be used for the feed", "")
client, err := manage.NewClientWithAuth(ctx, manage.NewClient(&service), handle, password)
if err != nil {
fmt.Printf("\nAuthentication failed: %v", err)
os.Exit(1)
}
var did *string
if feedgenHostname != "" {
did = toPtr("did:web:" + feedgenHostname)
}
err = manage.UpdateFeedGenerator(
ctx,
client,
toPtr(displayName),
toPtr(description),
toPtr(avatar),
did,
recordKey,
)
if err != nil {
fmt.Printf("\nFailed to update feed generator record: %v\n", err)
os.Exit(1)
}
fmt.Println("\nFeed generator updated successfully! 🎉")
},
}
var deleteCmd = &cobra.Command{
Use: "delete",
Short: "Delete a feed generator record",
Run: func(cmd *cobra.Command, args []string) {
handle := prompt("Enter your Bluesky handle", "")
if handle == "" {
fmt.Println("\nError: handle is required")
os.Exit(1)
}
password := promptPassword("Enter your Bluesky password (preferably an App Password)")
if handle == "" {
fmt.Println("\nError: password is required")
os.Exit(1)
}
service := prompt("Optionally, enter a custom PDS service to sign in with", manage.DefaultPDSHost)
recordKey := prompt("Enter short name of the record", "")
if recordKey == "" {
fmt.Println("\nError: short name is required")
os.Exit(1)
}
confirm := promptConfirm("Are you sure you want to delete this record? Any likes that your feed has will be lost", false)
if !confirm {
fmt.Println("\nAborting...")
return
}
client, err := manage.NewClientWithAuth(ctx, manage.NewClient(&service), handle, password)
if err != nil {
fmt.Printf("\nAuthentication failed: %v", err)
os.Exit(1)
}
err = manage.DeleteFeedGenerator(ctx, client, recordKey)
if err != nil {
fmt.Printf("\nFailed to delete feed generator record: %v\n", err)
os.Exit(1)
}
fmt.Println("\nFeed generator deleted successfully! 🎉")
},
}
rootCmd.AddCommand(createCmd)
rootCmd.AddCommand(updateCmd)
rootCmd.AddCommand(deleteCmd)
if err := rootCmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
// <-------------- Utils -------------->
func listenForTermination(do func()) {
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
<-sigChan
do()
}
func toPtr[T comparable](value T) *T {
var zero T
if value == zero {
return nil
}
return &value
}
func prompt(label string, defaultValue string) string {
if defaultValue != "" {
fmt.Printf("%s [%s]: ", label, defaultValue)
} else {
fmt.Printf("%s: ", label)
}
reader := bufio.NewReader(os.Stdin)
input, _ := reader.ReadString('\n')
input = strings.TrimSpace(input)
if input == "" {
return defaultValue
}
return input
}
func promptPassword(label string) string {
fmt.Printf("%s: ", label)
password, _ := term.ReadPassword(int(os.Stdin.Fd()))
fmt.Println()
return string(password)
}
func promptConfirm(label string, defaultValue bool) bool {
defaultStr := "y/N"
if defaultValue {
defaultStr = "Y/n"
}
fmt.Printf("%s [%s]: ", label, defaultStr)
reader := bufio.NewReader(os.Stdin)
input, _ := reader.ReadString('\n')
input = strings.ToLower(strings.TrimSpace(input))
if input == "" {
return defaultValue
}
return input == "y" || input == "yes"
}