mirror of
https://github.com/aykhans/bsky-feedgen.git
synced 2025-07-17 21:34:00 +00:00
Compare commits
15 Commits
feat/add-a
...
919fec0aa9
Author | SHA1 | Date | |
---|---|---|---|
919fec0aa9 | |||
3097aba9c3 | |||
6799ad241e | |||
3378baf0bc | |||
94df1dd259 | |||
ab3baf76b1 | |||
3c6fb06282 | |||
ea9d7bb67e | |||
56581c7332 | |||
8ea4602141 | |||
12e51b5a22 | |||
50af73f26a | |||
35907380fb | |||
bad7b4a304 | |||
9917f61db1 |
60
Makefile
60
Makefile
@@ -1,60 +0,0 @@
|
|||||||
# Equivalent Makefile for Taskfile.yaml
|
|
||||||
|
|
||||||
.PHONY: ftl fmt tidy lint run-consumer run-feedgen-az run-api run-manager generate-env
|
|
||||||
|
|
||||||
# Default value for ARGS if not provided on the command line
|
|
||||||
ARGS ?=
|
|
||||||
|
|
||||||
# Runs fmt, tidy, and lint sequentially
|
|
||||||
ftl:
|
|
||||||
$(MAKE) fmt
|
|
||||||
$(MAKE) tidy
|
|
||||||
$(MAKE) lint
|
|
||||||
|
|
||||||
# Format Go code
|
|
||||||
fmt:
|
|
||||||
gofmt -w -d .
|
|
||||||
|
|
||||||
# Tidy Go modules
|
|
||||||
tidy:
|
|
||||||
go mod tidy
|
|
||||||
|
|
||||||
# Run golangci-lint
|
|
||||||
lint:
|
|
||||||
golangci-lint run
|
|
||||||
|
|
||||||
# Run the consumer application, loading environment from dotenv files
|
|
||||||
run-consumer:
|
|
||||||
set -a; \
|
|
||||||
. config/app/.consumer.env; \
|
|
||||||
. config/app/.mongodb.env; \
|
|
||||||
set +a; \
|
|
||||||
go run cmd/consumer/main.go $(ARGS)
|
|
||||||
|
|
||||||
# Run the feedgen-az application, loading environment from dotenv files
|
|
||||||
run-feedgen-az:
|
|
||||||
set -a; \
|
|
||||||
. config/app/feedgen/.az.env; \
|
|
||||||
. config/app/.mongodb.env; \
|
|
||||||
set +a; \
|
|
||||||
go run cmd/feedgen/az/main.go $(ARGS)
|
|
||||||
|
|
||||||
# Run the api application, loading environment from dotenv files
|
|
||||||
run-api:
|
|
||||||
set -a; \
|
|
||||||
. config/app/.api.env; \
|
|
||||||
. config/app/.mongodb.env; \
|
|
||||||
set +a; \
|
|
||||||
go run cmd/api/main.go
|
|
||||||
|
|
||||||
# Run the manager application with arguments (no dotenv)
|
|
||||||
run-manager:
|
|
||||||
go run cmd/manager/main.go $(ARGS)
|
|
||||||
|
|
||||||
# Generate env files from templates
|
|
||||||
generate-env:
|
|
||||||
cp config/app/consumer.env.example config/app/.consumer.env
|
|
||||||
cp config/app/api.env.example config/app/.api.env
|
|
||||||
cp config/app/mongodb.env.example config/app/.mongodb.env
|
|
||||||
cp config/app/feedgen/az.env.example config/app/feedgen/.az.env
|
|
||||||
cp config/mongodb/env.example config/mongodb/.env
|
|
55
Taskfile.yml
55
Taskfile.yml
@@ -2,6 +2,9 @@
|
|||||||
|
|
||||||
version: "3"
|
version: "3"
|
||||||
|
|
||||||
|
vars:
|
||||||
|
DOCKER_REGISTRY: "git.aykhans.me/bsky/"
|
||||||
|
|
||||||
tasks:
|
tasks:
|
||||||
ftl:
|
ftl:
|
||||||
cmds:
|
cmds:
|
||||||
@@ -16,19 +19,19 @@ tasks:
|
|||||||
lint: golangci-lint run
|
lint: golangci-lint run
|
||||||
|
|
||||||
run-consumer:
|
run-consumer:
|
||||||
cmd: go run cmd/consumer/main.go {{.CLI_ARGS}}
|
cmd: go run ./cmd/consumer {{.CLI_ARGS}}
|
||||||
dotenv:
|
dotenv:
|
||||||
- config/app/.consumer.env
|
- config/app/.consumer.env
|
||||||
- config/app/.mongodb.env
|
- config/app/.mongodb.env
|
||||||
|
|
||||||
run-feedgen-az:
|
run-feedgen-az:
|
||||||
cmd: go run cmd/feedgen/az/main.go {{.CLI_ARGS}}
|
cmd: go run ./cmd/feedgen/az {{.CLI_ARGS}}
|
||||||
dotenv:
|
dotenv:
|
||||||
- config/app/feedgen/.az.env
|
- config/app/feedgen/.az.env
|
||||||
- config/app/.mongodb.env
|
- config/app/.mongodb.env
|
||||||
|
|
||||||
run-api:
|
run-api:
|
||||||
cmd: go run cmd/api/main.go
|
cmd: go run ./cmd/api {{.CLI_ARGS}}
|
||||||
dotenv:
|
dotenv:
|
||||||
- config/app/.api.env
|
- config/app/.api.env
|
||||||
- config/app/.mongodb.env
|
- config/app/.mongodb.env
|
||||||
@@ -55,21 +58,55 @@ tasks:
|
|||||||
|
|
||||||
docker-publish-api:
|
docker-publish-api:
|
||||||
desc: Publish docker image for api service
|
desc: Publish docker image for api service
|
||||||
|
vars:
|
||||||
|
GO_VERSION_FILE: ./cmd/api/version.go
|
||||||
|
IMAGE_NAME: feedgen-api
|
||||||
|
VERSION:
|
||||||
|
sh: grep -o 'const version = "[^"]*"' {{.GO_VERSION_FILE}} | grep -o '"[^"]*"' | tr -d '"'
|
||||||
|
VERSIONED_IMAGE: "{{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:{{.VERSION}}"
|
||||||
|
LATEST_IMAGE: "{{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:latest"
|
||||||
|
preconditions:
|
||||||
|
- test -f {{.GO_VERSION_FILE}}
|
||||||
|
- sh: '[ -n "{{.VERSION}}" ]'
|
||||||
|
msg: "Could not extract version from {{.GO_FILE}}"
|
||||||
cmds:
|
cmds:
|
||||||
- docker build -t git.aykhans.me/bsky/feedgen-api:latest -f ./cmd/api/Dockerfile .
|
- docker build -t {{.VERSIONED_IMAGE}} -f ./cmd/api/Dockerfile .
|
||||||
- docker push git.aykhans.me/bsky/feedgen-api:latest
|
- docker tag {{.VERSIONED_IMAGE}} {{.LATEST_IMAGE}}
|
||||||
|
- docker push {{.VERSIONED_IMAGE}}
|
||||||
|
- docker push {{.LATEST_IMAGE}}
|
||||||
|
- echo "Published {{.VERSIONED_IMAGE}} and {{.LATEST_IMAGE}}"
|
||||||
|
|
||||||
docker-publish-consumer:
|
docker-publish-consumer:
|
||||||
desc: Publish docker image for consumer service
|
desc: Publish docker image for consumer service
|
||||||
|
vars:
|
||||||
|
GO_VERSION_FILE: ./cmd/consumer/version.go
|
||||||
|
IMAGE_NAME: feedgen-consumer
|
||||||
|
VERSION:
|
||||||
|
sh: grep -o 'const version = "[^"]*"' {{.GO_VERSION_FILE}} | grep -o '"[^"]*"' | tr -d '"'
|
||||||
|
VERSIONED_IMAGE: "{{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:{{.VERSION}}"
|
||||||
|
LATEST_IMAGE: "{{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:latest"
|
||||||
cmds:
|
cmds:
|
||||||
- docker build -t git.aykhans.me/bsky/feedgen-consumer:latest -f ./cmd/consumer/Dockerfile .
|
- docker build -t {{.VERSIONED_IMAGE}} -f ./cmd/consumer/Dockerfile .
|
||||||
- docker push git.aykhans.me/bsky/feedgen-consumer:latest
|
- docker push {{.VERSIONED_IMAGE}}
|
||||||
|
- docker tag {{.VERSIONED_IMAGE}} {{.LATEST_IMAGE}}
|
||||||
|
- docker push {{.LATEST_IMAGE}}
|
||||||
|
- echo "Published {{.VERSIONED_IMAGE}} and {{.LATEST_IMAGE}}"
|
||||||
|
|
||||||
docker-publish-feedgen-az:
|
docker-publish-feedgen-az:
|
||||||
desc: Publish docker image for feedgen-az service
|
desc: Publish docker image for feedgen-az service
|
||||||
|
vars:
|
||||||
|
GO_VERSION_FILE: ./cmd/feedgen/az/version.go
|
||||||
|
IMAGE_NAME: feedgen-generator-az
|
||||||
|
VERSION:
|
||||||
|
sh: grep -o 'const version = "[^"]*"' {{.GO_VERSION_FILE}} | grep -o '"[^"]*"' | tr -d '"'
|
||||||
|
VERSIONED_IMAGE: "{{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:{{.VERSION}}"
|
||||||
|
LATEST_IMAGE: "{{.DOCKER_REGISTRY}}{{.IMAGE_NAME}}:latest"
|
||||||
cmds:
|
cmds:
|
||||||
- docker build -t git.aykhans.me/bsky/feedgen-generator-az:latest -f ./cmd/feedgen/az/Dockerfile .
|
- docker build -t {{.VERSIONED_IMAGE}} -f ./cmd/feedgen/az/Dockerfile .
|
||||||
- docker push git.aykhans.me/bsky/feedgen-generator-az:latest
|
- docker push {{.VERSIONED_IMAGE}}
|
||||||
|
- docker tag {{.VERSIONED_IMAGE}} {{.LATEST_IMAGE}}
|
||||||
|
- docker push {{.LATEST_IMAGE}}
|
||||||
|
- echo "Published {{.VERSIONED_IMAGE}} and {{.LATEST_IMAGE}}"
|
||||||
|
|
||||||
docker-publish-manager:
|
docker-publish-manager:
|
||||||
desc: Publish docker image for manager service
|
desc: Publish docker image for manager service
|
||||||
|
@@ -6,7 +6,7 @@ COPY go.mod go.sum ./
|
|||||||
COPY ../../pkg ./pkg
|
COPY ../../pkg ./pkg
|
||||||
COPY ../../cmd/api ./cmd/api
|
COPY ../../cmd/api ./cmd/api
|
||||||
|
|
||||||
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o api ./cmd/api/main.go
|
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o api ./cmd/api
|
||||||
|
|
||||||
FROM gcr.io/distroless/static-debian12:latest
|
FROM gcr.io/distroless/static-debian12:latest
|
||||||
|
|
||||||
|
@@ -2,8 +2,11 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
"github.com/aykhans/bsky-feedgen/pkg/api"
|
"github.com/aykhans/bsky-feedgen/pkg/api"
|
||||||
@@ -15,11 +18,21 @@ import (
|
|||||||
_ "go.uber.org/automaxprocs"
|
_ "go.uber.org/automaxprocs"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type flags struct {
|
||||||
|
version bool
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
go listenForTermination(func() { cancel() })
|
go listenForTermination(func() { cancel() })
|
||||||
|
|
||||||
|
flags := getFlags()
|
||||||
|
if flags.version == true {
|
||||||
|
fmt.Printf("API version: %v\n", version)
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
apiConfig, errMap := config.NewAPIConfig()
|
apiConfig, errMap := config.NewAPIConfig()
|
||||||
if errMap != nil {
|
if errMap != nil {
|
||||||
logger.Log.Error("API ENV error", "error", errMap.ToStringMap())
|
logger.Log.Error("API ENV error", "error", errMap.ToStringMap())
|
||||||
@@ -59,3 +72,33 @@ func listenForTermination(do func()) {
|
|||||||
<-sigChan
|
<-sigChan
|
||||||
do()
|
do()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getFlags() *flags {
|
||||||
|
flags := &flags{}
|
||||||
|
|
||||||
|
flag.Usage = func() {
|
||||||
|
fmt.Println(
|
||||||
|
`Usage:
|
||||||
|
|
||||||
|
consumer [flags]
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
-version version information
|
||||||
|
-h, -help Display this help message`)
|
||||||
|
}
|
||||||
|
|
||||||
|
flag.BoolVar(&flags.version, "version", false, "print version information")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
if args := flag.Args(); len(args) > 0 {
|
||||||
|
if len(args) == 1 {
|
||||||
|
fmt.Printf("unexpected argument: %s\n\n", args[0])
|
||||||
|
} else {
|
||||||
|
fmt.Printf("unexpected arguments: %v\n\n", strings.Join(args, ", "))
|
||||||
|
}
|
||||||
|
flag.CommandLine.Usage()
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return flags
|
||||||
|
}
|
||||||
|
3
cmd/api/version.go
Normal file
3
cmd/api/version.go
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
const version = "0.2.103"
|
@@ -6,7 +6,7 @@ COPY go.mod go.sum ./
|
|||||||
COPY ../../pkg ./pkg
|
COPY ../../pkg ./pkg
|
||||||
COPY ../../cmd/consumer ./cmd/consumer
|
COPY ../../cmd/consumer ./cmd/consumer
|
||||||
|
|
||||||
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o consumer ./cmd/consumer/main.go
|
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o consumer ./cmd/consumer
|
||||||
|
|
||||||
FROM gcr.io/distroless/static-debian12:latest
|
FROM gcr.io/distroless/static-debian12:latest
|
||||||
|
|
||||||
|
@@ -20,42 +20,24 @@ import (
|
|||||||
_ "go.uber.org/automaxprocs"
|
_ "go.uber.org/automaxprocs"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type flags struct {
|
||||||
|
version bool
|
||||||
|
cursorOption types.ConsumerCursor
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
go listenForTermination(func() { cancel() })
|
go listenForTermination(func() { cancel() })
|
||||||
|
|
||||||
flag.Usage = func() {
|
flags := getFlags()
|
||||||
fmt.Println(
|
if flags.version == true {
|
||||||
`Usage:
|
fmt.Printf("Consumer version: %v\n", version)
|
||||||
|
os.Exit(0)
|
||||||
consumer [flags]
|
|
||||||
|
|
||||||
Flags:
|
|
||||||
-h, -help Display this help message
|
|
||||||
-cursor string Specify the starting point for data consumption (default: last-consumed)
|
|
||||||
Options:
|
|
||||||
last-consumed: Resume from the last processed data in storage
|
|
||||||
first-stream: Start from the beginning of the firehose
|
|
||||||
current-stream: Start from the current position in the firehose stream`)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var cursorOption types.ConsumerCursor
|
if flags.cursorOption == "" {
|
||||||
flag.Var(&cursorOption, "cursor", "")
|
_ = flags.cursorOption.Set("")
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
if args := flag.Args(); len(args) > 0 {
|
|
||||||
if len(args) == 1 {
|
|
||||||
fmt.Printf("unexpected argument: %s\n\n", args[0])
|
|
||||||
} else {
|
|
||||||
fmt.Printf("unexpected arguments: %v\n\n", strings.Join(args, ", "))
|
|
||||||
}
|
|
||||||
flag.CommandLine.Usage()
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cursorOption == "" {
|
|
||||||
_ = cursorOption.Set("")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
consumerConfig, errMap := config.NewConsumerConfig()
|
consumerConfig, errMap := config.NewConsumerConfig()
|
||||||
@@ -89,7 +71,7 @@ Flags:
|
|||||||
ctx,
|
ctx,
|
||||||
postCollection,
|
postCollection,
|
||||||
"wss://bsky.network",
|
"wss://bsky.network",
|
||||||
cursorOption,
|
flags.cursorOption,
|
||||||
consumerConfig.PostMaxDate, // Save only posts created before PostMaxDate
|
consumerConfig.PostMaxDate, // Save only posts created before PostMaxDate
|
||||||
10*time.Second, // Save consumed data to MongoDB every 10 seconds
|
10*time.Second, // Save consumed data to MongoDB every 10 seconds
|
||||||
)
|
)
|
||||||
@@ -121,3 +103,39 @@ func listenForTermination(do func()) {
|
|||||||
<-sigChan
|
<-sigChan
|
||||||
do()
|
do()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getFlags() *flags {
|
||||||
|
flags := &flags{}
|
||||||
|
|
||||||
|
flag.Usage = func() {
|
||||||
|
fmt.Println(
|
||||||
|
`Usage:
|
||||||
|
|
||||||
|
consumer [flags]
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
-version version information
|
||||||
|
-h, -help Display this help message
|
||||||
|
-cursor string Specify the starting point for data consumption (default: last-consumed)
|
||||||
|
Options:
|
||||||
|
last-consumed: Resume from the last processed data in storage
|
||||||
|
first-stream: Start from the beginning of the firehose
|
||||||
|
current-stream: Start from the current position in the firehose stream`)
|
||||||
|
}
|
||||||
|
|
||||||
|
flag.BoolVar(&flags.version, "version", false, "print version information")
|
||||||
|
flag.Var(&flags.cursorOption, "cursor", "Specify the starting point for data consumption")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
if args := flag.Args(); len(args) > 0 {
|
||||||
|
if len(args) == 1 {
|
||||||
|
fmt.Printf("unexpected argument: %s\n\n", args[0])
|
||||||
|
} else {
|
||||||
|
fmt.Printf("unexpected arguments: %v\n\n", strings.Join(args, ", "))
|
||||||
|
}
|
||||||
|
flag.CommandLine.Usage()
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return flags
|
||||||
|
}
|
||||||
|
3
cmd/consumer/version.go
Normal file
3
cmd/consumer/version.go
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
const version = "0.1.0"
|
@@ -6,7 +6,7 @@ COPY go.mod go.sum ./
|
|||||||
COPY ../../pkg ./pkg
|
COPY ../../pkg ./pkg
|
||||||
COPY ../../cmd/feedgen/az ./cmd/feedgen/az
|
COPY ../../cmd/feedgen/az ./cmd/feedgen/az
|
||||||
|
|
||||||
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o feedgen ./cmd/feedgen/az/main.go
|
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o feedgen ./cmd/feedgen/az
|
||||||
|
|
||||||
FROM gcr.io/distroless/static-debian12:latest
|
FROM gcr.io/distroless/static-debian12:latest
|
||||||
|
|
||||||
|
@@ -20,41 +20,24 @@ import (
|
|||||||
_ "go.uber.org/automaxprocs"
|
_ "go.uber.org/automaxprocs"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type flags struct {
|
||||||
|
version bool
|
||||||
|
cursorOption types.GeneratorCursor
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
go listenForTermination(func() { cancel() })
|
go listenForTermination(func() { cancel() })
|
||||||
|
|
||||||
flag.Usage = func() {
|
flags := getFlags()
|
||||||
fmt.Println(
|
if flags.version == true {
|
||||||
`Usage:
|
fmt.Printf("Feedgen Az version: %v\n", version)
|
||||||
|
os.Exit(0)
|
||||||
feedgen-az [flags]
|
|
||||||
|
|
||||||
Flags:
|
|
||||||
-h, -help Display this help message
|
|
||||||
-cursor string Specify the starting point for feed data generation (default: last-generated)
|
|
||||||
Options:
|
|
||||||
last-generated: Resume from the last generated data in storage
|
|
||||||
first-post: Start from the beginning of the posts`)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var cursorOption types.GeneratorCursor
|
if flags.cursorOption == "" {
|
||||||
flag.Var(&cursorOption, "cursor", "")
|
_ = flags.cursorOption.Set("")
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
if args := flag.Args(); len(args) > 0 {
|
|
||||||
if len(args) == 1 {
|
|
||||||
fmt.Printf("unexpected argument: %s\n\n", args[0])
|
|
||||||
} else {
|
|
||||||
fmt.Printf("unexpected arguments: %v\n\n", strings.Join(args, ", "))
|
|
||||||
}
|
|
||||||
flag.CommandLine.Usage()
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cursorOption == "" {
|
|
||||||
_ = cursorOption.Set("")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
feedGenAzConfig, errMap := config.NewFeedGenAzConfig()
|
feedGenAzConfig, errMap := config.NewFeedGenAzConfig()
|
||||||
@@ -89,7 +72,7 @@ Flags:
|
|||||||
|
|
||||||
feedGeneratorAz := feedgenAz.NewGenerator(postCollection, feedAzCollection)
|
feedGeneratorAz := feedgenAz.NewGenerator(postCollection, feedAzCollection)
|
||||||
|
|
||||||
startCrons(ctx, feedGenAzConfig, feedGeneratorAz, feedAzCollection, cursorOption)
|
startCrons(ctx, feedGenAzConfig, feedGeneratorAz, feedAzCollection, flags.cursorOption)
|
||||||
logger.Log.Info("Cron jobs started")
|
logger.Log.Info("Cron jobs started")
|
||||||
|
|
||||||
<-ctx.Done()
|
<-ctx.Done()
|
||||||
@@ -139,3 +122,38 @@ func listenForTermination(do func()) {
|
|||||||
<-sigChan
|
<-sigChan
|
||||||
do()
|
do()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getFlags() *flags {
|
||||||
|
flags := &flags{}
|
||||||
|
|
||||||
|
flag.Usage = func() {
|
||||||
|
fmt.Println(
|
||||||
|
`Usage:
|
||||||
|
|
||||||
|
feedgen-az [flags]
|
||||||
|
|
||||||
|
Flags:
|
||||||
|
-version version information
|
||||||
|
-h, -help Display this help message
|
||||||
|
-cursor string Specify the starting point for feed data generation (default: last-generated)
|
||||||
|
Options:
|
||||||
|
last-generated: Resume from the last generated data in storage
|
||||||
|
first-post: Start from the beginning of the posts`)
|
||||||
|
}
|
||||||
|
|
||||||
|
flag.BoolVar(&flags.version, "version", false, "print version information")
|
||||||
|
flag.Var(&flags.cursorOption, "cursor", "Specify the starting point for feed data generation")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
if args := flag.Args(); len(args) > 0 {
|
||||||
|
if len(args) == 1 {
|
||||||
|
fmt.Printf("unexpected argument: %s\n\n", args[0])
|
||||||
|
} else {
|
||||||
|
fmt.Printf("unexpected arguments: %v\n\n", strings.Join(args, ", "))
|
||||||
|
}
|
||||||
|
flag.CommandLine.Usage()
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return flags
|
||||||
|
}
|
||||||
|
3
cmd/feedgen/az/version.go
Normal file
3
cmd/feedgen/az/version.go
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
const version = "0.1.103"
|
@@ -34,6 +34,9 @@ func Run(
|
|||||||
"GET /xrpc/app.bsky.feed.getFeedSkeleton",
|
"GET /xrpc/app.bsky.feed.getFeedSkeleton",
|
||||||
authMiddleware.JWTAuthMiddleware(http.HandlerFunc(feedHandler.GetFeedSkeleton)),
|
authMiddleware.JWTAuthMiddleware(http.HandlerFunc(feedHandler.GetFeedSkeleton)),
|
||||||
)
|
)
|
||||||
|
mux.HandleFunc("GET /{feed}/users", feedHandler.GetAllUsers)
|
||||||
|
mux.HandleFunc("GET /{feed}/users/valid/", feedHandler.GetValidUsers)
|
||||||
|
mux.HandleFunc("GET /{feed}/users/invalid/", feedHandler.GetInvalidUsers)
|
||||||
|
|
||||||
httpServer := &http.Server{
|
httpServer := &http.Server{
|
||||||
Addr: fmt.Sprintf(":%d", apiConfig.APIPort),
|
Addr: fmt.Sprintf(":%d", apiConfig.APIPort),
|
||||||
|
@@ -10,6 +10,7 @@ import (
|
|||||||
"github.com/aykhans/bsky-feedgen/pkg/api/middleware"
|
"github.com/aykhans/bsky-feedgen/pkg/api/middleware"
|
||||||
"github.com/aykhans/bsky-feedgen/pkg/api/response"
|
"github.com/aykhans/bsky-feedgen/pkg/api/response"
|
||||||
"github.com/aykhans/bsky-feedgen/pkg/feed"
|
"github.com/aykhans/bsky-feedgen/pkg/feed"
|
||||||
|
generatorAz "github.com/aykhans/bsky-feedgen/pkg/generator/az"
|
||||||
"github.com/aykhans/bsky-feedgen/pkg/types"
|
"github.com/aykhans/bsky-feedgen/pkg/types"
|
||||||
"github.com/aykhans/bsky-feedgen/pkg/utils"
|
"github.com/aykhans/bsky-feedgen/pkg/utils"
|
||||||
"github.com/bluesky-social/indigo/api/bsky"
|
"github.com/bluesky-social/indigo/api/bsky"
|
||||||
@@ -99,3 +100,46 @@ func (handler *FeedHandler) GetFeedSkeleton(w http.ResponseWriter, r *http.Reque
|
|||||||
Cursor: newCursor,
|
Cursor: newCursor,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (handler *FeedHandler) GetValidUsers(w http.ResponseWriter, r *http.Request) {
|
||||||
|
feed := r.PathValue("feed")
|
||||||
|
|
||||||
|
validUsers := make([]string, 0)
|
||||||
|
switch feed {
|
||||||
|
case "AzPulse":
|
||||||
|
validUsers = generatorAz.Users.GetValidUsers()
|
||||||
|
}
|
||||||
|
|
||||||
|
response.JSON(w, 200, response.M{
|
||||||
|
"feed": feed,
|
||||||
|
"users": validUsers,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (handler *FeedHandler) GetInvalidUsers(w http.ResponseWriter, r *http.Request) {
|
||||||
|
feed := r.PathValue("feed")
|
||||||
|
|
||||||
|
invalidUsers := make([]string, 0)
|
||||||
|
switch feed {
|
||||||
|
case "AzPulse":
|
||||||
|
invalidUsers = generatorAz.Users.GetInvalidUsers()
|
||||||
|
}
|
||||||
|
|
||||||
|
response.JSON(w, 200, response.M{
|
||||||
|
"feed": feed,
|
||||||
|
"users": invalidUsers,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (handler *FeedHandler) GetAllUsers(w http.ResponseWriter, r *http.Request) {
|
||||||
|
feed := r.PathValue("feed")
|
||||||
|
|
||||||
|
responseData := response.M{"feed": feed}
|
||||||
|
switch feed {
|
||||||
|
case "AzPulse":
|
||||||
|
responseData["valid_users"] = generatorAz.Users.GetValidUsers()
|
||||||
|
responseData["invalid_users"] = generatorAz.Users.GetInvalidUsers()
|
||||||
|
}
|
||||||
|
|
||||||
|
response.JSON(w, 200, responseData)
|
||||||
|
}
|
||||||
|
@@ -5,7 +5,6 @@ package middleware
|
|||||||
import (
|
import (
|
||||||
"crypto"
|
"crypto"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
|
|
||||||
atcrypto "github.com/bluesky-social/indigo/atproto/crypto"
|
atcrypto "github.com/bluesky-social/indigo/atproto/crypto"
|
||||||
"github.com/golang-jwt/jwt/v5"
|
"github.com/golang-jwt/jwt/v5"
|
||||||
@@ -44,7 +43,6 @@ func init() {
|
|||||||
jwt.RegisterSigningMethod(SigningMethodES256.Alg(), func() jwt.SigningMethod {
|
jwt.RegisterSigningMethod(SigningMethodES256.Alg(), func() jwt.SigningMethod {
|
||||||
return SigningMethodES256
|
return SigningMethodES256
|
||||||
})
|
})
|
||||||
fmt.Println("init Completed")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Errors returned on different problems.
|
// Errors returned on different problems.
|
||||||
|
@@ -39,7 +39,7 @@ func (f *FeedAz) Describe(_ context.Context) bsky.FeedDescribeFeedGenerator_Feed
|
|||||||
|
|
||||||
func (f *FeedAz) GetPage(
|
func (f *FeedAz) GetPage(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
_ string,
|
_ string, // user did
|
||||||
limit int64,
|
limit int64,
|
||||||
cursor string,
|
cursor string,
|
||||||
) ([]*bsky.FeedDefs_SkeletonFeedPost, *string, error) {
|
) ([]*bsky.FeedDefs_SkeletonFeedPost, *string, error) {
|
||||||
|
@@ -114,7 +114,7 @@ func (generator *Generator) IsValid(post *collections.Post) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
if isValidUser := users.IsValid(post.DID); isValidUser != nil {
|
if isValidUser := Users.IsValid(post.DID); isValidUser != nil {
|
||||||
return *isValidUser
|
return *isValidUser
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@@ -2,7 +2,7 @@ package az
|
|||||||
|
|
||||||
import "github.com/aykhans/bsky-feedgen/pkg/generator"
|
import "github.com/aykhans/bsky-feedgen/pkg/generator"
|
||||||
|
|
||||||
var users = generator.Users{
|
var Users = generator.Users{
|
||||||
// Invalid
|
// Invalid
|
||||||
"did:plc:5zww7zorx2ajw7hqrhuix3ba": false,
|
"did:plc:5zww7zorx2ajw7hqrhuix3ba": false,
|
||||||
"did:plc:c4vhz47h566t2ntgd7gtawen": false,
|
"did:plc:c4vhz47h566t2ntgd7gtawen": false,
|
||||||
@@ -14,6 +14,16 @@ var users = generator.Users{
|
|||||||
"did:plc:5vwjnzaibnwscbbcvkzhy57v": false,
|
"did:plc:5vwjnzaibnwscbbcvkzhy57v": false,
|
||||||
"did:plc:6mfp3coadoobuvlg6w2avw6x": false,
|
"did:plc:6mfp3coadoobuvlg6w2avw6x": false,
|
||||||
"did:plc:lm2uhaoqoe6yo76oeihndfyi": false,
|
"did:plc:lm2uhaoqoe6yo76oeihndfyi": false,
|
||||||
|
"did:plc:vizwdor43adw3277u2kkrssd": false,
|
||||||
|
"did:plc:oqatvbgbhvqbjl2w2o63ehgi": false,
|
||||||
|
"did:plc:gy7yilnydusx5hy2z3dltynp": false,
|
||||||
|
"did:plc:xk7cs24wk6njv42azm2yd7dv": false,
|
||||||
|
"did:plc:ijmt7f4p3dcfqtg3j3zshimn": false,
|
||||||
|
"did:plc:2q5dx6whenn7pnsrfn3jpd6h": false,
|
||||||
|
"did:plc:s2waw3gkmn7h2nn6od44apng": false,
|
||||||
|
"did:plc:4hm6gb7dzobynqrpypif3dck": false,
|
||||||
|
"did:plc:odvarii7w7soygxet3xvzop7": false,
|
||||||
|
"did:plc:5cbkdchsxjvz5fog2oo7m4le": false,
|
||||||
|
|
||||||
// Valid
|
// Valid
|
||||||
"did:plc:jbt4qi6psd7rutwzedtecsq7": true,
|
"did:plc:jbt4qi6psd7rutwzedtecsq7": true,
|
||||||
@@ -22,6 +32,9 @@ var users = generator.Users{
|
|||||||
"did:plc:phtq2rhgbwipyx5ie3apw44j": true,
|
"did:plc:phtq2rhgbwipyx5ie3apw44j": true,
|
||||||
"did:plc:jfdvklrs5n5qv7f25v6swc5h": true,
|
"did:plc:jfdvklrs5n5qv7f25v6swc5h": true,
|
||||||
"did:plc:u5ez5w6qslh6advti4wyddba": true,
|
"did:plc:u5ez5w6qslh6advti4wyddba": true,
|
||||||
"did:plc:cs2cbzojm6hmx5lfxiuft3mq": true,
|
|
||||||
"did:plc:x7alwnnjygt2aqcwblhazko7": true,
|
"did:plc:x7alwnnjygt2aqcwblhazko7": true,
|
||||||
|
"did:plc:mgciyhgfn65z7iazxuar6o6a": true,
|
||||||
|
"did:plc:ay2f5go4lxq2hspiaqohegac": true,
|
||||||
|
"did:plc:ftoopigdpuzqt2kpeyqxsofx": true,
|
||||||
|
"did:plc:cs2cbzojm6hmx5lfxiuft3mq": true,
|
||||||
}
|
}
|
||||||
|
@@ -4,6 +4,15 @@ import "github.com/aykhans/bsky-feedgen/pkg/utils"
|
|||||||
|
|
||||||
type Users map[string]bool
|
type Users map[string]bool
|
||||||
|
|
||||||
|
// IsValid checks if a given DID exists in the Users map and returns its validity status.
|
||||||
|
//
|
||||||
|
// Parameters:
|
||||||
|
//
|
||||||
|
// did: The Decentralized Identifier string to check
|
||||||
|
//
|
||||||
|
// Returns:
|
||||||
|
// - *bool: A pointer to the validity status if the DID exists in the map
|
||||||
|
// - nil: If the DID does not exist in the map
|
||||||
func (u Users) IsValid(did string) *bool {
|
func (u Users) IsValid(did string) *bool {
|
||||||
isValid, ok := u[did]
|
isValid, ok := u[did]
|
||||||
if ok == false {
|
if ok == false {
|
||||||
@@ -12,3 +21,49 @@ func (u Users) IsValid(did string) *bool {
|
|||||||
|
|
||||||
return utils.ToPtr(isValid)
|
return utils.ToPtr(isValid)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetValidUsers returns a slice of DIDs that are marked as valid in the Users map.
|
||||||
|
//
|
||||||
|
// Returns:
|
||||||
|
// - []string: A slice of valid DIDs, limited by the specified parameters
|
||||||
|
func (u Users) GetValidUsers() []string {
|
||||||
|
validUsers := make([]string, 0)
|
||||||
|
|
||||||
|
for did, isValid := range u {
|
||||||
|
if isValid {
|
||||||
|
validUsers = append(validUsers, did)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return validUsers
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetInvalidUsers returns a slice of DIDs that are marked as invalid in the Users map.
|
||||||
|
//
|
||||||
|
// Returns:
|
||||||
|
// - []string: A slice of invalid DIDs, limited by the specified parameters
|
||||||
|
func (u Users) GetInvalidUsers() []string {
|
||||||
|
invalidUsers := make([]string, 0)
|
||||||
|
|
||||||
|
for did, isValid := range u {
|
||||||
|
if !isValid {
|
||||||
|
invalidUsers = append(invalidUsers, did)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return invalidUsers
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAll returns a slice of all DIDs in the Users map, regardless of validity status.
|
||||||
|
//
|
||||||
|
// Returns:
|
||||||
|
// - []string: A slice containing all DIDs in the map
|
||||||
|
func (u Users) GetAll() []string {
|
||||||
|
allUsers := make([]string, 0, len(u))
|
||||||
|
|
||||||
|
for did := range u {
|
||||||
|
allUsers = append(allUsers, did)
|
||||||
|
}
|
||||||
|
|
||||||
|
return allUsers
|
||||||
|
}
|
||||||
|
Reference in New Issue
Block a user