mirror of
https://github.com/aykhans/movier.git
synced 2025-07-03 02:15:23 +00:00
Rewritten in go and python
This commit is contained in:
82
server/cmd/download.go
Normal file
82
server/cmd/download.go
Normal file
@ -0,0 +1,82 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/aykhans/movier/server/pkg/config"
|
||||
"github.com/aykhans/movier/server/pkg/dto"
|
||||
"github.com/aykhans/movier/server/pkg/utils"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func getDownloadCmd() *cobra.Command {
|
||||
downloadCmd := &cobra.Command{
|
||||
Use: "download",
|
||||
Short: "Movie Data Downloader",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
err := runDownload()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
return downloadCmd
|
||||
}
|
||||
|
||||
func runDownload() error {
|
||||
downloadPath := config.GetDownloadPath()
|
||||
extractPath := config.GetExtractPath()
|
||||
err := utils.MakeDirIfNotExist(downloadPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = utils.MakeDirIfNotExist(extractPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
download(downloadPath, extractPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func download(
|
||||
downloadPath string,
|
||||
extractPath string,
|
||||
) error {
|
||||
for _, downloadConfig := range config.DownloadConfigs {
|
||||
extracted, err := utils.IsDirExist(extractPath + "/" + downloadConfig.ExtractName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if extracted {
|
||||
log.Printf("File %s already extracted. Skipping...\n\n", downloadConfig.ExtractName)
|
||||
continue
|
||||
}
|
||||
|
||||
downloaded, err := utils.IsDirExist(downloadPath + "/" + downloadConfig.DownloadName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if downloaded {
|
||||
log.Printf("File %s already downloaded. Extracting...\n\n", downloadConfig.DownloadName)
|
||||
if err := dto.ExtractGzFile(
|
||||
downloadPath+"/"+downloadConfig.DownloadName,
|
||||
extractPath+"/"+downloadConfig.ExtractName,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("Downloading and extracting %s file...\n\n", downloadConfig.DownloadName)
|
||||
if err := dto.DownloadAndExtractGz(
|
||||
downloadConfig.URL,
|
||||
downloadPath+"/"+downloadConfig.DownloadName,
|
||||
extractPath+"/"+downloadConfig.ExtractName,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
102
server/cmd/filter.go
Normal file
102
server/cmd/filter.go
Normal file
@ -0,0 +1,102 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/aykhans/movier/server/pkg/config"
|
||||
"github.com/aykhans/movier/server/pkg/dto"
|
||||
|
||||
"github.com/aykhans/movier/server/pkg/storage/postgresql"
|
||||
"github.com/aykhans/movier/server/pkg/storage/postgresql/repository"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func getFilterCmd() *cobra.Command {
|
||||
filterCmd := &cobra.Command{
|
||||
Use: "filter",
|
||||
Short: "Movie Data Filter",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
err := runFilter()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
return filterCmd
|
||||
}
|
||||
|
||||
func runFilter() error {
|
||||
generalStartTime := time.Now()
|
||||
extractedPath := config.GetExtractPath()
|
||||
|
||||
log.Printf("Filtering basics data...\n\n")
|
||||
startTime := time.Now()
|
||||
basics, err := dto.FilterBasics(extractedPath + "/title.basics.tsv")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Basics data filtered. Found %d records (%s)\n\n", len(basics), time.Since(startTime))
|
||||
|
||||
log.Printf("Inserting basics data...\n\n")
|
||||
postgresURL, err := config.NewPostgresURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
db, err := postgresql.NewDB(postgresURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
imdbRepo := repository.NewIMDbRepository(db)
|
||||
startTime = time.Now()
|
||||
err = imdbRepo.InsertMultipleBasics(basics)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Basics data inserted. (%s)\n\n", time.Since(startTime))
|
||||
|
||||
log.Printf("Filtering principals data...\n\n")
|
||||
tconsts, err := imdbRepo.GetAllTconsts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(tconsts) == 0 {
|
||||
return fmt.Errorf("no tconsts found")
|
||||
}
|
||||
startTime = time.Now()
|
||||
principals, err := dto.FilterPrincipals(extractedPath+"/title.principals.tsv", tconsts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Principals data filtered. (%s)\n\n", time.Since(startTime))
|
||||
|
||||
log.Printf("Inserting principals data...\n\n")
|
||||
startTime = time.Now()
|
||||
err = imdbRepo.UpdateMultiplePrincipals(principals)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Principals data inserted. (%s)\n\n", time.Since(startTime))
|
||||
|
||||
log.Printf("Filtering ratings data...\n\n")
|
||||
startTime = time.Now()
|
||||
ratings, err := dto.FilterRatings(extractedPath+"/title.ratings.tsv", tconsts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Ratings data filtered. (%s)\n\n", time.Since(startTime))
|
||||
|
||||
log.Printf("Inserting ratings data...\n\n")
|
||||
startTime = time.Now()
|
||||
err = imdbRepo.UpdateMultipleRatings(ratings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Ratings data inserted. (%s)\n\n", time.Since(startTime))
|
||||
|
||||
log.Printf("Filtering done! (%s)\n", time.Since(generalStartTime))
|
||||
return nil
|
||||
}
|
||||
20
server/cmd/root.go
Normal file
20
server/cmd/root.go
Normal file
@ -0,0 +1,20 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "movier",
|
||||
Short: "Movie Recommendation System",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
func Execute() error {
|
||||
rootCmd.AddCommand(getDownloadCmd())
|
||||
rootCmd.AddCommand(getFilterCmd())
|
||||
rootCmd.AddCommand(getServeCmd())
|
||||
return rootCmd.Execute()
|
||||
}
|
||||
75
server/cmd/serve.go
Normal file
75
server/cmd/serve.go
Normal file
@ -0,0 +1,75 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/aykhans/movier/server/pkg/config"
|
||||
"github.com/aykhans/movier/server/pkg/handlers"
|
||||
"github.com/aykhans/movier/server/pkg/storage/postgresql"
|
||||
"github.com/aykhans/movier/server/pkg/storage/postgresql/repository"
|
||||
"github.com/spf13/cobra"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
func getServeCmd() *cobra.Command {
|
||||
serveCmd := &cobra.Command{
|
||||
Use: "serve",
|
||||
Short: "Movie Recommendation Serve",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
err := runServe()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
fmt.Println("Movie Recommendation Serve")
|
||||
},
|
||||
}
|
||||
return serveCmd
|
||||
}
|
||||
|
||||
func runServe() error {
|
||||
dbURL, err := config.NewPostgresURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db, err := postgresql.NewDB(dbURL)
|
||||
defer db.Close(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
imdbRepo := repository.NewIMDbRepository(db)
|
||||
|
||||
grpcRecommenderServiceTarget, err := config.NewRecommenderServiceGrpcTarget()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, err := grpc.NewClient(
|
||||
grpcRecommenderServiceTarget,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("did not connect to grpc recommender service: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
router := http.NewServeMux()
|
||||
imdbHandler := handlers.NewIMDbHandler(*imdbRepo, conn, config.GetBaseURL())
|
||||
|
||||
router.HandleFunc("GET /ping", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("pong"))
|
||||
})
|
||||
router.HandleFunc("GET /", imdbHandler.HandlerHome)
|
||||
router.HandleFunc("GET /recs", imdbHandler.HandlerGetRecommendations)
|
||||
|
||||
log.Printf("serving on port %d", config.ServePort)
|
||||
err = http.ListenAndServe(fmt.Sprintf(":%d", config.ServePort), handlers.CORSMiddleware(router))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user