From 89237193f44cabd1357cdeb4ef2d3c9d8f6fbaaa Mon Sep 17 00:00:00 2001 From: Aykhan Shahsuvarov Date: Sun, 10 Mar 2024 22:33:35 +0400 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20Cassandra=20support=20and=20u?= =?UTF-8?q?pdate=20README.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 ++ app/config/cassandra.env.example | 9 ++ app/config/settings.go | 31 ++++++ app/db/base.go | 29 +++++- app/db/cassandra.go | 98 ++++++++++++++++++ config/cassandra/Dockerfile | 6 ++ config/cassandra/Dockerfile.init.cassandra | 6 ++ config/cassandra/init-cassandra.env.example | 5 + config/cassandra/init-cassandra.sh | 19 ++++ docker-compose-cassandra.yml | 105 ++++++++++++++++++++ go.mod | 4 + go.sum | 13 +++ 12 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 config/cassandra/Dockerfile create mode 100644 config/cassandra/Dockerfile.init.cassandra create mode 100644 config/cassandra/init-cassandra.env.example create mode 100644 config/cassandra/init-cassandra.sh create mode 100644 docker-compose-cassandra.yml diff --git a/README.md b/README.md index b8f90ac..d8c0d2b 100644 --- a/README.md +++ b/README.md @@ -7,4 +7,15 @@ - Run the following command to start the application with PostgreSQL: ```bash docker compose -f docker-compose-postgres.yml up + ``` + +## Run with Cassandra + +- Rename and fill environment files: + - `/app/config/cassandra.env.example` to `/app/config/cassandra.env` + - `/config/cassandra/init-cassandra.env.example` to `/config/cassandra/init-cassandra.env` + +- Run the following command to start the application with PostgreSQL: + ```bash + docker compose -f docker-compose-cassandra.yml up ``` \ No newline at end of file diff --git a/app/config/cassandra.env.example b/app/config/cassandra.env.example index e69de29..8e92ce2 100644 --- a/app/config/cassandra.env.example +++ b/app/config/cassandra.env.example @@ -0,0 +1,9 @@ +CASSANDRA_USER=user +CASSANDRA_PASSWORD=password +CASSANDRA_KEYSPACE=ohmyurl +CASSANDRA_DB=url +CASSANDRA_CLUSTERS=ohmyurl-cassandra-1:9042,ohmyurl-cassandra-2:9042,ohmyurl-cassandra-3:9042 +LISTEN_PORT_CREATE=8080 +LISTEN_PORT_FORWARD=8081 +FORWARD_DOMAIN=http://localhost/ +CREATE_DOMAIN=http://127.0.0.1 \ No newline at end of file diff --git a/app/config/settings.go b/app/config/settings.go index 0f36b5e..22d93a2 100644 --- a/app/config/settings.go +++ b/app/config/settings.go @@ -2,6 +2,7 @@ package config import ( "os" + "strconv" "strings" ) @@ -28,6 +29,16 @@ type PostgresConfig struct { DBNAME string } +type CassandraConfig struct { + USER string + PASSWORD string + KEYSPACE string + CLUSTERS []string + APP_LABEL string + URL_START_ID int + URL_END_ID int +} + func GetAppConfig() *AppConfig { return &AppConfig{ LISTEN_PORT_CREATE: GetEnvOrPanic("LISTEN_PORT_CREATE"), @@ -55,6 +66,18 @@ func GetPostgresConfig() *PostgresConfig { } } +func GetCassandraConfig() *CassandraConfig { + return &CassandraConfig{ + USER: GetEnvOrPanic("CASSANDRA_USER"), + PASSWORD: GetEnvOrPanic("CASSANDRA_PASSWORD"), + CLUSTERS: strings.Split(GetEnvOrPanic("CASSANDRA_CLUSTERS"), ","), + KEYSPACE: GetEnvOrPanic("CASSANDRA_KEYSPACE"), + APP_LABEL: GetEnvOrPanic("CASSANDRA_APP_LABEL"), + URL_START_ID: Str2IntOrPanic(GetEnvOrPanic("CASSANDRA_URL_START_ID")), + URL_END_ID: Str2IntOrPanic(GetEnvOrPanic("CASSANDRA_URL_END_ID")), + } +} + func GetDB() DBName { dbName := strings.ToLower(GetEnvOrPanic("DB")) switch dbName { @@ -84,3 +107,11 @@ func GetEnvOrPanic(key string) string { } return value } + +func Str2IntOrPanic(value string) int { + i, err := strconv.Atoi(value) + if err != nil { + panic(err) + } + return i +} diff --git a/app/db/base.go b/app/db/base.go index 4482bc0..665360a 100644 --- a/app/db/base.go +++ b/app/db/base.go @@ -3,6 +3,7 @@ package db import ( "fmt" "github.com/aykhans/oh-my-url/app/config" + "github.com/gocql/gocql" gormPostgres "gorm.io/driver/postgres" "gorm.io/gorm" "time" @@ -43,6 +44,32 @@ func GetDB() DB { panic(err) } return &Postgres{gormDB: db} + + case "cassandra": + cassandraConf := config.GetCassandraConfig() + cluster := gocql.NewCluster(cassandraConf.CLUSTERS...) + cluster.Keyspace = cassandraConf.KEYSPACE + cluster.Authenticator = gocql.PasswordAuthenticator{ + Username: cassandraConf.USER, + Password: cassandraConf.PASSWORD, + } + + var db *gocql.Session + var err error + for i := 0; i < 60; i++ { + db, err = cluster.CreateSession() + if err == nil { + break + } + time.Sleep(3 * time.Second) + } + if err != nil { + panic(err) + } + return &Cassandra{ + db: db, + currentID: &CurrentID{}, + } } - return nil + panic("unknown db") } diff --git a/app/db/cassandra.go b/app/db/cassandra.go index 3a49c63..8fbcbc4 100644 --- a/app/db/cassandra.go +++ b/app/db/cassandra.go @@ -1 +1,99 @@ package db + +import ( + "errors" + "log" + "sync" + + "github.com/aykhans/oh-my-url/app/utils" + "github.com/gocql/gocql" +) + +type CurrentID struct { + ID int + mu sync.RWMutex +} + +type Cassandra struct { + db *gocql.Session + currentID *CurrentID +} + +func (c *Cassandra) Init() { + err := c.db.Query(` + CREATE TABLE IF NOT EXISTS url ( + id int, + key text, + url text, + count int, + PRIMARY KEY ((key), count) + ) WITH CLUSTERING ORDER BY (count DESC) + AND compaction = {'class': 'LeveledCompactionStrategy'}; + `).Consistency(gocql.All).Exec() + if err != nil { + panic(err) + } + + var id int + err = c.db. + Query(`SELECT MAX(id) FROM url LIMIT 1`). + Consistency(gocql.One). + Scan(&id) + + if err != nil { + panic(err) + } + c.currentID.ID = id +} + +func (c *Cassandra) CreateURL(url string) (string, error) { + c.currentID.mu.Lock() + defer c.currentID.mu.Unlock() + + id := c.currentID.ID + 1 + key := utils.GenerateKey(id) + m := make(map[string]interface{}) + + query := `INSERT INTO url (id, key, url, count) VALUES (?, ?, ?, ?) IF NOT EXISTS` + applied, err := c.db.Query(query, id, key, url, 0).Consistency(gocql.All).MapScanCAS(m) + if err != nil { + log.Println(err) + return "", err + } + if !applied { + log.Println("Failed to insert unique key") + return "", errors.New("an error occurred, please try again later") + } + c.currentID.ID = id + + return key, nil +} + +func (c *Cassandra) GetURL(key string) (string, error) { + var url string + err := c.db. + Query(`SELECT url FROM url WHERE key = ? LIMIT 1`, key). + Consistency(gocql.One). + Scan(&url) + if err != nil { + return "", err + } + return url, nil +} + +// just in case +// const urlKeyCreateFunction = ` +// CREATE FUNCTION IF NOT EXISTS oh_my_url.generate_url_key(n int) RETURNS NULL ON NULL INPUT RETURNS text LANGUAGE java AS ' +// String keyCharacters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +// StringBuilder result = new StringBuilder(); +// int base = keyCharacters.length(); + +// while (n > 0) { +// n--; +// result.append(keyCharacters.charAt(n % base)); +// n = (int) Math.floor(n / base); +// } + +// return result.reverse().toString(); +// '; +// ` diff --git a/config/cassandra/Dockerfile b/config/cassandra/Dockerfile new file mode 100644 index 0000000..8190e19 --- /dev/null +++ b/config/cassandra/Dockerfile @@ -0,0 +1,6 @@ +FROM cassandra:4 + +RUN sed -i -r 's/authenticator: AllowAllAuthenticator/authenticator: PasswordAuthenticator/' /etc/cassandra/cassandra.yaml +# RUN sed -i -r 's/user_defined_functions_enabled: false/user_defined_functions_enabled: true/' /etc/cassandra/cassandra.yaml + +CMD ["cassandra", "-f"] diff --git a/config/cassandra/Dockerfile.init.cassandra b/config/cassandra/Dockerfile.init.cassandra new file mode 100644 index 0000000..406c1c1 --- /dev/null +++ b/config/cassandra/Dockerfile.init.cassandra @@ -0,0 +1,6 @@ +FROM cassandra:4 + +COPY init-cassandra.sh /init-cassandra.sh +RUN chmod +x init-cassandra.sh + +CMD ["/init-cassandra.sh"] \ No newline at end of file diff --git a/config/cassandra/init-cassandra.env.example b/config/cassandra/init-cassandra.env.example new file mode 100644 index 0000000..7bd54b4 --- /dev/null +++ b/config/cassandra/init-cassandra.env.example @@ -0,0 +1,5 @@ +CASSANDRA_USERNAME=username +CASSANDRA_PASSWORD=password +CASSANDRA_KEYSPACE=keyspace +MAX_HEAP_SIZE=2G +HEAP_NEWSIZE=500M \ No newline at end of file diff --git a/config/cassandra/init-cassandra.sh b/config/cassandra/init-cassandra.sh new file mode 100644 index 0000000..5bdd634 --- /dev/null +++ b/config/cassandra/init-cassandra.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ -z "$CASSANDRA_USERNAME" ] || [ -z "$CASSANDRA_PASSWORD" ]; then + echo "Error: Username or password environment variables are not set." + exit 1 +fi + +until cqlsh -u cassandra -p cassandra -e "DESCRIBE KEYSPACES" ohmyurl-cassandra-1 || cqlsh -u $CASSANDRA_USERNAME -p $CASSANDRA_PASSWORD -e "DESCRIBE KEYSPACES" ohmyurl-cassandra-1; do + echo >&2 "Cassandra is unavailable - sleeping" + sleep 1 +done + +echo >&2 "Cassandra is up - executing command" + +cqlsh -u cassandra -p cassandra -e "CREATE ROLE IF NOT EXISTS $CASSANDRA_USERNAME WITH PASSWORD = '$CASSANDRA_PASSWORD' AND SUPERUSER = true AND LOGIN = true;" ohmyurl-cassandra-1 +cqlsh -u $CASSANDRA_USERNAME -p $CASSANDRA_PASSWORD -e "ALTER ROLE cassandra WITH PASSWORD='abcdef' AND SUPERUSER=false AND LOGIN = false;" ohmyurl-cassandra-1 +cqlsh -u $CASSANDRA_USERNAME -p $CASSANDRA_PASSWORD -e "CREATE KEYSPACE IF NOT EXISTS $CASSANDRA_KEYSPACE WITH REPLICATION = {'class': 'NetworkTopologyStrategy', 'datacenter1': 3};" ohmyurl-cassandra-1 + +exit 0 diff --git a/docker-compose-cassandra.yml b/docker-compose-cassandra.yml new file mode 100644 index 0000000..7751727 --- /dev/null +++ b/docker-compose-cassandra.yml @@ -0,0 +1,105 @@ +version: "3.9" + +services: + ohmyurl-cassandra-1: + container_name: "ohmyurl-cassandra-1" + hostname: "ohmyurl-cassandra-1" + build: ./config/cassandra + image: ohmyurl-cassandra:4 + environment: &cassandra-environment + - MAX_HEAP_SIZE=4G + - HEAP_NEWSIZE=800M + - CASSANDRA_SEEDS=ohmyurl-cassandra-1,ohmyurl-cassandra-2,ohmyurl-cassandra-3 + ports: + - "9042:9042" + volumes: + - cassandra_1_data:/var/lib/cassandra + networks: + - ohmyurl-net + init: true + + ohmyurl-cassandra-2: + container_name: "ohmyurl-cassandra-2" + hostname: "ohmyurl-cassandra-2" + image: ohmyurl-cassandra:4 + environment: *cassandra-environment + depends_on: + - ohmyurl-cassandra-1 + ports: + - "9043:9042" + volumes: + - cassandra_2_data:/var/lib/cassandra + networks: + - ohmyurl-net + init: true + + ohmyurl-cassandra-3: + container_name: "ohmyurl-cassandra-3" + hostname: "ohmyurl-cassandra-3" + image: ohmyurl-cassandra:4 + environment: *cassandra-environment + depends_on: + - ohmyurl-cassandra-1 + ports: + - "9044:9042" + volumes: + - cassandra_3_data:/var/lib/cassandra + networks: + - ohmyurl-net + init: true + + ohmyurl-init-cassandra: + container_name: "ohmyurl-init-cassandra" + build: + context: ./config/cassandra + dockerfile: Dockerfile.init.cassandra + image: "ohmyurl-init-cassandra" + env_file: + - ./config/cassandra/init-cassandra.env + depends_on: + - ohmyurl-cassandra-1 + networks: + - ohmyurl-net + + ohmyurl-web: + container_name: "ohmyurl-web" + hostname: "ohmyurl-web" + build: . + image: ohmyurl-web:1.1 + environment: + - DB=cassandra + - CASSANDRA_APP_LABEL=ohmyurl-1 + - CASSANDRA_URL_START_ID=1 + - CASSANDRA_URL_END_ID=10000 + env_file: + - ./app/config/cassandra.env + ports: + - "8080:8080" + - "8081:8081" + depends_on: + - ohmyurl-cassandra-1 + networks: + - ohmyurl-net + init: true + + ohmyurl-nginx: + container_name: "ohmyurl-nginx" + image: nginx:1.25.3-alpine + ports: + - 80:80 + volumes: + - ./config/nginx/nginx.conf:/etc/nginx/conf.d/default.conf + depends_on: + - ohmyurl-web + networks: + - ohmyurl-net + init: true + +networks: + ohmyurl-net: + driver: bridge + +volumes: + cassandra_1_data: + cassandra_2_data: + cassandra_3_data: \ No newline at end of file diff --git a/go.mod b/go.mod index 7fa80e9..d38db21 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,9 @@ require ( require ( github.com/buraksezer/consistent v0.10.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect + github.com/gocql/gocql v1.6.0 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/pgx/v5 v5.5.2 // indirect @@ -19,4 +22,5 @@ require ( golang.org/x/crypto v0.18.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/text v0.14.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index eed01a1..5a985a6 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU= github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -6,6 +8,12 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= +github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= @@ -22,6 +30,9 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -41,6 +52,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=