Merge pull request #25 from aykhans/feat/add-random-value-selection

Add random value selection
This commit is contained in:
Aykhan Shahsuvarov 2024-09-18 20:30:10 +04:00 committed by GitHub
commit 30cc86dc63
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 365 additions and 200 deletions

View File

@ -1,4 +1,4 @@
FROM golang:1.22.5-alpine AS builder FROM golang:1.22.6-alpine AS builder
WORKDIR /dodo WORKDIR /dodo

View File

@ -99,6 +99,8 @@ docker run --rm -v ./path/config.json:/dodo/config.json -i aykhans/dodo -u https
``` ```
## CLI and JSON Config Parameters ## CLI and JSON Config Parameters
If the Headers, Params, Cookies and Body fields have multiple values, each request will choose a random value from the list.
| Parameter | JSON config file | CLI Flag | CLI Short Flag | Type | Description | Default | | Parameter | JSON config file | CLI Flag | CLI Short Flag | Type | Description | Default |
| ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- |
| Config file | - | --config-file | -c | String | Path to the JSON config file | - | | Config file | - | --config-file | -c | String | Path to the JSON config file | - |
@ -108,11 +110,8 @@ docker run --rm -v ./path/config.json:/dodo/config.json -i aykhans/dodo -u https
| Request count | request_count | --request-count | -r | Integer | Total number of requests to send | 1000 | | Request count | request_count | --request-count | -r | Integer | Total number of requests to send | 1000 |
| Dodos count (Threads) | dodos_count | --dodos-count | -d | Integer | Number of dodos (threads) to send requests in parallel | 1 | | Dodos count (Threads) | dodos_count | --dodos-count | -d | Integer | Number of dodos (threads) to send requests in parallel | 1 |
| Timeout | timeout | --timeout | -t | Integer | Timeout for canceling each request (milliseconds) | 10000 | | Timeout | timeout | --timeout | -t | Integer | Timeout for canceling each request (milliseconds) | 10000 |
| Params | params | - | - | Key-Value {string: string} | Request parameters | - | | Params | params | - | - | Key-Value {String: [String]} | Request parameters | - |
| Headers | headers | - | - | Key-Value {string: string} | Request headers | - | | Headers | headers | - | - | Key-Value {String: [String]} | Request headers | - |
| Cookies | cookies | - | - | Key-Value {string: string} | Request cookies | - | | Cookies | cookies | - | - | Key-Value {String: [String]} | Request cookies | - |
| Body | body | - | - | String | Request body | - | | Body | body | - | - | [String] | Request body | - |
| Proxy | proxies | - | - | List[Key-Value {string: string}] | List of proxies (will check active proxies before sending requests) | - | | Proxy | proxies | - | - | List[Key-Value {string: string}] | List of proxies (will check active proxies before sending requests) | - |
## Examples
![dodo_example](https://raw.githubusercontent.com/aykhans/dodo/main/assets/dodo-example.gif)

Binary file not shown.

Before

Width:  |  Height:  |  Size: 144 KiB

View File

@ -7,7 +7,7 @@
"params": {}, "params": {},
"headers": {}, "headers": {},
"cookies": {}, "cookies": {},
"body": "", "body": [""],
"proxies": [ "proxies": [
{ {
"url": "http://example.com:8080", "url": "http://example.com:8080",

View File

@ -6,11 +6,12 @@ import (
"os" "os"
"time" "time"
"github.com/aykhans/dodo/utils"
"github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/table"
) )
const ( const (
VERSION string = "0.4.2" VERSION string = "0.5.0"
DefaultUserAgent string = "Dodo/" + VERSION DefaultUserAgent string = "Dodo/" + VERSION
ProxyCheckURL string = "https://www.google.com" ProxyCheckURL string = "https://www.google.com"
DefaultMethod string = "GET" DefaultMethod string = "GET"
@ -30,11 +31,11 @@ type RequestConfig struct {
Timeout time.Duration Timeout time.Duration
DodosCount uint DodosCount uint
RequestCount uint RequestCount uint
Params map[string]string Params map[string][]string
Headers map[string]string Headers map[string][]string
Cookies map[string]string Cookies map[string][]string
Proxies []Proxy Proxies []Proxy
Body string Body []string
Yes bool Yes bool
} }
@ -55,17 +56,17 @@ func (config *RequestConfig) Print() {
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Dodos", config.DodosCount}) t.AppendRow(table.Row{"Dodos", config.DodosCount})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Request Count", config.RequestCount}) t.AppendRow(table.Row{"Request", config.RequestCount})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Params Count", len(config.Params)}) t.AppendRow(table.Row{"Params", utils.MarshalJSON(config.Params, 3)})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Headers Count", len(config.Headers)}) t.AppendRow(table.Row{"Headers", utils.MarshalJSON(config.Headers, 3)})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Cookies Count", len(config.Cookies)}) t.AppendRow(table.Row{"Cookies", utils.MarshalJSON(config.Cookies, 3)})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Proxies Count", len(config.Proxies)}) t.AppendRow(table.Row{"Proxies", utils.MarshalJSON(config.Proxies, 3)})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Body", config.Body}) t.AppendRow(table.Row{"Body", utils.MarshalJSON(config.Body, 3)})
t.Render() t.Render()
} }
@ -134,11 +135,11 @@ type Proxy struct {
type JSONConfig struct { type JSONConfig struct {
Config Config
Params map[string]string `json:"params"` Params map[string][]string `json:"params"`
Headers map[string]string `json:"headers"` Headers map[string][]string `json:"headers"`
Cookies map[string]string `json:"cookies"` Cookies map[string][]string `json:"cookies"`
Proxies []Proxy `json:"proxies" validate:"dive"` Proxies []Proxy `json:"proxies" validate:"dive"`
Body string `json:"body"` Body []string `json:"body"`
} }
func (config *JSONConfig) MergeConfigs(newConfig *JSONConfig) { func (config *JSONConfig) MergeConfigs(newConfig *JSONConfig) {
@ -152,7 +153,7 @@ func (config *JSONConfig) MergeConfigs(newConfig *JSONConfig) {
if len(newConfig.Cookies) != 0 { if len(newConfig.Cookies) != 0 {
config.Cookies = newConfig.Cookies config.Cookies = newConfig.Cookies
} }
if newConfig.Body != "" { if len(newConfig.Body) != 0 {
config.Body = newConfig.Body config.Body = newConfig.Body
} }
if len(newConfig.Proxies) != 0 { if len(newConfig.Proxies) != 0 {

2
go.mod
View File

@ -1,6 +1,6 @@
module github.com/aykhans/dodo module github.com/aykhans/dodo
go 1.22.3 go 1.22.6
require ( require (
github.com/go-playground/validator/v10 v10.22.1 github.com/go-playground/validator/v10 v10.22.1

View File

@ -106,6 +106,7 @@ func main() {
return return
} else if customerrors.Is(err, customerrors.ErrNoInternet) { } else if customerrors.Is(err, customerrors.ErrNoInternet) {
utils.PrintAndExit("No internet connection") utils.PrintAndExit("No internet connection")
return
} }
panic(err) panic(err)
} }

View File

@ -9,26 +9,17 @@ import (
"time" "time"
"github.com/aykhans/dodo/config" "github.com/aykhans/dodo/config"
customerrors "github.com/aykhans/dodo/custom_errors"
"github.com/aykhans/dodo/readers" "github.com/aykhans/dodo/readers"
"github.com/aykhans/dodo/utils" "github.com/aykhans/dodo/utils"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
"github.com/valyala/fasthttp/fasthttpproxy" "github.com/valyala/fasthttp/fasthttpproxy"
) )
// getClientDoFunc returns a ClientDoFunc function that can be used to make HTTP requests. type ClientGeneratorFunc func() *fasthttp.HostClient
//
// The function first checks if there are any proxies available. If there are, it retrieves the active proxy clients // getClients initializes and returns a slice of fasthttp.HostClient based on the provided parameters.
// using the getActiveProxyClients function. If the context is canceled during this process, it returns nil. // It can either return clients with proxies or a single client without proxies.
// It then checks the number of active proxy clients and prompts the user to continue if there are none. func getClients(
// If the user chooses to continue, it creates a fasthttp.HostClient with the appropriate settings and returns
// a ClientDoFunc function using the getSharedClientDoFunc function.
// If there is only one active proxy client, it uses that client to create the ClientDoFunc function.
// If there are multiple active proxy clients, it uses the getSharedRandomClientDoFunc function to create the ClientDoFunc function.
//
// If there are no proxies available, it creates a fasthttp.HostClient with the appropriate settings and returns
// a ClientDoFunc function using the getSharedClientDoFunc function.
func getClientDoFunc(
ctx context.Context, ctx context.Context,
timeout time.Duration, timeout time.Duration,
proxies []config.Proxy, proxies []config.Proxy,
@ -36,7 +27,7 @@ func getClientDoFunc(
maxConns uint, maxConns uint,
yes bool, yes bool,
URL *url.URL, URL *url.URL,
) ClientDoFunc { ) []*fasthttp.HostClient {
isTLS := URL.Scheme == "https" isTLS := URL.Scheme == "https"
if len(proxies) > 0 { if len(proxies) > 0 {
@ -71,26 +62,9 @@ func getClientDoFunc(
} }
} }
fmt.Println() fmt.Println()
if activeProxyClientsCount == 0 { if activeProxyClientsCount > 0 {
client := &fasthttp.HostClient{ return activeProxyClients
MaxConns: int(maxConns),
IsTLS: isTLS,
Addr: URL.Host,
MaxIdleConnDuration: timeout,
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
} }
return getSharedClientDoFunc(client, timeout)
} else if activeProxyClientsCount == 1 {
client := activeProxyClients[0]
return getSharedClientDoFunc(client, timeout)
}
return getSharedRandomClientDoFunc(
activeProxyClients,
activeProxyClientsCount,
timeout,
)
} }
client := &fasthttp.HostClient{ client := &fasthttp.HostClient{
@ -102,7 +76,7 @@ func getClientDoFunc(
WriteTimeout: timeout, WriteTimeout: timeout,
ReadTimeout: timeout, ReadTimeout: timeout,
} }
return getSharedClientDoFunc(client, timeout) return []*fasthttp.HostClient{client}
} }
// getActiveProxyClients divides the proxies into slices based on the number of dodos and // getActiveProxyClients divides the proxies into slices based on the number of dodos and
@ -303,75 +277,17 @@ func getDialFunc(proxy *config.Proxy, timeout time.Duration) (fasthttp.DialFunc,
return dialer, nil return dialer, nil
} }
// getSharedRandomClientDoFunc is equivalent to getSharedClientDoFunc but uses a random client from the provided slice. // getSharedClientFuncMultiple returns a ClientGeneratorFunc that cycles through a list of fasthttp.HostClient instances.
func getSharedRandomClientDoFunc( // The function uses a local random number generator to determine the starting index and stop index for cycling through the clients.
clients []*fasthttp.HostClient, // The returned function isn't thread-safe and should be used in a single-threaded context.
clientsCount uint, func getSharedClientFuncMultiple(clients []*fasthttp.HostClient, localRand *rand.Rand) ClientGeneratorFunc {
timeout time.Duration, return utils.RandomValueCycle(clients, localRand)
) ClientDoFunc {
clientsCountInt := int(clientsCount)
return func(ctx context.Context, request *fasthttp.Request) (*fasthttp.Response, error) {
client := clients[rand.Intn(clientsCountInt)]
defer client.CloseIdleConnections()
response := fasthttp.AcquireResponse()
ch := make(chan error)
go func() {
err := client.DoTimeout(request, response, timeout)
ch <- err
}()
select {
case err := <-ch:
if err != nil {
fasthttp.ReleaseResponse(response)
return nil, err
}
return response, nil
case <-time.After(timeout):
fasthttp.ReleaseResponse(response)
return nil, customerrors.ErrTimeout
case <-ctx.Done():
return nil, customerrors.ErrInterrupt
}
}
} }
// getSharedClientDoFunc is a function that returns a ClientDoFunc, which is a function type used for making HTTP requests using a shared client. // getSharedClientFuncSingle returns a ClientGeneratorFunc that always returns the provided fasthttp.HostClient instance.
// It takes a client of type *fasthttp.HostClient and a timeout of type time.Duration as input parameters. // This can be useful for sharing a single client instance across multiple requests.
// The returned ClientDoFunc function can be used to make an HTTP request with the given client and timeout. func getSharedClientFuncSingle(client *fasthttp.HostClient) ClientGeneratorFunc {
// It takes a context.Context and a *fasthttp.Request as input parameters and returns a *fasthttp.Response and an error. return func() *fasthttp.HostClient {
// The function internally creates a new response using fasthttp.AcquireResponse() and a channel to handle errors. return client
// It then spawns a goroutine to execute the client.DoTimeout() method with the given request, response, and timeout.
// The function uses a select statement to handle three cases:
// - If an error is received from the channel, it checks if the error is not nil. If it's not nil, it releases the response and returns nil and the error.
// Otherwise, it returns the response and nil.
// - If the timeout duration is reached, it releases the response and returns nil and a custom timeout error.
// - If the context is canceled, it returns nil and a custom interrupt error.
//
// The function ensures that idle connections are closed by calling client.CloseIdleConnections() using a defer statement.
func getSharedClientDoFunc(
client *fasthttp.HostClient,
timeout time.Duration,
) ClientDoFunc {
return func(ctx context.Context, request *fasthttp.Request) (*fasthttp.Response, error) {
defer client.CloseIdleConnections()
response := fasthttp.AcquireResponse()
ch := make(chan error)
go func() {
err := client.DoTimeout(request, response, timeout)
ch <- err
}()
select {
case err := <-ch:
if err != nil {
fasthttp.ReleaseResponse(response)
return nil, err
}
return response, nil
case <-time.After(timeout):
fasthttp.ReleaseResponse(response)
return nil, customerrors.ErrTimeout
case <-ctx.Done():
return nil, customerrors.ErrInterrupt
}
} }
} }

View File

@ -1,15 +1,135 @@
package requests package requests
import ( import (
"context"
"math/rand"
"net/url" "net/url"
"time"
"github.com/aykhans/dodo/config" "github.com/aykhans/dodo/config"
customerrors "github.com/aykhans/dodo/custom_errors"
"github.com/aykhans/dodo/utils"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
) )
// newRequest creates a new fasthttp.Request object with the provided parameters. type RequestGeneratorFunc func() *fasthttp.Request
// It sets the request URI, host header, headers, cookies, params, method, and body.
// Request represents an HTTP request to be sent using the fasthttp client.
// It isn't thread-safe and should be used by a single goroutine.
type Request struct {
getClient ClientGeneratorFunc
getRequest RequestGeneratorFunc
}
// Send sends the HTTP request using the fasthttp client with a specified timeout.
// It returns the HTTP response or an error if the request fails or times out.
func (r *Request) Send(ctx context.Context, timeout time.Duration) (*fasthttp.Response, error) {
client := r.getClient()
request := r.getRequest()
defer client.CloseIdleConnections()
defer fasthttp.ReleaseRequest(request)
response := fasthttp.AcquireResponse()
ch := make(chan error)
go func() {
err := client.DoTimeout(request, response, timeout)
ch <- err
}()
select {
case err := <-ch:
if err != nil {
fasthttp.ReleaseResponse(response)
return nil, err
}
return response, nil
case <-time.After(timeout):
fasthttp.ReleaseResponse(response)
return nil, customerrors.ErrTimeout
case <-ctx.Done():
return nil, customerrors.ErrInterrupt
}
}
// newRequest creates a new Request instance based on the provided configuration and clients.
// It initializes a random number generator using the current time and a unique identifier (uid).
// Depending on the number of clients provided, it sets up a function to select the appropriate client.
// It also sets up a function to generate the request based on the provided configuration.
func newRequest( func newRequest(
requestConfig config.RequestConfig,
clients []*fasthttp.HostClient,
uid int64,
) *Request {
localRand := rand.New(rand.NewSource(time.Now().UnixNano() + uid))
clientsCount := len(clients)
if clientsCount < 1 {
panic("no clients")
}
getClient := ClientGeneratorFunc(nil)
if clientsCount == 1 {
getClient = getSharedClientFuncSingle(clients[0])
} else {
getClient = getSharedClientFuncMultiple(clients, localRand)
}
getRequest := getRequestGeneratorFunc(
requestConfig.URL,
requestConfig.Headers,
requestConfig.Cookies,
requestConfig.Params,
requestConfig.Method,
requestConfig.Body,
localRand,
)
requests := &Request{
getClient: getClient,
getRequest: getRequest,
}
return requests
}
// getRequestGeneratorFunc returns a RequestGeneratorFunc which generates HTTP requests
// with the specified parameters.
// The function uses a local random number generator to select bodies, headers, cookies, and parameters
// if multiple options are provided.
func getRequestGeneratorFunc(
URL *url.URL,
Headers map[string][]string,
Cookies map[string][]string,
Params map[string][]string,
Method string,
Bodies []string,
localRand *rand.Rand,
) RequestGeneratorFunc {
bodiesLen := len(Bodies)
getBody := func() string { return "" }
if bodiesLen == 1 {
getBody = func() string { return Bodies[0] }
} else if bodiesLen > 1 {
getBody = utils.RandomValueCycle(Bodies, localRand)
}
getHeaders := getKeyValueSetFunc(Headers, localRand)
getCookies := getKeyValueSetFunc(Cookies, localRand)
getParams := getKeyValueSetFunc(Params, localRand)
return func() *fasthttp.Request {
return newFasthttpRequest(
URL,
getHeaders(),
getCookies(),
getParams(),
Method,
getBody(),
)
}
}
// newFasthttpRequest creates a new fasthttp.Request object with the provided parameters.
// It sets the request URI, host header, headers, cookies, params, method, and body.
func newFasthttpRequest(
URL *url.URL, URL *url.URL,
Headers map[string]string, Headers map[string]string,
Cookies map[string]string, Cookies map[string]string,
@ -70,3 +190,60 @@ func setRequestMethod(req *fasthttp.Request, method string) {
func setRequestBody(req *fasthttp.Request, body string) { func setRequestBody(req *fasthttp.Request, body string) {
req.SetBody([]byte(body)) req.SetBody([]byte(body))
} }
// getKeyValueSetFunc generates a function that returns a map of key-value pairs based on the provided key-value set.
// The generated function will either return fixed values or random values depending on the input.
//
// Returns:
// - A function that returns a map of key-value pairs. If the input map contains multiple values for a key,
// the returned function will generate random values for that key. If the input map contains a single value
// for a key, the returned function will always return that value. If the input map is empty for a key,
// the returned function will generate an empty string for that key.
func getKeyValueSetFunc[
KeyValueSet map[string][]string,
KeyValue map[string]string,
](keyValueSet KeyValueSet, localRand *rand.Rand) func() KeyValue {
getKeyValueSlice := []map[string]func() string{}
isRandom := false
for key, values := range keyValueSet {
valuesLen := len(values)
// if values is empty, return a function that generates empty string
// if values has only one element, return a function that generates that element
// if values has more than one element, return a function that generates a random element
getKeyValue := func() string { return "" }
if valuesLen == 1 {
getKeyValue = func() string { return values[0] }
} else if valuesLen > 1 {
getKeyValue = utils.RandomValueCycle(values, localRand)
isRandom = true
}
getKeyValueSlice = append(
getKeyValueSlice,
map[string]func() string{key: getKeyValue},
)
}
// if isRandom is true, return a function that generates random values,
// otherwise return a function that generates fixed values to avoid unnecessary random number generation
if isRandom {
return func() KeyValue {
keyValues := make(KeyValue, len(getKeyValueSlice))
for _, keyValue := range getKeyValueSlice {
for key, value := range keyValue {
keyValues[key] = value()
}
}
return keyValues
}
} else {
keyValues := make(KeyValue, len(getKeyValueSlice))
for _, keyValue := range getKeyValueSlice {
for key, value := range keyValue {
keyValues[key] = value()
}
}
return func() KeyValue { return keyValues }
}
}

View File

@ -1,14 +1,12 @@
package requests package requests
import ( import (
"context"
"fmt" "fmt"
"os" "os"
"time" "time"
"github.com/aykhans/dodo/utils" "github.com/aykhans/dodo/utils"
"github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/table"
"github.com/valyala/fasthttp"
) )
type Response struct { type Response struct {
@ -17,9 +15,7 @@ type Response struct {
Time time.Duration Time time.Duration
} }
type Responses []Response type Responses []*Response
type ClientDoFunc func(ctx context.Context, request *fasthttp.Request) (*fasthttp.Response, error)
// Print prints the responses in a tabular format, including information such as // Print prints the responses in a tabular format, including information such as
// response count, minimum time, maximum time, and average time. // response count, minimum time, maximum time, and average time.

View File

@ -11,10 +11,18 @@ import (
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
) )
// Run executes the HTTP requests based on the provided request configuration. // Run executes the main logic for processing requests based on the provided configuration.
// It checks for internet connection and returns an error if there is no connection. // It first checks for an internet connection with a timeout context. If no connection is found,
// If the context is canceled while checking proxies, it returns the ErrInterrupt. // it returns an error. Then, it initializes clients based on the request configuration and
// If the context is canceled while sending requests, it returns the response objects obtained so far. // releases the dodos. If the context is canceled and no responses are collected, it returns an interrupt error.
//
// Parameters:
// - ctx: The context for managing request lifecycle and cancellation.
// - requestConfig: The configuration for the request, including timeout, proxies, and other settings.
//
// Returns:
// - Responses: A collection of responses from the executed requests.
// - error: An error if the operation fails, such as no internet connection or an interrupt.
func Run(ctx context.Context, requestConfig *config.RequestConfig) (Responses, error) { func Run(ctx context.Context, requestConfig *config.RequestConfig) (Responses, error) {
checkConnectionCtx, checkConnectionCtxCancel := context.WithTimeout(ctx, 8*time.Second) checkConnectionCtx, checkConnectionCtxCancel := context.WithTimeout(ctx, 8*time.Second)
if !checkConnection(checkConnectionCtx) { if !checkConnection(checkConnectionCtx) {
@ -23,7 +31,7 @@ func Run(ctx context.Context, requestConfig *config.RequestConfig) (Responses, e
} }
checkConnectionCtxCancel() checkConnectionCtxCancel()
clientDoFunc := getClientDoFunc( clients := getClients(
ctx, ctx,
requestConfig.Timeout, requestConfig.Timeout,
requestConfig.Proxies, requestConfig.Proxies,
@ -32,26 +40,11 @@ func Run(ctx context.Context, requestConfig *config.RequestConfig) (Responses, e
requestConfig.Yes, requestConfig.Yes,
requestConfig.URL, requestConfig.URL,
) )
if clientDoFunc == nil { if clients == nil {
return nil, customerrors.ErrInterrupt return nil, customerrors.ErrInterrupt
} }
request := newRequest( responses := releaseDodos(ctx, requestConfig, clients)
requestConfig.URL,
requestConfig.Headers,
requestConfig.Cookies,
requestConfig.Params,
requestConfig.Method,
requestConfig.Body,
)
defer fasthttp.ReleaseRequest(request)
responses := releaseDodos(
ctx,
request,
clientDoFunc,
requestConfig.GetValidDodosCountForRequests(),
requestConfig.RequestCount,
)
if ctx.Err() != nil && len(responses) == 0 { if ctx.Err() != nil && len(responses) == 0 {
return nil, customerrors.ErrInterrupt return nil, customerrors.ErrInterrupt
} }
@ -59,30 +52,36 @@ func Run(ctx context.Context, requestConfig *config.RequestConfig) (Responses, e
return responses, nil return responses, nil
} }
// releaseDodos sends multiple HTTP requests concurrently using multiple "dodos" (goroutines). // releaseDodos sends requests concurrently using multiple dodos (goroutines) and returns the aggregated responses.
// It takes a mainRequest as the base request, timeout duration for each request, clientDoFunc for customizing the client behavior, //
// dodosCount as the number of goroutines to be used, and requestCount as the total number of requests to be sent. // The function performs the following steps:
// It returns the responses received from all the requests. // 1. Initializes wait groups and other necessary variables.
// 2. Starts a goroutine to stream progress updates.
// 3. Distributes the total request count among the dodos.
// 4. Starts a goroutine for each dodo to send requests concurrently.
// 5. Waits for all dodos to complete their requests.
// 6. Cancels the progress streaming context and waits for the progress goroutine to finish.
// 7. Flattens and returns the aggregated responses.
func releaseDodos( func releaseDodos(
ctx context.Context, ctx context.Context,
mainRequest *fasthttp.Request, requestConfig *config.RequestConfig,
clientDoFunc ClientDoFunc, clients []*fasthttp.HostClient,
dodosCount uint,
requestCount uint,
) Responses { ) Responses {
var ( var (
wg sync.WaitGroup wg sync.WaitGroup
streamWG sync.WaitGroup streamWG sync.WaitGroup
requestCountPerDodo uint requestCountPerDodo uint
dodosCountInt = int(dodosCount) dodosCount uint = requestConfig.GetValidDodosCountForRequests()
dodosCountInt int = int(dodosCount)
requestCount uint = uint(requestConfig.RequestCount)
responses = make([][]*Response, dodosCount)
increase = make(chan int64, requestCount)
) )
wg.Add(dodosCountInt) wg.Add(dodosCountInt)
streamWG.Add(1) streamWG.Add(1)
responses := make([][]Response, dodosCount)
increase := make(chan int64, requestCount)
streamCtx, streamCtxCancel := context.WithCancel(context.Background()) streamCtx, streamCtxCancel := context.WithCancel(context.Background())
go streamProgress(streamCtx, &streamWG, int64(requestCount), "Dodos Working🔥", increase) go streamProgress(streamCtx, &streamWG, int64(requestCount), "Dodos Working🔥", increase)
for i := range dodosCount { for i := range dodosCount {
@ -92,16 +91,14 @@ func releaseDodos(
requestCountPerDodo = ((i + 1) * requestCount / dodosCount) - requestCountPerDodo = ((i + 1) * requestCount / dodosCount) -
(i * requestCount / dodosCount) (i * requestCount / dodosCount)
} }
dodoSpecificRequest := &fasthttp.Request{}
mainRequest.CopyTo(dodoSpecificRequest)
go sendRequest( go sendRequest(
ctx, ctx,
dodoSpecificRequest, newRequest(*requestConfig, clients, int64(i)),
requestConfig.Timeout,
requestCountPerDodo,
&responses[i], &responses[i],
increase, increase,
requestCountPerDodo,
clientDoFunc,
&wg, &wg,
) )
} }
@ -111,29 +108,19 @@ func releaseDodos(
return utils.Flatten(responses) return utils.Flatten(responses)
} }
// sendRequest sends an HTTP request using the provided clientDo function and handles the response. // sendRequest sends a specified number of HTTP requests concurrently with a given timeout.
// // It appends the responses to the provided responseData slice and sends the count of completed requests
// Parameters: // to the increase channel. The function terminates early if the context is canceled or if a custom
// - ctx: The context to control cancellation and timeout. // interrupt error is encountered.
// - request: The HTTP request to be sent.
// - responseData: A slice to store the response data.
// - increase: A channel to signal the completion of a request.
// - requestCount: The number of requests to be sent.
// - clientDo: A function to execute the HTTP request.
// - wg: A wait group to signal the completion of the function.
//
// The function sends the specified number of requests, handles errors, and appends the response data
// to the responseData slice.
func sendRequest( func sendRequest(
ctx context.Context, ctx context.Context,
request *fasthttp.Request, request *Request,
responseData *[]Response, timeout time.Duration,
increase chan<- int64,
requestCount uint, requestCount uint,
clientDo ClientDoFunc, responseData *[]*Response,
increase chan<- int64,
wg *sync.WaitGroup, wg *sync.WaitGroup,
) { ) {
defer fasthttp.ReleaseRequest(request)
defer wg.Done() defer wg.Done()
for range requestCount { for range requestCount {
@ -143,14 +130,17 @@ func sendRequest(
func() { func() {
startTime := time.Now() startTime := time.Now()
response, err := clientDo(ctx, request) response, err := request.Send(ctx, timeout)
completedTime := time.Since(startTime) completedTime := time.Since(startTime)
if response != nil {
defer fasthttp.ReleaseResponse(response)
}
if err != nil { if err != nil {
if err == customerrors.ErrInterrupt { if err == customerrors.ErrInterrupt {
return return
} }
*responseData = append(*responseData, Response{ *responseData = append(*responseData, &Response{
StatusCode: 0, StatusCode: 0,
Error: err, Error: err,
Time: completedTime, Time: completedTime,
@ -158,9 +148,8 @@ func sendRequest(
increase <- 1 increase <- 1
return return
} }
defer fasthttp.ReleaseResponse(response)
*responseData = append(*responseData, Response{ *responseData = append(*responseData, &Response{
StatusCode: response.StatusCode(), StatusCode: response.StatusCode(),
Error: nil, Error: nil,
Time: completedTime, Time: completedTime,

54
utils/convert.go Normal file
View File

@ -0,0 +1,54 @@
package utils
import (
"encoding/json"
"reflect"
"strings"
)
func MarshalJSON(v any, maxSliceSize uint) string {
rv := reflect.ValueOf(v)
if rv.Kind() == reflect.Slice && rv.Len() == 0 {
return "[]"
}
data, err := json.MarshalIndent(truncateLists(v, int(maxSliceSize)), "", " ")
if err != nil {
return "{}"
}
return strings.Replace(string(data), `"..."`, "...", -1)
}
func truncateLists(v interface{}, maxItems int) interface{} {
rv := reflect.ValueOf(v)
switch rv.Kind() {
case reflect.Slice, reflect.Array:
if rv.Len() > maxItems {
newSlice := reflect.MakeSlice(rv.Type(), maxItems, maxItems)
reflect.Copy(newSlice, rv.Slice(0, maxItems))
newSlice = reflect.Append(newSlice, reflect.ValueOf("..."))
return newSlice.Interface()
}
case reflect.Map:
newMap := reflect.MakeMap(rv.Type())
for _, key := range rv.MapKeys() {
newMap.SetMapIndex(key, reflect.ValueOf(truncateLists(rv.MapIndex(key).Interface(), maxItems)))
}
return newMap.Interface()
case reflect.Struct:
newStruct := reflect.New(rv.Type()).Elem()
for i := 0; i < rv.NumField(); i++ {
newStruct.Field(i).Set(reflect.ValueOf(truncateLists(rv.Field(i).Interface(), maxItems)))
}
return newStruct.Interface()
case reflect.Ptr:
if rv.IsNil() {
return nil
}
return truncateLists(rv.Elem().Interface(), maxItems)
}
return v
}

View File

@ -1,7 +1,9 @@
package utils package utils
func Flatten[T any](nested [][]T) []T { import "math/rand"
flattened := make([]T, 0)
func Flatten[T any](nested [][]*T) []*T {
flattened := make([]*T, 0)
for _, n := range nested { for _, n := range nested {
flattened = append(flattened, n...) flattened = append(flattened, n...)
} }
@ -16,3 +18,33 @@ func Contains[T comparable](slice []T, item T) bool {
} }
return false return false
} }
// RandomValueCycle returns a function that cycles through the provided slice of values
// in a random order. Each call to the returned function will yield a value from the slice.
// The order of values is determined by the provided random number generator.
//
// The returned function will cycle through the values in a random order until all values
// have been returned at least once. After all values have been returned, the function will
// reset and start cycling through the values in a random order again.
// The returned function isn't thread-safe and should be used in a single-threaded context.
func RandomValueCycle[Value any](values []Value, localRand *rand.Rand) func() Value {
var (
clientsCount int = len(values)
currentIndex int = localRand.Intn(clientsCount)
stopIndex int = currentIndex
)
return func() Value {
client := values[currentIndex]
currentIndex++
if currentIndex == clientsCount {
currentIndex = 0
}
if currentIndex == stopIndex {
currentIndex = localRand.Intn(clientsCount)
stopIndex = currentIndex
}
return client
}
}