added random value selection feature to request objects (#19)

This commit is contained in:
Aykhan Shahsuvarov 2024-09-12 01:03:15 +04:00
parent d0e5c343ea
commit 8ad0bb5697
7 changed files with 266 additions and 60 deletions

View File

@ -6,6 +6,7 @@ import (
"os" "os"
"time" "time"
"github.com/aykhans/dodo/utils"
"github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/table"
) )
@ -30,11 +31,11 @@ type RequestConfig struct {
Timeout time.Duration Timeout time.Duration
DodosCount uint DodosCount uint
RequestCount uint RequestCount uint
Params map[string]string Params map[string][]string
Headers map[string]string Headers map[string][]string
Cookies map[string]string Cookies map[string][]string
Proxies []Proxy Proxies []Proxy
Body string Body []string
Yes bool Yes bool
} }
@ -55,17 +56,17 @@ func (config *RequestConfig) Print() {
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Dodos", config.DodosCount}) t.AppendRow(table.Row{"Dodos", config.DodosCount})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Request Count", config.RequestCount}) t.AppendRow(table.Row{"Request", config.RequestCount})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Params Count", len(config.Params)}) t.AppendRow(table.Row{"Params", utils.MarshalJSON(config.Params, 3)})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Headers Count", len(config.Headers)}) t.AppendRow(table.Row{"Headers", utils.MarshalJSON(config.Headers, 3)})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Cookies Count", len(config.Cookies)}) t.AppendRow(table.Row{"Cookies", utils.MarshalJSON(config.Cookies, 3)})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Proxies Count", len(config.Proxies)}) t.AppendRow(table.Row{"Proxies", utils.MarshalJSON(config.Proxies, 3)})
t.AppendSeparator() t.AppendSeparator()
t.AppendRow(table.Row{"Body", config.Body}) t.AppendRow(table.Row{"Body", utils.MarshalJSON(config.Body, 3)})
t.Render() t.Render()
} }
@ -134,11 +135,11 @@ type Proxy struct {
type JSONConfig struct { type JSONConfig struct {
Config Config
Params map[string]string `json:"params"` Params map[string][]string `json:"params"`
Headers map[string]string `json:"headers"` Headers map[string][]string `json:"headers"`
Cookies map[string]string `json:"cookies"` Cookies map[string][]string `json:"cookies"`
Proxies []Proxy `json:"proxies" validate:"dive"` Proxies []Proxy `json:"proxies" validate:"dive"`
Body string `json:"body"` Body []string `json:"body"`
} }
func (config *JSONConfig) MergeConfigs(newConfig *JSONConfig) { func (config *JSONConfig) MergeConfigs(newConfig *JSONConfig) {
@ -152,7 +153,7 @@ func (config *JSONConfig) MergeConfigs(newConfig *JSONConfig) {
if len(newConfig.Cookies) != 0 { if len(newConfig.Cookies) != 0 {
config.Cookies = newConfig.Cookies config.Cookies = newConfig.Cookies
} }
if newConfig.Body != "" { if len(newConfig.Body) != 0 {
config.Body = newConfig.Body config.Body = newConfig.Body
} }
if len(newConfig.Proxies) != 0 { if len(newConfig.Proxies) != 0 {

View File

@ -16,6 +16,8 @@ import (
"github.com/valyala/fasthttp/fasthttpproxy" "github.com/valyala/fasthttp/fasthttpproxy"
) )
type ClientDoFunc func(ctx context.Context, request *fasthttp.Request) (*fasthttp.Response, error)
// getClientDoFunc returns a ClientDoFunc function that can be used to make HTTP requests. // getClientDoFunc returns a ClientDoFunc function that can be used to make HTTP requests.
// //
// The function first checks if there are any proxies available. If there are, it retrieves the active proxy clients // The function first checks if there are any proxies available. If there are, it retrieves the active proxy clients

View File

@ -1,12 +1,83 @@
package requests package requests
import ( import (
"context"
"math/rand"
"net/url" "net/url"
"github.com/aykhans/dodo/config" "github.com/aykhans/dodo/config"
customerrors "github.com/aykhans/dodo/custom_errors"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
) )
// getRequests generates a list of HTTP requests based on the provided parameters.
//
// Parameters:
// - ctx: The context to control cancellation and deadlines.
// - URL: The base URL for the requests.
// - Headers: A map of headers to include in each request.
// - Cookies: A map of cookies to include in each request.
// - Params: A map of query parameters to include in each request.
// - Method: The HTTP method to use for the requests (e.g., GET, POST).
// - Bodies: A list of request bodies to cycle through for each request.
// - RequestCount: The number of requests to generate.
//
// Returns:
// - A list of fasthttp.Request objects based on the provided parameters.
// - An error if the context is canceled.
func getRequests(
ctx context.Context,
URL *url.URL,
Headers map[string][]string,
Cookies map[string][]string,
Params map[string][]string,
Method string,
Bodies []string,
RequestCount uint,
) ([]*fasthttp.Request, error) {
requests := make([]*fasthttp.Request, 0, RequestCount)
bodiesLen := len(Bodies)
getBody := func() string { return "" }
if bodiesLen == 1 {
getBody = func() string { return Bodies[0] }
} else if bodiesLen > 1 {
currentIndex := 0
stopIndex := bodiesLen - 1
getBody = func() string {
body := Bodies[currentIndex%bodiesLen]
if currentIndex == stopIndex {
currentIndex = rand.Intn(bodiesLen)
stopIndex = currentIndex - 1
} else {
currentIndex = (currentIndex + 1) % bodiesLen
}
return body
}
}
getHeaders := getKeyValueSetFunc(Headers)
getCookies := getKeyValueSetFunc(Cookies)
getParams := getKeyValueSetFunc(Params)
for range RequestCount {
if ctx.Err() != nil {
return nil, customerrors.ErrInterrupt
}
request := newRequest(
URL,
getHeaders(),
getCookies(),
getParams(),
Method,
getBody(),
)
requests = append(requests, request)
}
return requests, nil
}
// newRequest creates a new fasthttp.Request object with the provided parameters. // newRequest creates a new fasthttp.Request object with the provided parameters.
// It sets the request URI, host header, headers, cookies, params, method, and body. // It sets the request URI, host header, headers, cookies, params, method, and body.
func newRequest( func newRequest(
@ -70,3 +141,73 @@ func setRequestMethod(req *fasthttp.Request, method string) {
func setRequestBody(req *fasthttp.Request, body string) { func setRequestBody(req *fasthttp.Request, body string) {
req.SetBody([]byte(body)) req.SetBody([]byte(body))
} }
// getKeyValueSetFunc generates a function that returns a map of key-value pairs based on the provided key-value set.
// The generated function will either return fixed values or random values depending on the input.
//
// Returns:
// - A function that returns a map of key-value pairs. If the input map contains multiple values for a key,
// the returned function will generate random values for that key. If the input map contains a single value
// for a key, the returned function will always return that value. If the input map is empty for a key,
// the returned function will generate an empty string for that key.
func getKeyValueSetFunc[
KeyValueSet map[string][]string,
KeyValue map[string]string,
](keyValueSet KeyValueSet) func() KeyValue {
getKeyValueSlice := []map[string]func() string{}
isRandom := false
for key, values := range keyValueSet {
valuesLen := len(values)
// if values is empty, return a function that generates empty string
// if values has only one element, return a function that generates that element
// if values has more than one element, return a function that generates a random element
getKeyValue := func() string { return "" }
if valuesLen == 1 {
getKeyValue = func() string { return values[0] }
} else if valuesLen > 1 {
currentIndex := 0
stopIndex := valuesLen - 1
getKeyValue = func() string {
value := values[currentIndex%valuesLen]
if currentIndex == stopIndex {
currentIndex = rand.Intn(valuesLen)
stopIndex = currentIndex - 1
} else {
currentIndex = (currentIndex + 1) % valuesLen
}
return value
}
isRandom = true
}
getKeyValueSlice = append(
getKeyValueSlice,
map[string]func() string{key: getKeyValue},
)
}
// if isRandom is true, return a function that generates random values,
// otherwise return a function that generates fixed values to avoid unnecessary random number generation
if isRandom {
return func() KeyValue {
keyValues := make(KeyValue, len(getKeyValueSlice))
for _, keyValue := range getKeyValueSlice {
for key, value := range keyValue {
keyValues[key] = value()
}
}
return keyValues
}
} else {
keyValues := make(KeyValue, len(getKeyValueSlice))
for _, keyValue := range getKeyValueSlice {
for key, value := range keyValue {
keyValues[key] = value()
}
}
return func() KeyValue { return keyValues }
}
}

View File

@ -1,14 +1,12 @@
package requests package requests
import ( import (
"context"
"fmt" "fmt"
"os" "os"
"time" "time"
"github.com/aykhans/dodo/utils" "github.com/aykhans/dodo/utils"
"github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/table"
"github.com/valyala/fasthttp"
) )
type Response struct { type Response struct {
@ -17,9 +15,7 @@ type Response struct {
Time time.Duration Time time.Duration
} }
type Responses []Response type Responses []*Response
type ClientDoFunc func(ctx context.Context, request *fasthttp.Request) (*fasthttp.Response, error)
// Print prints the responses in a tabular format, including information such as // Print prints the responses in a tabular format, including information such as
// response count, minimum time, maximum time, and average time. // response count, minimum time, maximum time, and average time.

View File

@ -36,21 +36,25 @@ func Run(ctx context.Context, requestConfig *config.RequestConfig) (Responses, e
return nil, customerrors.ErrInterrupt return nil, customerrors.ErrInterrupt
} }
request := newRequest( requests, err := getRequests(
ctx,
requestConfig.URL, requestConfig.URL,
requestConfig.Headers, requestConfig.Headers,
requestConfig.Cookies, requestConfig.Cookies,
requestConfig.Params, requestConfig.Params,
requestConfig.Method, requestConfig.Method,
requestConfig.Body, requestConfig.Body,
requestConfig.RequestCount,
) )
defer fasthttp.ReleaseRequest(request) if err != nil {
return nil, err
}
responses := releaseDodos( responses := releaseDodos(
ctx, ctx,
request, requests,
clientDoFunc, clientDoFunc,
requestConfig.GetValidDodosCountForRequests(), requestConfig.GetValidDodosCountForRequests(),
requestConfig.RequestCount,
) )
if ctx.Err() != nil && len(responses) == 0 { if ctx.Err() != nil && len(responses) == 0 {
return nil, customerrors.ErrInterrupt return nil, customerrors.ErrInterrupt
@ -59,51 +63,59 @@ func Run(ctx context.Context, requestConfig *config.RequestConfig) (Responses, e
return responses, nil return responses, nil
} }
// releaseDodos sends multiple HTTP requests concurrently using multiple "dodos" (goroutines). // releaseDodos sends HTTP requests concurrently using multiple "dodos" (goroutines).
// It takes a mainRequest as the base request, timeout duration for each request, clientDoFunc for customizing the client behavior, //
// dodosCount as the number of goroutines to be used, and requestCount as the total number of requests to be sent. // Parameters:
// It returns the responses received from all the requests. // - ctx: The context to control the lifecycle of the requests.
// - requests: A slice of HTTP requests to be sent.
// - clientDoFunc: A function to execute the HTTP requests.
// - dodosCount: The number of dodos (goroutines) to use for sending the requests.
//
// Returns:
// - A slice of Response objects containing the results of the requests.
//
// The function divides the requests into equal parts based on the number of dodos.
// It then sends each part concurrently using a separate goroutine.
func releaseDodos( func releaseDodos(
ctx context.Context, ctx context.Context,
mainRequest *fasthttp.Request, requests []*fasthttp.Request,
clientDoFunc ClientDoFunc, clientDoFunc ClientDoFunc,
dodosCount uint, dodosCount uint,
requestCount uint,
) Responses { ) Responses {
var ( var (
wg sync.WaitGroup wg sync.WaitGroup
streamWG sync.WaitGroup streamWG sync.WaitGroup
requestCountPerDodo uint requestCountPerDodo uint
dodosCountInt = int(dodosCount) dodosCountInt int = int(dodosCount)
totalRequestCount uint = uint(len(requests))
requestCount uint = 0
responses = make([][]*Response, dodosCount)
increase = make(chan int64, totalRequestCount)
) )
wg.Add(dodosCountInt) wg.Add(dodosCountInt)
streamWG.Add(1) streamWG.Add(1)
responses := make([][]Response, dodosCount)
increase := make(chan int64, requestCount)
streamCtx, streamCtxCancel := context.WithCancel(context.Background()) streamCtx, streamCtxCancel := context.WithCancel(context.Background())
go streamProgress(streamCtx, &streamWG, int64(requestCount), "Dodos Working🔥", increase)
go streamProgress(streamCtx, &streamWG, int64(totalRequestCount), "Dodos Working🔥", increase)
for i := range dodosCount { for i := range dodosCount {
if i+1 == dodosCount { if i+1 == dodosCount {
requestCountPerDodo = requestCount - (i * requestCount / dodosCount) requestCountPerDodo = totalRequestCount - (i * totalRequestCount / dodosCount)
} else { } else {
requestCountPerDodo = ((i + 1) * requestCount / dodosCount) - requestCountPerDodo = ((i + 1) * totalRequestCount / dodosCount) -
(i * requestCount / dodosCount) (i * totalRequestCount / dodosCount)
} }
dodoSpecificRequest := &fasthttp.Request{}
mainRequest.CopyTo(dodoSpecificRequest)
go sendRequest( go sendRequest(
ctx, ctx,
dodoSpecificRequest, requests[requestCount:requestCount+requestCountPerDodo],
&responses[i], &responses[i],
increase, increase,
requestCountPerDodo,
clientDoFunc, clientDoFunc,
&wg, &wg,
) )
requestCount += requestCountPerDodo
} }
wg.Wait() wg.Wait()
streamCtxCancel() streamCtxCancel()
@ -111,37 +123,37 @@ func releaseDodos(
return utils.Flatten(responses) return utils.Flatten(responses)
} }
// sendRequest sends an HTTP request using the provided clientDo function and handles the response. // sendRequest sends multiple HTTP requests concurrently and collects their responses.
// //
// Parameters: // Parameters:
// - ctx: The context to control cancellation and timeout. // - ctx: The context to control cancellation and timeout.
// - request: The HTTP request to be sent. // - requests: A slice of pointers to fasthttp.Request objects to be sent.
// - responseData: A slice to store the response data. // - responseData: A pointer to a slice of *Response objects to store the results.
// - increase: A channel to signal the completion of a request. // - increase: A channel to signal the completion of each request.
// - requestCount: The number of requests to be sent.
// - clientDo: A function to execute the HTTP request. // - clientDo: A function to execute the HTTP request.
// - wg: A wait group to signal the completion of the function. // - wg: A wait group to synchronize the completion of the requests.
// //
// The function sends the specified number of requests, handles errors, and appends the response data // The function iterates over the provided requests, sending each one using the clientDo function.
// to the responseData slice. // It measures the time taken for each request and appends the response data to responseData.
// If an error occurs, it appends an error response. The function signals completion through the increase channel
// and ensures proper resource cleanup by releasing requests and responses.
func sendRequest( func sendRequest(
ctx context.Context, ctx context.Context,
request *fasthttp.Request, requests []*fasthttp.Request,
responseData *[]Response, responseData *[]*Response,
increase chan<- int64, increase chan<- int64,
requestCount uint,
clientDo ClientDoFunc, clientDo ClientDoFunc,
wg *sync.WaitGroup, wg *sync.WaitGroup,
) { ) {
defer fasthttp.ReleaseRequest(request)
defer wg.Done() defer wg.Done()
for range requestCount { for _, request := range requests {
if ctx.Err() != nil { if ctx.Err() != nil {
return return
} }
func() { func() {
defer fasthttp.ReleaseRequest(request)
startTime := time.Now() startTime := time.Now()
response, err := clientDo(ctx, request) response, err := clientDo(ctx, request)
completedTime := time.Since(startTime) completedTime := time.Since(startTime)
@ -150,7 +162,7 @@ func sendRequest(
if err == customerrors.ErrInterrupt { if err == customerrors.ErrInterrupt {
return return
} }
*responseData = append(*responseData, Response{ *responseData = append(*responseData, &Response{
StatusCode: 0, StatusCode: 0,
Error: err, Error: err,
Time: completedTime, Time: completedTime,
@ -160,7 +172,7 @@ func sendRequest(
} }
defer fasthttp.ReleaseResponse(response) defer fasthttp.ReleaseResponse(response)
*responseData = append(*responseData, Response{ *responseData = append(*responseData, &Response{
StatusCode: response.StatusCode(), StatusCode: response.StatusCode(),
Error: nil, Error: nil,
Time: completedTime, Time: completedTime,

54
utils/convert.go Normal file
View File

@ -0,0 +1,54 @@
package utils
import (
"encoding/json"
"reflect"
"strings"
)
func MarshalJSON(v any, maxSliceSize uint) string {
rv := reflect.ValueOf(v)
if rv.Kind() == reflect.Slice && rv.Len() == 0 {
return "[]"
}
data, err := json.MarshalIndent(truncateLists(v, int(maxSliceSize)), "", " ")
if err != nil {
return "{}"
}
return strings.Replace(string(data), `"..."`, "...", -1)
}
func truncateLists(v interface{}, maxItems int) interface{} {
rv := reflect.ValueOf(v)
switch rv.Kind() {
case reflect.Slice, reflect.Array:
if rv.Len() > maxItems {
newSlice := reflect.MakeSlice(rv.Type(), maxItems, maxItems)
reflect.Copy(newSlice, rv.Slice(0, maxItems))
newSlice = reflect.Append(newSlice, reflect.ValueOf("..."))
return newSlice.Interface()
}
case reflect.Map:
newMap := reflect.MakeMap(rv.Type())
for _, key := range rv.MapKeys() {
newMap.SetMapIndex(key, reflect.ValueOf(truncateLists(rv.MapIndex(key).Interface(), maxItems)))
}
return newMap.Interface()
case reflect.Struct:
newStruct := reflect.New(rv.Type()).Elem()
for i := 0; i < rv.NumField(); i++ {
newStruct.Field(i).Set(reflect.ValueOf(truncateLists(rv.Field(i).Interface(), maxItems)))
}
return newStruct.Interface()
case reflect.Ptr:
if rv.IsNil() {
return nil
}
return truncateLists(rv.Elem().Interface(), maxItems)
}
return v
}

View File

@ -1,7 +1,7 @@
package utils package utils
func Flatten[T any](nested [][]T) []T { func Flatten[T any](nested [][]*T) []*T {
flattened := make([]T, 0) flattened := make([]*T, 0)
for _, n := range nested { for _, n := range nested {
flattened = append(flattened, n...) flattened = append(flattened, n...)
} }