🔨 Restructure entire project logic

- Moved readers to the config package
- Added an option to read remote config files
- Moved the validation package to the config package and removed the validator dependency
- Moved the customerrors package to the config package
- Replaced fatih/color with jedib0t/go-pretty/v6/text
- Removed proxy check functionality
- Added param, header, cookie, body, and proxy flags to the CLI
- Allowed multiple values for the same key in params, headers, and cookies
This commit is contained in:
2025-03-16 21:20:33 +04:00
parent 8f811e1bec
commit 00f0bcb2de
35 changed files with 1461 additions and 1492 deletions

View File

@@ -2,16 +2,12 @@ package requests
import (
"context"
"fmt"
"errors"
"math/rand"
"net/url"
"sync"
"time"
"github.com/aykhans/dodo/config"
"github.com/aykhans/dodo/readers"
"github.com/aykhans/dodo/utils"
"github.com/fatih/color"
"github.com/valyala/fasthttp"
"github.com/valyala/fasthttp/fasthttpproxy"
)
@@ -23,71 +19,38 @@ type ClientGeneratorFunc func() *fasthttp.HostClient
func getClients(
ctx context.Context,
timeout time.Duration,
proxies []config.Proxy,
dodosCount uint,
proxies []url.URL,
maxConns uint,
yes bool,
noProxyCheck bool,
URL *url.URL,
URL url.URL,
) []*fasthttp.HostClient {
isTLS := URL.Scheme == "https"
if proxiesLen := len(proxies); proxiesLen > 0 {
// If noProxyCheck is true, we will return the clients without checking the proxies.
if noProxyCheck {
clients := make([]*fasthttp.HostClient, 0, proxiesLen)
addr := URL.Host
if isTLS && URL.Port() == "" {
addr += ":443"
}
for _, proxy := range proxies {
dialFunc, err := getDialFunc(&proxy, timeout)
if err != nil {
continue
}
clients = append(clients, &fasthttp.HostClient{
MaxConns: int(maxConns),
IsTLS: isTLS,
Addr: addr,
Dial: dialFunc,
MaxIdleConnDuration: timeout,
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
},
)
}
return clients
clients := make([]*fasthttp.HostClient, 0, proxiesLen)
addr := URL.Host
if isTLS && URL.Port() == "" {
addr += ":443"
}
// Else, we will check the proxies and return the active ones.
activeProxyClients := getActiveProxyClients(
ctx, proxies, timeout, dodosCount, maxConns, URL,
)
if ctx.Err() != nil {
return nil
}
activeProxyClientsCount := uint(len(activeProxyClients))
var yesOrNoMessage string
var yesOrNoDefault bool
if activeProxyClientsCount == 0 {
yesOrNoDefault = false
yesOrNoMessage = color.YellowString("No active proxies found. Do you want to continue?")
} else {
yesOrNoMessage = color.YellowString("Found %d active proxies. Do you want to continue?", activeProxyClientsCount)
}
if !yes {
response := readers.CLIYesOrNoReader("\n"+yesOrNoMessage, yesOrNoDefault)
if !response {
utils.PrintAndExit("Exiting...")
for _, proxy := range proxies {
dialFunc, err := getDialFunc(&proxy, timeout)
if err != nil {
continue
}
clients = append(clients, &fasthttp.HostClient{
MaxConns: int(maxConns),
IsTLS: isTLS,
Addr: addr,
Dial: dialFunc,
MaxIdleConnDuration: timeout,
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
},
)
}
fmt.Println()
if activeProxyClientsCount > 0 {
return activeProxyClients
}
return clients
}
client := &fasthttp.HostClient{
@@ -102,200 +65,19 @@ func getClients(
return []*fasthttp.HostClient{client}
}
// getActiveProxyClients divides the proxies into slices based on the number of dodos and
// launches goroutines to find active proxy clients for each slice.
// It uses a progress tracker to monitor the progress of the search.
// Once all goroutines have completed, the function waits for them to finish and
// returns a flattened slice of active proxy clients.
func getActiveProxyClients(
ctx context.Context,
proxies []config.Proxy,
timeout time.Duration,
dodosCount uint,
maxConns uint,
URL *url.URL,
) []*fasthttp.HostClient {
activeProxyClientsArray := make([][]*fasthttp.HostClient, dodosCount)
proxiesCount := len(proxies)
dodosCountInt := int(dodosCount)
var (
wg sync.WaitGroup
streamWG sync.WaitGroup
)
wg.Add(dodosCountInt)
streamWG.Add(1)
var proxiesSlice []config.Proxy
increase := make(chan int64, proxiesCount)
streamCtx, streamCtxCancel := context.WithCancel(context.Background())
go streamProgress(streamCtx, &streamWG, int64(proxiesCount), "Searching for active proxies🌐", increase)
for i := range dodosCountInt {
if i+1 == dodosCountInt {
proxiesSlice = proxies[i*proxiesCount/dodosCountInt:]
} else {
proxiesSlice = proxies[i*proxiesCount/dodosCountInt : (i+1)*proxiesCount/dodosCountInt]
}
go findActiveProxyClients(
ctx,
proxiesSlice,
timeout,
&activeProxyClientsArray[i],
increase,
maxConns,
URL,
&wg,
)
}
wg.Wait()
streamCtxCancel()
streamWG.Wait()
return utils.Flatten(activeProxyClientsArray)
}
// findActiveProxyClients checks a list of proxies to determine which ones are active
// and appends the active ones to the provided activeProxyClients slice.
//
// Parameters:
// - ctx: The context to control cancellation and timeout.
// - proxies: A slice of Proxy configurations to be checked.
// - timeout: The duration to wait for each proxy check before timing out.
// - activeProxyClients: A pointer to a slice where active proxy clients will be appended.
// - increase: A channel to signal the increase of checked proxies count.
// - URL: The URL to be used for checking the proxies.
// - wg: A WaitGroup to signal when the function is done.
//
// The function sends a GET request to each proxy using the provided URL. If the proxy
// responds with a status code of 200, it is considered active and added to the activeProxyClients slice.
// The function respects the context's cancellation and timeout settings.
func findActiveProxyClients(
ctx context.Context,
proxies []config.Proxy,
timeout time.Duration,
activeProxyClients *[]*fasthttp.HostClient,
increase chan<- int64,
maxConns uint,
URL *url.URL,
wg *sync.WaitGroup,
) {
defer wg.Done()
request := fasthttp.AcquireRequest()
defer fasthttp.ReleaseRequest(request)
request.SetRequestURI(config.ProxyCheckURL)
request.Header.SetMethod("GET")
for _, proxy := range proxies {
if ctx.Err() != nil {
return
}
func() {
defer func() { increase <- 1 }()
response := fasthttp.AcquireResponse()
defer fasthttp.ReleaseResponse(response)
dialFunc, err := getDialFunc(&proxy, timeout)
if err != nil {
return
}
client := &fasthttp.Client{
Dial: dialFunc,
}
defer client.CloseIdleConnections()
ch := make(chan error)
go func() {
err := client.DoTimeout(request, response, timeout)
ch <- err
}()
select {
case err := <-ch:
if err != nil {
return
}
break
case <-time.After(timeout):
return
case <-ctx.Done():
return
}
isTLS := URL.Scheme == "https"
addr := URL.Host
if isTLS && URL.Port() == "" {
addr += ":443"
}
if response.StatusCode() == 200 {
*activeProxyClients = append(
*activeProxyClients,
&fasthttp.HostClient{
MaxConns: int(maxConns),
IsTLS: isTLS,
Addr: addr,
Dial: dialFunc,
MaxIdleConnDuration: timeout,
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
},
)
}
}()
}
}
// getDialFunc returns a fasthttp.DialFunc based on the provided proxy configuration.
// It takes a pointer to a config.Proxy struct as input and returns a fasthttp.DialFunc and an error.
// The function parses the proxy URL, determines the scheme (socks5, socks5h, http, or https),
// and creates a dialer accordingly. If the proxy URL is invalid or the scheme is not supported,
// it returns an error.
func getDialFunc(proxy *config.Proxy, timeout time.Duration) (fasthttp.DialFunc, error) {
parsedProxyURL, err := url.Parse(proxy.URL)
if err != nil {
return nil, err
}
// getDialFunc returns the appropriate fasthttp.DialFunc based on the provided proxy URL scheme.
// It supports SOCKS5 ('socks5' or 'socks5h') and HTTP ('http') proxy schemes.
// For HTTP proxies, the timeout parameter determines connection timeouts.
// Returns an error if the proxy scheme is unsupported.
func getDialFunc(proxy *url.URL, timeout time.Duration) (fasthttp.DialFunc, error) {
var dialer fasthttp.DialFunc
if parsedProxyURL.Scheme == "socks5" || parsedProxyURL.Scheme == "socks5h" {
if proxy.Username != "" {
dialer = fasthttpproxy.FasthttpSocksDialer(
fmt.Sprintf(
"%s://%s:%s@%s",
parsedProxyURL.Scheme,
proxy.Username,
proxy.Password,
parsedProxyURL.Host,
),
)
} else {
dialer = fasthttpproxy.FasthttpSocksDialer(
fmt.Sprintf(
"%s://%s",
parsedProxyURL.Scheme,
parsedProxyURL.Host,
),
)
}
} else if parsedProxyURL.Scheme == "http" {
if proxy.Username != "" {
dialer = fasthttpproxy.FasthttpHTTPDialerTimeout(
fmt.Sprintf(
"%s:%s@%s",
proxy.Username, proxy.Password, parsedProxyURL.Host,
),
timeout,
)
} else {
dialer = fasthttpproxy.FasthttpHTTPDialerTimeout(
parsedProxyURL.Host,
timeout,
)
}
if proxy.Scheme == "socks5" || proxy.Scheme == "socks5h" {
dialer = fasthttpproxy.FasthttpSocksDialerDualStack(proxy.String())
} else if proxy.Scheme == "http" {
dialer = fasthttpproxy.FasthttpHTTPDialerDualStackTimeout(proxy.String(), timeout)
} else {
return nil, err
return nil, errors.New("unsupported proxy scheme")
}
return dialer, nil
}

View File

@@ -7,7 +7,6 @@ import (
"time"
"github.com/jedib0t/go-pretty/v6/progress"
"github.com/valyala/fasthttp"
)
// streamProgress streams the progress of a task to the console using a progress bar.
@@ -37,9 +36,11 @@ func streamProgress(
for {
select {
case <-ctx.Done():
if ctx.Err() != context.Canceled {
dodosTracker.MarkAsErrored()
}
fmt.Printf("\r")
dodosTracker.MarkAsErrored()
time.Sleep(time.Millisecond * 300)
time.Sleep(time.Millisecond * 500)
pw.Stop()
return
@@ -48,28 +49,3 @@ func streamProgress(
}
}
}
// checkConnection checks the internet connection by making requests to different websites.
// It returns true if the connection is successful, otherwise false.
func checkConnection(ctx context.Context) bool {
ch := make(chan bool)
go func() {
_, _, err := fasthttp.Get(nil, "https://www.google.com")
if err != nil {
_, _, err = fasthttp.Get(nil, "https://www.bing.com")
if err != nil {
_, _, err = fasthttp.Get(nil, "https://www.yahoo.com")
ch <- err == nil
}
ch <- true
}
ch <- true
}()
select {
case <-ctx.Done():
return false
case res := <-ch:
return res
}
}

View File

@@ -7,7 +7,7 @@ import (
"time"
"github.com/aykhans/dodo/config"
customerrors "github.com/aykhans/dodo/custom_errors"
"github.com/aykhans/dodo/types"
"github.com/aykhans/dodo/utils"
"github.com/valyala/fasthttp"
)
@@ -43,9 +43,9 @@ func (r *Request) Send(ctx context.Context, timeout time.Duration) (*fasthttp.Re
return response, nil
case <-time.After(timeout):
fasthttp.ReleaseResponse(response)
return nil, customerrors.ErrTimeout
return nil, types.ErrTimeout
case <-ctx.Done():
return nil, customerrors.ErrInterrupt
return nil, types.ErrInterrupt
}
}
@@ -74,9 +74,9 @@ func newRequest(
getRequest := getRequestGeneratorFunc(
requestConfig.URL,
requestConfig.Params,
requestConfig.Headers,
requestConfig.Cookies,
requestConfig.Params,
requestConfig.Method,
requestConfig.Body,
localRand,
@@ -90,37 +90,36 @@ func newRequest(
return requests
}
// getRequestGeneratorFunc returns a RequestGeneratorFunc which generates HTTP requests
// with the specified parameters.
// The function uses a local random number generator to select bodies, headers, cookies, and parameters
// if multiple options are provided.
// getRequestGeneratorFunc returns a RequestGeneratorFunc which generates HTTP requests with the specified parameters.
// The function uses a local random number generator to select bodies, headers, cookies, and parameters if multiple options are provided.
func getRequestGeneratorFunc(
URL *url.URL,
Headers map[string][]string,
Cookies map[string][]string,
Params map[string][]string,
Method string,
Bodies []string,
URL url.URL,
params types.Params,
headers types.Headers,
cookies types.Cookies,
method string,
bodies []string,
localRand *rand.Rand,
) RequestGeneratorFunc {
bodiesLen := len(Bodies)
bodiesLen := len(bodies)
getBody := func() string { return "" }
if bodiesLen == 1 {
getBody = func() string { return Bodies[0] }
getBody = func() string { return bodies[0] }
} else if bodiesLen > 1 {
getBody = utils.RandomValueCycle(Bodies, localRand)
getBody = utils.RandomValueCycle(bodies, localRand)
}
getHeaders := getKeyValueSetFunc(Headers, localRand)
getCookies := getKeyValueSetFunc(Cookies, localRand)
getParams := getKeyValueSetFunc(Params, localRand)
getParams := getKeyValueGeneratorFunc(params, localRand)
getHeaders := getKeyValueGeneratorFunc(headers, localRand)
getCookies := getKeyValueGeneratorFunc(cookies, localRand)
return func() *fasthttp.Request {
return newFasthttpRequest(
URL,
getParams(),
getHeaders(),
getCookies(),
getParams(),
Method,
method,
getBody(),
)
}
@@ -129,12 +128,12 @@ func getRequestGeneratorFunc(
// newFasthttpRequest creates a new fasthttp.Request object with the provided parameters.
// It sets the request URI, host header, headers, cookies, params, method, and body.
func newFasthttpRequest(
URL *url.URL,
Headers map[string]string,
Cookies map[string]string,
Params map[string]string,
Method string,
Body string,
URL url.URL,
params []types.KeyValue[string, string],
headers []types.KeyValue[string, string],
cookies []types.KeyValue[string, string],
method string,
body string,
) *fasthttp.Request {
request := fasthttp.AcquireRequest()
request.SetRequestURI(URL.Path)
@@ -142,12 +141,12 @@ func newFasthttpRequest(
// Set the host of the request to the host header
// If the host header is not set, the request will fail
// If there is host header in the headers, it will be overwritten
request.Header.Set("Host", URL.Host)
setRequestHeaders(request, Headers)
setRequestCookies(request, Cookies)
setRequestParams(request, Params)
setRequestMethod(request, Method)
setRequestBody(request, Body)
request.Header.SetHost(URL.Host)
setRequestParams(request, params)
setRequestHeaders(request, headers)
setRequestCookies(request, cookies)
setRequestMethod(request, method)
setRequestBody(request, body)
if URL.Scheme == "https" {
request.URI().SetScheme("https")
}
@@ -155,28 +154,28 @@ func newFasthttpRequest(
return request
}
// setRequestHeaders sets the headers of the given request with the provided key-value pairs.
func setRequestHeaders(req *fasthttp.Request, headers map[string]string) {
req.Header.Set("User-Agent", config.DefaultUserAgent)
for key, value := range headers {
req.Header.Set(key, value)
// setRequestParams adds the query parameters of the given request based on the provided key-value pairs.
func setRequestParams(req *fasthttp.Request, params []types.KeyValue[string, string]) {
for _, param := range params {
req.URI().QueryArgs().Add(param.Key, param.Value)
}
}
// setRequestCookies sets the cookies in the given request.
func setRequestCookies(req *fasthttp.Request, cookies map[string]string) {
for key, value := range cookies {
req.Header.SetCookie(key, value)
// setRequestHeaders adds the headers of the given request with the provided key-value pairs.
func setRequestHeaders(req *fasthttp.Request, headers []types.KeyValue[string, string]) {
for _, header := range headers {
req.Header.Add(header.Key, header.Value)
}
if req.Header.UserAgent() == nil {
req.Header.SetUserAgent(config.DefaultUserAgent)
}
}
// setRequestParams sets the query parameters of the given request based on the provided map of key-value pairs.
func setRequestParams(req *fasthttp.Request, params map[string]string) {
urlParams := url.Values{}
for key, value := range params {
urlParams.Add(key, value)
// setRequestCookies adds the cookies of the given request with the provided key-value pairs.
func setRequestCookies(req *fasthttp.Request, cookies []types.KeyValue[string, string]) {
for _, cookie := range cookies {
req.Header.Add("Cookie", cookie.Key+"="+cookie.Value)
}
req.URI().SetQueryString(urlParams.Encode())
}
// setRequestMethod sets the HTTP request method for the given request.
@@ -190,59 +189,62 @@ func setRequestBody(req *fasthttp.Request, body string) {
req.SetBody([]byte(body))
}
// getKeyValueSetFunc generates a function that returns a map of key-value pairs based on the provided key-value set.
// The generated function will either return fixed values or random values depending on the input.
// getKeyValueGeneratorFunc creates a function that generates key-value pairs for HTTP requests.
// It takes a slice of key-value pairs where each key maps to a slice of possible values,
// and a random number generator.
//
// Returns:
// - A function that returns a map of key-value pairs. If the input map contains multiple values for a key,
// the returned function will generate random values for that key. If the input map contains a single value
// for a key, the returned function will always return that value. If the input map is empty for a key,
// the returned function will generate an empty string for that key.
func getKeyValueSetFunc[
KeyValueSet map[string][]string,
KeyValue map[string]string,
](keyValueSet KeyValueSet, localRand *rand.Rand) func() KeyValue {
// If any key has multiple possible values, the function will randomly select one value for each
// call (using the provided random number generator). If all keys have at most one value, the
// function will always return the same set of key-value pairs for efficiency.
func getKeyValueGeneratorFunc[
T []types.KeyValue[string, string],
](
keyValueSlice []types.KeyValue[string, []string],
localRand *rand.Rand,
) func() T {
getKeyValueSlice := []map[string]func() string{}
isRandom := false
for key, values := range keyValueSet {
valuesLen := len(values)
// if values is empty, return a function that generates empty string
// if values has only one element, return a function that generates that element
// if values has more than one element, return a function that generates a random element
getKeyValue := func() string { return "" }
for _, kv := range keyValueSlice {
valuesLen := len(kv.Value)
getValueFunc := func() string { return "" }
if valuesLen == 1 {
getKeyValue = func() string { return values[0] }
getValueFunc = func() string { return kv.Value[0] }
} else if valuesLen > 1 {
getKeyValue = utils.RandomValueCycle(values, localRand)
getValueFunc = utils.RandomValueCycle(kv.Value, localRand)
isRandom = true
}
getKeyValueSlice = append(
getKeyValueSlice,
map[string]func() string{key: getKeyValue},
map[string]func() string{kv.Key: getValueFunc},
)
}
// if isRandom is true, return a function that generates random values,
// otherwise return a function that generates fixed values to avoid unnecessary random number generation
if isRandom {
return func() KeyValue {
keyValues := make(KeyValue, len(getKeyValueSlice))
for _, keyValue := range getKeyValueSlice {
return func() T {
keyValues := make(T, len(getKeyValueSlice))
for i, keyValue := range getKeyValueSlice {
for key, value := range keyValue {
keyValues[key] = value()
keyValues[i] = types.KeyValue[string, string]{
Key: key,
Value: value(),
}
}
}
return keyValues
}
} else {
keyValues := make(KeyValue, len(getKeyValueSlice))
for _, keyValue := range getKeyValueSlice {
keyValues := make(T, len(getKeyValueSlice))
for i, keyValue := range getKeyValueSlice {
for key, value := range keyValue {
keyValues[key] = value()
keyValues[i] = types.KeyValue[string, string]{
Key: key,
Value: value(),
}
}
}
return func() KeyValue { return keyValues }
return func() T { return keyValues }
}
}

View File

@@ -4,7 +4,7 @@ import (
"os"
"time"
. "github.com/aykhans/dodo/types"
"github.com/aykhans/dodo/types"
"github.com/aykhans/dodo/utils"
"github.com/jedib0t/go-pretty/v6/table"
)
@@ -32,8 +32,8 @@ func (responses Responses) Print() {
Min: responses[0].Time,
Max: responses[0].Time,
}
mergedResponses := make(map[string]Durations)
var allDurations Durations
mergedResponses := make(map[string]types.Durations)
var allDurations types.Durations
for _, response := range responses {
if response.Time < total.Min {

View File

@@ -7,48 +7,33 @@ import (
"time"
"github.com/aykhans/dodo/config"
customerrors "github.com/aykhans/dodo/custom_errors"
"github.com/aykhans/dodo/types"
"github.com/aykhans/dodo/utils"
"github.com/valyala/fasthttp"
)
// Run executes the main logic for processing requests based on the provided configuration.
// It first checks for an internet connection with a timeout context. If no connection is found,
// it returns an error. Then, it initializes clients based on the request configuration and
// releases the dodos. If the context is canceled and no responses are collected, it returns an interrupt error.
// It initializes clients based on the request configuration and releases the dodos.
// If the context is canceled and no responses are collected, it returns an interrupt error.
//
// Parameters:
// - ctx: The context for managing request lifecycle and cancellation.
// - requestConfig: The configuration for the request, including timeout, proxies, and other settings.
//
// Returns:
// - Responses: A collection of responses from the executed requests.
// - error: An error if the operation fails, such as no internet connection or an interrupt.
func Run(ctx context.Context, requestConfig *config.RequestConfig) (Responses, error) {
checkConnectionCtx, checkConnectionCtxCancel := context.WithTimeout(ctx, 8*time.Second)
if !checkConnection(checkConnectionCtx) {
checkConnectionCtxCancel()
return nil, customerrors.ErrNoInternet
}
checkConnectionCtxCancel()
clients := getClients(
ctx,
requestConfig.Timeout,
requestConfig.Proxies,
requestConfig.GetValidDodosCountForProxies(),
requestConfig.GetMaxConns(fasthttp.DefaultMaxConnsPerHost),
requestConfig.Yes,
requestConfig.NoProxyCheck,
requestConfig.URL,
)
if clients == nil {
return nil, customerrors.ErrInterrupt
return nil, types.ErrInterrupt
}
responses := releaseDodos(ctx, requestConfig, clients)
if ctx.Err() != nil && len(responses) == 0 {
return nil, customerrors.ErrInterrupt
return nil, types.ErrInterrupt
}
return responses, nil
@@ -139,7 +124,7 @@ func sendRequest(
}
if err != nil {
if err == customerrors.ErrInterrupt {
if err == types.ErrInterrupt {
return
}
*responseData = append(*responseData, &Response{