Merge pull request #2 from aykhans/refactor/requests

refactor/requests
This commit is contained in:
Aykhan Shahsuvarov 2024-07-24 16:43:12 +04:00 committed by GitHub
commit f37a6d7747
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 529 additions and 336 deletions

View File

@ -2,7 +2,7 @@
"method": "GET", "method": "GET",
"url": "https://example.com", "url": "https://example.com",
"timeout": 10000, "timeout": 10000,
"dodos_count": 1, "dodos_count": 50,
"request_count": 1000, "request_count": 1000,
"params": {}, "params": {},
"headers": {}, "headers": {},
@ -10,7 +10,7 @@
"body": "", "body": "",
"proxies": [ "proxies": [
{ {
"url": "http://example:8080", "url": "http://example.com:8080",
"username": "username", "username": "username",
"password": "password" "password": "password"
}, },

View File

@ -2,6 +2,7 @@ package config
import ( import (
"fmt" "fmt"
"net/url"
"os" "os"
"time" "time"
@ -23,9 +24,9 @@ type IConfig interface {
MergeConfigs(newConfig IConfig) IConfig MergeConfigs(newConfig IConfig) IConfig
} }
type DodoConfig struct { type RequestConfig struct {
Method string Method string
URL string URL *url.URL
Timeout time.Duration Timeout time.Duration
DodosCount int DodosCount int
RequestCount int RequestCount int
@ -36,7 +37,7 @@ type DodoConfig struct {
Body string Body string
} }
func (config *DodoConfig) Print() { func (config *RequestConfig) Print() {
t := table.NewWriter() t := table.NewWriter()
t.SetOutputMirror(os.Stdout) t.SetOutputMirror(os.Stdout)
t.SetStyle(table.StyleLight) t.SetStyle(table.StyleLight)
@ -55,6 +56,14 @@ func (config *DodoConfig) Print() {
t.Render() t.Render()
} }
func (config *RequestConfig) GetValidDodosCountForRequests() int {
return min(config.DodosCount, config.RequestCount)
}
func (config *RequestConfig) GetValidDodosCountForProxies() int {
return min(config.DodosCount, len(config.Proxies), MaxDodosCountForProxies)
}
type Config struct { type Config struct {
Method string `json:"method" validate:"http_method"` // custom validations: http_method Method string `json:"method" validate:"http_method"` // custom validations: http_method
URL string `json:"url" validate:"http_url,required"` URL string `json:"url" validate:"http_url,required"`

4
go.mod
View File

@ -7,17 +7,21 @@ require (
github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/jedib0t/go-pretty/v6 v6.5.9
github.com/spf13/cobra v1.8.1 github.com/spf13/cobra v1.8.1
github.com/spf13/pflag v1.0.5 github.com/spf13/pflag v1.0.5
github.com/valyala/fasthttp v1.55.0
golang.org/x/net v0.27.0 golang.org/x/net v0.27.0
) )
require ( require (
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.4 // indirect github.com/gabriel-vasile/mimetype v1.4.4 // indirect
github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/leodido/go-urn v1.4.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/rivo/uniseg v0.4.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
golang.org/x/crypto v0.25.0 // indirect golang.org/x/crypto v0.25.0 // indirect
golang.org/x/sys v0.22.0 // indirect golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.22.0 // indirect golang.org/x/term v0.22.0 // indirect

8
go.sum
View File

@ -1,3 +1,5 @@
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -15,6 +17,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+UV8OU= github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+UV8OU=
github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
@ -31,6 +35,10 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=

47
main.go
View File

@ -1,18 +1,18 @@
package main package main
import ( import (
"net/url"
"os" "os"
"strings" "strings"
"time" "time"
"github.com/aykhans/dodo/config" "github.com/aykhans/dodo/config"
"github.com/aykhans/dodo/custom_errors" customerrors "github.com/aykhans/dodo/custom_errors"
"github.com/aykhans/dodo/readers" "github.com/aykhans/dodo/readers"
"github.com/aykhans/dodo/requests" "github.com/aykhans/dodo/requests"
"github.com/aykhans/dodo/utils" "github.com/aykhans/dodo/utils"
"github.com/aykhans/dodo/validation" "github.com/aykhans/dodo/validation"
goValidator "github.com/go-playground/validator/v10" goValidator "github.com/go-playground/validator/v10"
"github.com/jedib0t/go-pretty/v6/table"
) )
func main() { func main() {
@ -61,9 +61,13 @@ func main() {
) )
} }
dodoConf := &config.DodoConfig{ parsedURL, err := url.Parse(conf.URL)
if err != nil {
utils.PrintErrAndExit(err)
}
dodoConf := &config.RequestConfig{
Method: conf.Method, Method: conf.Method,
URL: conf.URL, URL: parsedURL,
Timeout: time.Duration(conf.Timeout) * time.Millisecond, Timeout: time.Duration(conf.Timeout) * time.Millisecond,
DodosCount: conf.DodosCount, DodosCount: conf.DodosCount,
RequestCount: conf.RequestCount, RequestCount: conf.RequestCount,
@ -73,39 +77,8 @@ func main() {
Proxies: jsonConf.Proxies, Proxies: jsonConf.Proxies,
Body: jsonConf.Body, Body: jsonConf.Body,
} }
dodoConf.Print() dodoConf.Print()
responses, err := requests.Run(dodoConf)
if err != nil {
utils.PrintErrAndExit(err)
}
t := table.NewWriter() responses := requests.Run(dodoConf)
t.SetOutputMirror(os.Stdout) responses.Print()
t.SetStyle(table.StyleLight)
t.AppendHeader(table.Row{
"Response",
"Count",
"Min Time",
"Max Time",
"Average Time",
})
for _, mergedResponse := range responses.MergeDodoResponses() {
t.AppendRow(table.Row{
mergedResponse.Response,
mergedResponse.Count,
mergedResponse.MinTime,
mergedResponse.MaxTime,
mergedResponse.AvgTime,
})
t.AppendSeparator()
}
t.AppendFooter(table.Row{
"Total",
responses.Len(),
responses.MinTime(),
responses.MaxTime(),
responses.AvgTime(),
})
t.Render()
} }

View File

@ -2,247 +2,528 @@ package requests
import ( import (
"fmt" "fmt"
"io"
"math/rand" "math/rand"
"net/http"
"net/url" "net/url"
"strings" "os"
"sync" "sync"
"time" "time"
"github.com/aykhans/dodo/config" "github.com/aykhans/dodo/config"
"github.com/aykhans/dodo/custom_errors"
"github.com/aykhans/dodo/readers" "github.com/aykhans/dodo/readers"
"github.com/aykhans/dodo/utils" "github.com/aykhans/dodo/utils"
"github.com/jedib0t/go-pretty/v6/progress" "github.com/jedib0t/go-pretty/v6/progress"
"github.com/jedib0t/go-pretty/v6/table"
"github.com/valyala/fasthttp"
"github.com/valyala/fasthttp/fasthttpproxy"
) )
type DodoResponse struct { type Response struct {
Response string StatusCode int
Time time.Duration Error error
Time time.Duration
} }
type DodoResponses []DodoResponse type Responses []Response
type MergedDodoResponse struct { type ClientFunc func() *fasthttp.HostClient
Response string
Count int
AvgTime time.Duration
MinTime time.Duration
MaxTime time.Duration
}
func (d DodoResponses) Len() int { // Print prints the responses in a tabular format, including information such as
return len(d) // response count, minimum time, maximum time, and average time.
} func (respones *Responses) Print() {
var (
totalMinDuration time.Duration = (*respones)[0].Time
totalMaxDuration time.Duration = (*respones)[0].Time
totalDuration time.Duration
totalCount int = len(*respones)
)
mergedResponses := make(map[string][]time.Duration)
func (d DodoResponses) MinTime() time.Duration { for _, response := range *respones {
minTime := d[0].Time if response.Time < totalMinDuration {
for _, response := range d { totalMinDuration = response.Time
if response.Time < minTime { }
minTime = response.Time if response.Time > totalMaxDuration {
totalMaxDuration = response.Time
}
totalDuration += response.Time
if response.Error != nil {
mergedResponses[response.Error.Error()] = append(
mergedResponses[response.Error.Error()],
response.Time,
)
} else {
mergedResponses[fmt.Sprintf("%d", response.StatusCode)] = append(
mergedResponses[fmt.Sprintf("%d", response.StatusCode)],
response.Time,
)
} }
} }
return minTime
}
func (d DodoResponses) MaxTime() time.Duration { t := table.NewWriter()
maxTime := d[0].Time t.SetOutputMirror(os.Stdout)
for _, response := range d { t.SetStyle(table.StyleLight)
if response.Time > maxTime { t.AppendHeader(table.Row{
maxTime = response.Time "Response",
} "Count",
} "Min Time",
return maxTime "Max Time",
} "Average Time",
func (d DodoResponses) AvgTime() time.Duration {
var sum time.Duration
for _, response := range d {
sum += response.Time
}
return sum / time.Duration(len(d))
}
func (d DodoResponses) MergeDodoResponses() []MergedDodoResponse {
mergedResponses := make(map[string]*struct {
count int
minTime time.Duration
maxTime time.Duration
totalTime time.Duration
}) })
for _, response := range d { for key, durations := range mergedResponses {
if _, ok := mergedResponses[response.Response]; !ok { t.AppendRow(table.Row{
mergedResponses[response.Response] = &struct { key,
count int len(durations),
minTime time.Duration utils.MinDuration(durations...),
maxTime time.Duration utils.MaxDuration(durations...),
totalTime time.Duration utils.AvgDuration(durations...),
}{
count: 1,
minTime: response.Time,
maxTime: response.Time,
totalTime: response.Time,
}
} else {
mergedResponses[response.Response].count++
mergedResponses[response.Response].totalTime += response.Time
if response.Time < mergedResponses[response.Response].minTime {
mergedResponses[response.Response].minTime = response.Time
}
if response.Time > mergedResponses[response.Response].maxTime {
mergedResponses[response.Response].maxTime = response.Time
}
}
}
var result []MergedDodoResponse
for response, data := range mergedResponses {
result = append(result, MergedDodoResponse{
Response: response,
Count: data.count,
AvgTime: data.totalTime / time.Duration(data.count),
MinTime: data.minTime,
MaxTime: data.maxTime,
}) })
t.AppendSeparator()
} }
return result t.AppendRow(table.Row{
"Total",
totalCount,
totalMinDuration,
totalMaxDuration,
totalDuration / time.Duration(totalCount),
})
t.Render()
} }
func Run(conf *config.DodoConfig) (DodoResponses, error) { // Run executes the HTTP requests based on the provided request configuration.
params := setParams(conf.URL, conf.Params) // It returns the Responses type, which contains the responses received from all the requests.
headers := getHeaders(conf.Headers) func Run(requestConfig *config.RequestConfig) Responses {
if !checkConnection() {
dodosCountForRequest, dodosCountForProxies := conf.DodosCount, conf.DodosCount utils.PrintAndExit("No internet connection")
if dodosCountForRequest > conf.RequestCount {
dodosCountForRequest = conf.RequestCount
} }
proxiesCount := len(conf.Proxies) clientFunc := getClientFunc(
if dodosCountForProxies > proxiesCount { requestConfig.Timeout,
dodosCountForProxies = proxiesCount requestConfig.Proxies,
} requestConfig.GetValidDodosCountForRequests(),
dodosCountForProxies = min(dodosCountForProxies, config.MaxDodosCountForProxies) requestConfig.URL,
)
var wg sync.WaitGroup request := newRequest(
wg.Add(dodosCountForRequest + 1) requestConfig.URL,
var requestCountPerDodo int requestConfig.Headers,
responses := make([][]DodoResponse, dodosCountForRequest) requestConfig.Cookies,
getClient := getClientFunc(conf.Proxies, conf.Timeout, dodosCountForProxies) requestConfig.Params,
requestConfig.Method,
requestConfig.Body,
)
defer fasthttp.ReleaseRequest(request)
responses := releaseDodos(
request,
requestConfig.Timeout,
clientFunc,
requestConfig.GetValidDodosCountForRequests(),
requestConfig.RequestCount,
)
countSlice := make([]int, dodosCountForRequest) return responses
go printProgress(&wg, conf.RequestCount, "Dodos Working🔥", &countSlice) }
for i := 0; i < dodosCountForRequest; i++ { // releaseDodos sends multiple HTTP requests concurrently using multiple "dodos" (goroutines).
if i+1 == dodosCountForRequest { // It takes a mainRequest as the base request, timeout duration for each request, clientFunc for customizing the client behavior,
requestCountPerDodo = conf.RequestCount - // dodosCount as the number of goroutines to be used, and requestCount as the total number of requests to be sent.
(i * conf.RequestCount / dodosCountForRequest) // It returns the responses received from all the requests.
func releaseDodos(
mainRequest *fasthttp.Request,
timeout time.Duration,
clientFunc ClientFunc,
dodosCount int,
requestCount int,
) Responses {
var (
wg sync.WaitGroup
requestCountPerDodo int
)
wg.Add(dodosCount + 1) // +1 for progress tracker
responses := make([][]Response, dodosCount)
countSlice := make([]int, dodosCount)
go streamProgress(&wg, requestCount, "Dodos Working🔥", &countSlice)
for i := 0; i < dodosCount; i++ {
if i+1 == dodosCount {
requestCountPerDodo = requestCount -
(i * requestCount / dodosCount)
} else { } else {
requestCountPerDodo = ((i + 1) * conf.RequestCount / dodosCountForRequest) - requestCountPerDodo = ((i + 1) * requestCount / dodosCount) -
(i * conf.RequestCount / dodosCountForRequest) (i * requestCount / dodosCount)
} }
dodoSpecificRequest := &fasthttp.Request{}
mainRequest.CopyTo(dodoSpecificRequest)
go sendRequest( go sendRequest(
dodoSpecificRequest,
timeout,
&responses[i], &responses[i],
&countSlice[i], &countSlice[i],
requestCountPerDodo, requestCountPerDodo,
conf.Method, clientFunc,
params,
conf.Body,
headers,
conf.Cookies,
getClient,
&wg, &wg,
) )
} }
wg.Wait() wg.Wait()
return utils.Flatten(responses), nil return utils.Flatten(responses)
} }
// sendRequest sends multiple requests concurrently using the provided parameters.
// It releases the request and response object and marks the completion of the wait group after each request.
// For each request, it acquires a response object, gets a client, and measures the time taken to complete the request.
// If an error occurs during the request, the error is recorded in the responseData slice.
func sendRequest( func sendRequest(
responseData *[]DodoResponse, request *fasthttp.Request,
timeout time.Duration,
responseData *[]Response,
counter *int, counter *int,
requestCout int, requestCount int,
method string, getClient ClientFunc,
params string,
body string,
headers http.Header,
cookies map[string]string,
getClient func() http.Client,
wg *sync.WaitGroup, wg *sync.WaitGroup,
) { ) {
defer fasthttp.ReleaseRequest(request)
defer wg.Done() defer wg.Done()
for j := 0; j < requestCout; j++ {
for range requestCount {
func() { func() {
defer func() { *counter++ }() defer func() { *counter++ }()
req, _ := http.NewRequest(
method, response := fasthttp.AcquireResponse()
params, defer fasthttp.ReleaseResponse(response)
getBodyReader(body),
)
req.Header = headers
setCookies(req, cookies)
client := getClient() client := getClient()
startTime := time.Now() startTime := time.Now()
resp, err := client.Do(req) err := client.DoTimeout(request, response, timeout)
completedTime := time.Since(startTime) completedTime := time.Since(startTime)
if err != nil { if err != nil {
*responseData = append( *responseData = append(*responseData, Response{
*responseData, StatusCode: 0,
DodoResponse{ Error: err,
Response: customerrors.RequestErrorsFormater(err), Time: completedTime,
Time: completedTime, })
},
)
return return
} }
defer resp.Body.Close()
*responseData = append( *responseData = append(*responseData, Response{
*responseData, StatusCode: response.StatusCode(),
DodoResponse{ Error: nil,
Response: resp.Status, Time: completedTime,
Time: completedTime, })
},
)
}() }()
} }
} }
func setCookies(req *http.Request, cookies map[string]string) { // getClientFunc returns a ClientFunc based on the provided parameters.
for key, value := range cookies { // If there are proxies available, it checks for active proxies and prompts the user to continue.
req.AddCookie(&http.Cookie{Name: key, Value: value}) // If there are no active proxies, it asks the user if they want to continue.
// If the user chooses to continue, it returns a ClientFunc with a shared client or a randomized client.
// If there are no proxies available, it returns a ClientFunc with a shared client.
func getClientFunc(
timeout time.Duration,
proxies []config.Proxy,
dodosCount int,
URL *url.URL,
) ClientFunc {
isTLS := URL.Scheme == "https"
if len(proxies) > 0 {
activeProxyClients := getActiveProxyClients(
proxies, timeout, dodosCount, URL,
)
activeProxyClientsCount := len(activeProxyClients)
var yesOrNoMessage string
if activeProxyClientsCount == 0 {
yesOrNoMessage = utils.Colored(
utils.Colors.Red,
"No active proxies found. Do you want to continue?",
)
} else {
yesOrNoMessage = utils.Colored(
utils.Colors.Yellow,
fmt.Sprintf(
"Found %d active proxies. Do you want to continue?",
activeProxyClientsCount,
),
)
}
fmt.Println()
proceed := readers.CLIYesOrNoReader(yesOrNoMessage)
if !proceed {
utils.PrintAndExit("Exiting...")
}
fmt.Println()
if activeProxyClientsCount == 0 {
client := &fasthttp.HostClient{
IsTLS: isTLS,
Addr: URL.Host,
MaxIdleConnDuration: timeout,
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
}
return getSharedClientFunc(client)
} else if activeProxyClientsCount == 1 {
client := &activeProxyClients[0]
return getSharedClientFunc(client)
}
return getRandomizedClientFunc(activeProxyClients, activeProxyClientsCount)
}
client := &fasthttp.HostClient{
IsTLS: isTLS,
Addr: URL.Host,
MaxIdleConnDuration: timeout,
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
}
return getSharedClientFunc(client)
}
// getActiveProxyClients divides the proxies into slices based on the number of dodos and
// launches goroutines to find active proxy clients for each slice.
// It uses a progress tracker to monitor the progress of the search.
// Once all goroutines have completed, the function waits for them to finish and
// returns a flattened slice of active proxy clients.
func getActiveProxyClients(
proxies []config.Proxy,
timeout time.Duration,
dodosCount int,
URL *url.URL,
) []fasthttp.HostClient {
activeProxyClientsArray := make([][]fasthttp.HostClient, dodosCount)
proxiesCount := len(proxies)
var wg sync.WaitGroup
wg.Add(dodosCount + 1) // +1 for progress tracker
var proxiesSlice []config.Proxy
countSlice := make([]int, dodosCount)
go streamProgress(&wg, proxiesCount, "Searching for active proxies🌐", &countSlice)
for i := 0; i < dodosCount; i++ {
if i+1 == dodosCount {
proxiesSlice = proxies[i*proxiesCount/dodosCount:]
} else {
proxiesSlice = proxies[i*proxiesCount/dodosCount : (i+1)*proxiesCount/dodosCount]
}
go findActiveProxyClients(
proxiesSlice,
timeout,
&activeProxyClientsArray[i],
&countSlice[i],
URL,
&wg,
)
}
wg.Wait()
return utils.Flatten(activeProxyClientsArray)
}
// findActiveProxyClients finds the active proxy clients by sending a GET request to each proxy in the given slice.
// The function runs each request in a separate goroutine and updates the activeProxyClients slice with the active proxy clients.
// It also increments the count for each successful request.
// The function is designed to be used as a concurrent operation, and it uses the WaitGroup to wait for all goroutines to finish.
func findActiveProxyClients(
proxies []config.Proxy,
timeout time.Duration,
activeProxyClients *[]fasthttp.HostClient,
count *int,
URL *url.URL,
wg *sync.WaitGroup,
) {
defer wg.Done()
request := fasthttp.AcquireRequest()
defer fasthttp.ReleaseRequest(request)
request.SetRequestURI(config.ProxyCheckURL)
request.Header.SetMethod("GET")
for _, proxy := range proxies {
func() {
defer func() { *count++ }()
response := fasthttp.AcquireResponse()
defer fasthttp.ReleaseResponse(response)
dialFunc, err := getDialFunc(&proxy, timeout)
if err != nil {
return
}
client := &fasthttp.Client{
Dial: dialFunc,
}
err = client.DoTimeout(request, response, timeout)
if err != nil {
return
}
if response.StatusCode() == 200 {
*activeProxyClients = append(
*activeProxyClients,
fasthttp.HostClient{
IsTLS: URL.Scheme == "https",
Addr: URL.Host + ":443",
Dial: dialFunc,
MaxIdleConnDuration: timeout,
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
},
)
}
}()
} }
} }
func getHeaders(headers map[string]string) http.Header { // getDialFunc returns a fasthttp.DialFunc based on the provided proxy configuration.
httpHeaders := make(http.Header, len(headers)) // It takes a pointer to a config.Proxy struct as input and returns a fasthttp.DialFunc and an error.
httpHeaders.Set("User-Agent", config.DefaultUserAgent) // The function parses the proxy URL, determines the scheme (socks5, socks5h, http, or https),
// and creates a dialer accordingly. If the proxy URL is invalid or the scheme is not supported,
// it returns an error.
func getDialFunc(proxy *config.Proxy, timeout time.Duration) (fasthttp.DialFunc, error) {
parsedProxyURL, err := url.Parse(proxy.URL)
if err != nil {
return nil, err
}
var dialer fasthttp.DialFunc
if parsedProxyURL.Scheme == "socks5" || parsedProxyURL.Scheme == "socks5h" {
if proxy.Username != "" {
dialer = fasthttpproxy.FasthttpSocksDialer(
fmt.Sprintf(
"%s://%s:%s@%s",
parsedProxyURL.Scheme,
proxy.Username,
proxy.Password,
parsedProxyURL.Host,
),
)
} else {
dialer = fasthttpproxy.FasthttpSocksDialer(
fmt.Sprintf(
"%s://%s",
parsedProxyURL.Scheme,
parsedProxyURL.Host,
),
)
}
} else if parsedProxyURL.Scheme == "http" {
if proxy.Username != "" {
dialer = fasthttpproxy.FasthttpHTTPDialerTimeout(
fmt.Sprintf(
"%s:%s@%s",
proxy.Username, proxy.Password, parsedProxyURL.Host,
),
timeout,
)
} else {
dialer = fasthttpproxy.FasthttpHTTPDialerTimeout(
parsedProxyURL.Host,
timeout,
)
}
} else {
return nil, err
}
return dialer, nil
}
// getRandomizedClientFunc returns a ClientFunc that randomly selects a HostClient from the given list of clients.
// The clientsCount parameter specifies the number of clients in the slice.
// The returned ClientFunc can be used to share the same client instance across multiple goroutines.
func getRandomizedClientFunc(
clients []fasthttp.HostClient,
clientsCount int,
) ClientFunc {
return func() *fasthttp.HostClient {
return &clients[rand.Intn(clientsCount)]
}
}
// getSharedClientFunc returns a ClientFunc that returns the provided client.
// The returned ClientFunc can be used to share the same client instance across multiple goroutines.
func getSharedClientFunc(client *fasthttp.HostClient) ClientFunc {
return func() *fasthttp.HostClient {
return client
}
}
// newRequest creates a new fasthttp.Request object with the provided parameters.
// It sets the request URI, host header, headers, cookies, params, method, and body.
func newRequest(
URL *url.URL,
Headers map[string]string,
Cookies map[string]string,
Params map[string]string,
Method string,
Body string,
) *fasthttp.Request {
request := fasthttp.AcquireRequest()
request.SetRequestURI(URL.Path)
// Set the host of the request to the host header
// If the host header is not set, the request will fail
// If there is host header in the headers, it will be overwritten
request.Header.Set("Host", URL.Host)
setRequestHeaders(request, Headers)
setRequestCookies(request, Cookies)
setRequestParams(request, Params)
setRequestMethod(request, Method)
setRequestBody(request, Body)
if URL.Scheme == "https" {
request.URI().SetScheme("https")
}
return request
}
// setRequestHeaders sets the headers of the given request with the provided key-value pairs.
func setRequestHeaders(req *fasthttp.Request, headers map[string]string) {
for key, value := range headers { for key, value := range headers {
httpHeaders.Add(key, value) req.Header.Set(key, value)
} }
return httpHeaders
} }
func getBodyReader(bodyString string) io.Reader { // setRequestCookies sets the cookies in the given request.
if bodyString == "" { func setRequestCookies(req *fasthttp.Request, cookies map[string]string) {
return http.NoBody for key, value := range cookies {
req.Header.SetCookie(key, value)
} }
return strings.NewReader(bodyString)
} }
func setParams(baseURL string, params map[string]string) string { // setRequestParams sets the query parameters of the given request based on the provided map of key-value pairs.
if len(params) == 0 { func setRequestParams(req *fasthttp.Request, params map[string]string) {
return baseURL
}
urlParams := url.Values{} urlParams := url.Values{}
for key, value := range params { for key, value := range params {
urlParams.Add(key, value) urlParams.Add(key, value)
} }
baseURLWithParams := fmt.Sprintf("%s?%s", baseURL, urlParams.Encode()) req.URI().SetQueryString(urlParams.Encode())
return baseURLWithParams
} }
func printProgress(wg *sync.WaitGroup, total int, message string, countSlice *[]int) { // setRequestMethod sets the HTTP request method for the given request.
func setRequestMethod(req *fasthttp.Request, method string) {
req.Header.SetMethod(method)
}
// setRequestBody sets the request body of the given fasthttp.Request object.
// The body parameter is a string that will be converted to a byte slice and set as the request body.
func setRequestBody(req *fasthttp.Request, body string) {
req.SetBody([]byte(body))
}
// streamProgress displays the progress of a stream operation.
// It takes a wait group, the total number of items to process, a message to display,
// and a pointer to a slice of counts for each item processed.
// The function runs in a separate goroutine and updates the progress bar until all items are processed.
// Once all items are processed, it marks the progress bar as done and stops rendering.
func streamProgress(
wg *sync.WaitGroup,
total int,
message string,
countSlice *[]int,
) {
defer wg.Done() defer wg.Done()
pw := progress.NewWriter() pw := progress.NewWriter()
pw.SetTrackerPosition(progress.PositionRight) pw.SetTrackerPosition(progress.PositionRight)
@ -250,7 +531,10 @@ func printProgress(wg *sync.WaitGroup, total int, message string, countSlice *[]
pw.SetTrackerLength(40) pw.SetTrackerLength(40)
pw.SetUpdateFrequency(time.Millisecond * 250) pw.SetUpdateFrequency(time.Millisecond * 250)
go pw.Render() go pw.Render()
dodosTracker := progress.Tracker{Message: message, Total: int64(total)} dodosTracker := progress.Tracker{
Message: message,
Total: int64(total),
}
pw.AppendTracker(&dodosTracker) pw.AppendTracker(&dodosTracker)
for { for {
totalCount := 0 totalCount := 0
@ -268,132 +552,16 @@ func printProgress(wg *sync.WaitGroup, total int, message string, countSlice *[]
pw.Stop() pw.Stop()
} }
func getClientFunc(proxies []config.Proxy, timeout time.Duration, dodosCount int) func() http.Client { // checkConnection checks the internet connection by making requests to different websites.
if len(proxies) > 0 { // It returns true if the connection is successful, otherwise false.
activeProxyClientsArray := make([][]http.Client, dodosCount) func checkConnection() bool {
proxiesCount := len(proxies) _, _, err := fasthttp.Get(nil, "https://www.google.com")
var wg sync.WaitGroup
wg.Add(dodosCount + 1)
var proxiesSlice []config.Proxy
countSlice := make([]int, dodosCount)
go printProgress(&wg, proxiesCount, "Searching for active proxies🌐", &countSlice)
for i := 0; i < dodosCount; i++ {
if i+1 == dodosCount {
proxiesSlice = proxies[i*proxiesCount/dodosCount:]
} else {
proxiesSlice = proxies[i*proxiesCount/dodosCount : (i+1)*proxiesCount/dodosCount]
}
go findActiveProxyClients(
proxiesSlice,
timeout,
&activeProxyClientsArray[i],
&countSlice[i],
&wg,
)
}
wg.Wait()
activeProxyClients := utils.Flatten(activeProxyClientsArray)
activeProxyClientsCount := len(activeProxyClients)
var yesOrNoMessage string
if activeProxyClientsCount == 0 {
yesOrNoMessage = utils.Colored(
utils.Colors.Red,
"No active proxies found. Do you want to continue?",
)
} else {
yesOrNoMessage = utils.Colored(
utils.Colors.Yellow,
fmt.Sprintf("Found %d active proxies. Do you want to continue?", activeProxyClientsCount),
)
}
fmt.Println()
proceed := readers.CLIYesOrNoReader(yesOrNoMessage)
if !proceed {
utils.PrintAndExit("Exiting...")
}
fmt.Println()
if activeProxyClientsCount == 0 {
return func() http.Client {
return getNewClient(timeout)
}
}
return func() http.Client {
return getRandomClient(activeProxyClients, activeProxyClientsCount)
}
}
return func() http.Client {
return getNewClient(timeout)
}
}
func findActiveProxyClients(
proxies []config.Proxy,
timeout time.Duration,
activeProxyClients *[]http.Client,
counter *int,
wg *sync.WaitGroup) {
defer wg.Done()
for _, proxy := range proxies {
func() {
defer func() { *counter++ }()
transport, err := getTransport(proxy)
if err != nil {
return
}
client := &http.Client{
Transport: transport,
Timeout: timeout,
}
resp, err := client.Get(config.ProxyCheckURL)
if err != nil {
return
}
defer resp.Body.Close()
if resp.StatusCode == 200 {
*activeProxyClients = append(
*activeProxyClients,
http.Client{
Transport: transport,
Timeout: timeout,
},
)
}
}()
}
}
func getTransport(proxy config.Proxy) (*http.Transport, error) {
proxyURL, err := url.Parse(proxy.URL)
if err != nil { if err != nil {
return nil, err _, _, err = fasthttp.Get(nil, "https://www.bing.com")
} if err != nil {
if proxy.Username != "" { _, _, err = fasthttp.Get(nil, "https://www.yahoo.com")
transport := &http.Transport{ return err == nil
Proxy: http.ProxyURL(proxyURL),
} }
return transport, nil
} }
return true
transport := &http.Transport{
Proxy: http.ProxyURL(
&url.URL{
Scheme: proxyURL.Scheme,
Host: proxyURL.Host,
User: url.UserPassword(proxy.Username, proxy.Password),
},
),
}
return transport, nil
}
func getRandomClient(clients []http.Client, clientsCount int) http.Client {
randomIndex := rand.Intn(clientsCount)
return clients[randomIndex]
}
func getNewClient(timeout time.Duration) http.Client {
return http.Client{Timeout: timeout}
} }

31
utils/time.go Normal file
View File

@ -0,0 +1,31 @@
package utils
import "time"
func MinDuration(durations ...time.Duration) time.Duration {
min := durations[0]
for _, d := range durations {
if d < min {
min = d
}
}
return min
}
func MaxDuration(durations ...time.Duration) time.Duration {
max := durations[0]
for _, d := range durations {
if d > max {
max = d
}
}
return max
}
func AvgDuration(durations ...time.Duration) time.Duration {
total := time.Duration(0)
for _, d := range durations {
total += d
}
return total / time.Duration(len(durations))
}