Merge pull request #181 from aykhans/perf/reduce-request-memory

perf: reduce memory allocations in request generator

.github/FUNDING.yml

@@ -1,2 +1 @@
 buy_me_a_coffee: aykhan
-custom: https://commerce.coinbase.com/checkout/0f33d2fb-54a6-44f5-8783-006ebf70d1a0

.github/workflows/lint.yaml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v5
       - uses: actions/setup-go@v6
         with:
-          go-version: 1.26.0
+          go-version: 1.26.1
       - name: go fix
         run: |
           go fix ./...
@@ -24,4 +24,4 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v9
         with:
-          version: v2.9.0
+          version: v2.11.2

.github/workflows/release.yaml

@@ -35,7 +35,7 @@ jobs:
         run: |
           echo "VERSION=$(git describe --tags --always)" >> $GITHUB_ENV
           echo "GIT_COMMIT=$(git rev-parse HEAD)" >> $GITHUB_ENV
-          echo "GO_VERSION=1.26.0" >> $GITHUB_ENV
+          echo "GO_VERSION=1.26.1" >> $GITHUB_ENV
 
       - name: Set up Go
         if: github.event_name == 'release' || inputs.build_binaries

Dockerfile

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.26.0
+ARG GO_VERSION=1.26.1
 
 FROM docker.io/library/golang:${GO_VERSION}-alpine AS builder
 

Taskfile.yaml

@@ -3,7 +3,7 @@ version: "3"
 
 vars:
   BIN_DIR: ./bin
-  GOLANGCI_LINT_VERSION: v2.9.0
+  GOLANGCI_LINT_VERSION: v2.11.2
   GOLANGCI: "{{.BIN_DIR}}/golangci-lint-{{.GOLANGCI_LINT_VERSION}}"
 
 tasks:

cmd/cli/main.go

@@ -14,7 +14,7 @@ import (
 )
 
 func main() {
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec // G118: cancel is called in listenForTermination goroutine
 	go listenForTermination(func() { cancel() })
 
 	combinedConfig := config.ReadAllConfigs()

go.mod

@@ -1,6 +1,6 @@
 module go.aykhans.me/sarin
 
-go 1.26.0
+go 1.26.1
 
 require (
 	github.com/brianvoe/gofakeit/v7 v7.14.1
@@ -15,7 +15,7 @@ require (
 	github.com/yuin/gopher-lua v1.1.1
 	go.aykhans.me/utils v1.0.7
 	go.yaml.in/yaml/v4 v4.0.0-rc.3
-	golang.org/x/net v0.51.0
+	golang.org/x/net v0.52.0
 )
 
 require (
@@ -53,7 +53,7 @@ require (
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/yuin/goldmark v1.7.16 // indirect
 	github.com/yuin/goldmark-emoji v1.0.6 // indirect
-	golang.org/x/sys v0.41.0 // indirect
-	golang.org/x/term v0.40.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
 )

go.sum

@@ -111,16 +111,16 @@ go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
 go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
-golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
-golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
-golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
-golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
+golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/config/config.go

@@ -418,7 +418,7 @@ func (config Config) Validate() error {
 		validationErrors = append(validationErrors, types.NewFieldValidationError("Duration", "0", errors.New("duration must be greater than 0")))
 	}
 
-	if *config.Timeout < 1 {
+	if config.Timeout == nil || *config.Timeout < 1 {
 		validationErrors = append(validationErrors, types.NewFieldValidationError("Timeout", "0", errors.New("timeout must be greater than 0")))
 	}
 

internal/config/env.go

@@ -157,7 +157,7 @@ func (parser ConfigENVParser) Parse() (*Config, error) {
 				types.NewFieldParseError(
 					parser.getFullEnvName("DURATION"),
 					duration,
-					errors.New("invalid value duration, expected a duration string (e.g., '10s', '1h30m')"),
+					errors.New("invalid value for duration, expected a duration string (e.g., '10s', '1h30m')"),
 				),
 			)
 		} else {
@@ -173,7 +173,7 @@ func (parser ConfigENVParser) Parse() (*Config, error) {
 				types.NewFieldParseError(
 					parser.getFullEnvName("TIMEOUT"),
 					timeout,
-					errors.New("invalid value duration, expected a duration string (e.g., '10s', '1h30m')"),
+					errors.New("invalid value for duration, expected a duration string (e.g., '10s', '1h30m')"),
 				),
 			)
 		} else {

internal/sarin/client.go

@@ -172,7 +172,6 @@ func fasthttpSocksDialerDualStackTimeout(ctx context.Context, proxyURL *url.URL,
 				return nil, types.NewProxyDialError(proxyStr, err)
 			}
 
-			// Cap DNS resolution to half the timeout to reserve time for dial
 			dnsCtx, dnsCancel := context.WithTimeout(ctx, timeout)
 			ips, err := net.DefaultResolver.LookupIP(dnsCtx, "ip", host)
 			dnsCancel()
@@ -244,7 +243,7 @@ func fasthttpHTTPSDialerDualStackTimeout(proxyURL *url.URL, timeout time.Duratio
 		}
 
 		// Upgrade to TLS
-		tlsConn := tls.Client(conn, &tls.Config{ //nolint:gosec
+		tlsConn := tls.Client(conn, &tls.Config{
 			ServerName: proxyURL.Hostname(),
 		})
 		if err := tlsConn.Handshake(); err != nil {

internal/sarin/helpers.go

@@ -8,7 +8,7 @@ import (
 func NewDefaultRandSource() rand.Source {
 	now := time.Now().UnixNano()
 	return rand.NewPCG(
-		uint64(now),     //nolint:gosec // G115: Safe conversion; UnixNano timestamp used as random seed, bit pattern is intentional
-		uint64(now>>32), //nolint:gosec // G115: Safe conversion; right-shifted timestamp for seed entropy, overflow is acceptable
+		uint64(now),
+		uint64(now>>32),
 	)
 }

internal/sarin/request.go

@@ -43,19 +43,34 @@ func NewRequestGenerator(
 	randSource := NewDefaultRandSource()
 	//nolint:gosec // G404: Using non-cryptographic rand for load testing, not security
 	localRand := rand.New(randSource)
-	templateFuncMap := NewDefaultTemplateFuncMap(randSource, fileCache)
 
-	pathGenerator, isPathGeneratorDynamic := createTemplateFunc(requestURL.Path, templateFuncMap)
-	methodGenerator, isMethodGeneratorDynamic := NewMethodGeneratorFunc(localRand, methods, templateFuncMap)
-	paramsGenerator, isParamsGeneratorDynamic := NewParamsGeneratorFunc(localRand, params, templateFuncMap)
-	headersGenerator, isHeadersGeneratorDynamic := NewHeadersGeneratorFunc(localRand, headers, templateFuncMap)
-	cookiesGenerator, isCookiesGeneratorDynamic := NewCookiesGeneratorFunc(localRand, cookies, templateFuncMap)
+	// Funcs() is only called if a value actually contains template syntax.
+	// The root template is shared across all createTemplateFunc calls so Funcs() is called at most once.
+	var templateRoot *template.Template
+	lazyTemplateRoot := func() *template.Template {
+		if templateRoot == nil {
+			templateRoot = template.New("").Funcs(NewDefaultTemplateFuncMap(randSource, fileCache))
+		}
+		return templateRoot
+	}
+
+	pathGenerator, isPathGeneratorDynamic := createTemplateFunc(requestURL.Path, lazyTemplateRoot)
+	methodGenerator, isMethodGeneratorDynamic := NewMethodGeneratorFunc(localRand, methods, lazyTemplateRoot)
+	paramsGenerator, isParamsGeneratorDynamic := NewParamsGeneratorFunc(localRand, params, lazyTemplateRoot)
+	headersGenerator, isHeadersGeneratorDynamic := NewHeadersGeneratorFunc(localRand, headers, lazyTemplateRoot)
+	cookiesGenerator, isCookiesGeneratorDynamic := NewCookiesGeneratorFunc(localRand, cookies, lazyTemplateRoot)
 
 	bodyTemplateFuncMapData := &BodyTemplateFuncMapData{}
-	bodyTemplateFuncMap := NewDefaultBodyTemplateFuncMap(randSource, bodyTemplateFuncMapData, fileCache)
-	bodyGenerator, isBodyGeneratorDynamic := NewBodyGeneratorFunc(localRand, bodies, bodyTemplateFuncMap)
+	var bodyTemplateRoot *template.Template
+	lazyBodyTemplateRoot := func() *template.Template {
+		if bodyTemplateRoot == nil {
+			bodyTemplateRoot = template.New("").Funcs(NewDefaultBodyTemplateFuncMap(randSource, bodyTemplateFuncMapData, fileCache))
+		}
+		return bodyTemplateRoot
+	}
+	bodyGenerator, isBodyGeneratorDynamic := NewBodyGeneratorFunc(localRand, bodies, lazyBodyTemplateRoot)
 
-	valuesGenerator := NewValuesGeneratorFunc(values, templateFuncMap)
+	valuesGenerator := NewValuesGeneratorFunc(values, lazyTemplateRoot)
 
 	hasScripts := scriptTransformer != nil && !scriptTransformer.IsEmpty()
 
@@ -91,7 +106,7 @@ func NewRequestGenerator(
 				return err
 			}
 
-			bodyTemplateFuncMapData.ClearFormDataContenType()
+			bodyTemplateFuncMapData.ClearFormDataContentType()
 			if err = bodyGenerator(reqData, data); err != nil {
 				return err
 			}
@@ -99,8 +114,8 @@ func NewRequestGenerator(
 			if err = headersGenerator(reqData, data); err != nil {
 				return err
 			}
-			if bodyTemplateFuncMapData.GetFormDataContenType() != "" {
-				reqData.Headers["Content-Type"] = append(reqData.Headers["Content-Type"], bodyTemplateFuncMapData.GetFormDataContenType())
+			if bodyTemplateFuncMapData.GetFormDataContentType() != "" {
+				reqData.Headers["Content-Type"] = append(reqData.Headers["Content-Type"], bodyTemplateFuncMapData.GetFormDataContentType())
 			}
 
 			if err = paramsGenerator(reqData, data); err != nil {
@@ -170,8 +185,8 @@ func applyRequestDataToFastHTTP(reqData *script.RequestData, req *fasthttp.Reque
 	}
 }
 
-func NewMethodGeneratorFunc(localRand *rand.Rand, methods []string, templateFunctions template.FuncMap) (requestDataGenerator, bool) {
-	methodGenerator, isDynamic := buildStringSliceGenerator(localRand, methods, templateFunctions)
+func NewMethodGeneratorFunc(localRand *rand.Rand, methods []string, lazyRoot func() *template.Template) (requestDataGenerator, bool) {
+	methodGenerator, isDynamic := buildStringSliceGenerator(localRand, methods, lazyRoot)
 
 	var (
 		method string
@@ -188,8 +203,8 @@ func NewMethodGeneratorFunc(localRand *rand.Rand, methods []string, templateFunc
 	}, isDynamic
 }
 
-func NewBodyGeneratorFunc(localRand *rand.Rand, bodies []string, templateFunctions template.FuncMap) (requestDataGenerator, bool) {
-	bodyGenerator, isDynamic := buildStringSliceGenerator(localRand, bodies, templateFunctions)
+func NewBodyGeneratorFunc(localRand *rand.Rand, bodies []string, lazyRoot func() *template.Template) (requestDataGenerator, bool) {
+	bodyGenerator, isDynamic := buildStringSliceGenerator(localRand, bodies, lazyRoot)
 
 	var (
 		body string
@@ -206,8 +221,8 @@ func NewBodyGeneratorFunc(localRand *rand.Rand, bodies []string, templateFunctio
 	}, isDynamic
 }
 
-func NewParamsGeneratorFunc(localRand *rand.Rand, params types.Params, templateFunctions template.FuncMap) (requestDataGenerator, bool) {
-	generators, isDynamic := buildKeyValueGenerators(localRand, params, templateFunctions)
+func NewParamsGeneratorFunc(localRand *rand.Rand, params types.Params, lazyRoot func() *template.Template) (requestDataGenerator, bool) {
+	generators, isDynamic := buildKeyValueGenerators(localRand, params, lazyRoot)
 
 	var (
 		key, value string
@@ -231,8 +246,8 @@ func NewParamsGeneratorFunc(localRand *rand.Rand, params types.Params, templateF
 	}, isDynamic
 }
 
-func NewHeadersGeneratorFunc(localRand *rand.Rand, headers types.Headers, templateFunctions template.FuncMap) (requestDataGenerator, bool) {
-	generators, isDynamic := buildKeyValueGenerators(localRand, headers, templateFunctions)
+func NewHeadersGeneratorFunc(localRand *rand.Rand, headers types.Headers, lazyRoot func() *template.Template) (requestDataGenerator, bool) {
+	generators, isDynamic := buildKeyValueGenerators(localRand, headers, lazyRoot)
 
 	var (
 		key, value string
@@ -256,8 +271,8 @@ func NewHeadersGeneratorFunc(localRand *rand.Rand, headers types.Headers, templa
 	}, isDynamic
 }
 
-func NewCookiesGeneratorFunc(localRand *rand.Rand, cookies types.Cookies, templateFunctions template.FuncMap) (requestDataGenerator, bool) {
-	generators, isDynamic := buildKeyValueGenerators(localRand, cookies, templateFunctions)
+func NewCookiesGeneratorFunc(localRand *rand.Rand, cookies types.Cookies, lazyRoot func() *template.Template) (requestDataGenerator, bool) {
+	generators, isDynamic := buildKeyValueGenerators(localRand, cookies, lazyRoot)
 
 	var (
 		key, value string
@@ -281,11 +296,11 @@ func NewCookiesGeneratorFunc(localRand *rand.Rand, cookies types.Cookies, templa
 	}, isDynamic
 }
 
-func NewValuesGeneratorFunc(values []string, templateFunctions template.FuncMap) func() (valuesData, error) {
+func NewValuesGeneratorFunc(values []string, lazyRoot func() *template.Template) func() (valuesData, error) {
 	generators := make([]func(_ any) (string, error), len(values))
 
 	for i, v := range values {
-		generators[i], _ = createTemplateFunc(v, templateFunctions)
+		generators[i], _ = createTemplateFunc(v, lazyRoot)
 	}
 
 	var (
@@ -313,8 +328,12 @@ func NewValuesGeneratorFunc(values []string, templateFunctions template.FuncMap)
 	}
 }
 
-func createTemplateFunc(value string, templateFunctions template.FuncMap) (func(data any) (string, error), bool) {
-	tmpl, err := template.New("").Funcs(templateFunctions).Parse(value)
+func createTemplateFunc(value string, lazyRoot func() *template.Template) (func(data any) (string, error), bool) {
+	if !strings.Contains(value, "{{") {
+		return func(_ any) (string, error) { return value, nil }, false
+	}
+
+	tmpl, err := lazyRoot().New("").Parse(value)
 	if err == nil && hasTemplateActions(tmpl) {
 		var err error
 		return func(data any) (string, error) {
@@ -340,7 +359,7 @@ type keyValueItem interface {
 func buildKeyValueGenerators[T keyValueItem](
 	localRand *rand.Rand,
 	items []T,
-	templateFunctions template.FuncMap,
+	lazyRoot func() *template.Template,
 ) ([]keyValueGenerator, bool) {
 	isDynamic := false
 	generators := make([]keyValueGenerator, len(items))
@@ -350,7 +369,7 @@ func buildKeyValueGenerators[T keyValueItem](
 		keyValue := types.KeyValue[string, []string](item)
 
 		// Generate key function
-		keyFunc, keyIsDynamic := createTemplateFunc(keyValue.Key, templateFunctions)
+		keyFunc, keyIsDynamic := createTemplateFunc(keyValue.Key, lazyRoot)
 		if keyIsDynamic {
 			isDynamic = true
 		}
@@ -358,7 +377,7 @@ func buildKeyValueGenerators[T keyValueItem](
 		// Generate value functions
 		valueFuncs := make([]func(data any) (string, error), len(keyValue.Value))
 		for j, v := range keyValue.Value {
-			valueFunc, valueIsDynamic := createTemplateFunc(v, templateFunctions)
+			valueFunc, valueIsDynamic := createTemplateFunc(v, lazyRoot)
 			if valueIsDynamic {
 				isDynamic = true
 			}
@@ -381,7 +400,7 @@ func buildKeyValueGenerators[T keyValueItem](
 func buildStringSliceGenerator(
 	localRand *rand.Rand,
 	values []string,
-	templateFunctions template.FuncMap,
+	lazyRoot func() *template.Template,
 ) (func() func(data any) (string, error), bool) {
 	// Return a function that returns an empty string generator if values is empty
 	if len(values) == 0 {
@@ -393,7 +412,7 @@ func buildStringSliceGenerator(
 	valueFuncs := make([]func(data any) (string, error), len(values))
 
 	for i, value := range values {
-		valueFunc, valueIsDynamic := createTemplateFunc(value, templateFunctions)
+		valueFunc, valueIsDynamic := createTemplateFunc(value, lazyRoot)
 		if valueIsDynamic {
 			isDynamic = true
 		}

internal/sarin/sarin.go

@@ -484,13 +484,11 @@ func newHostClients(
 		proxiesRaw[i] = url.URL(proxy)
 	}
 
-	maxConns := max(fasthttp.DefaultMaxConnsPerHost, workers)
-	maxConns = ((maxConns * 50 / 100) + maxConns)
 	return NewHostClients(
 		ctx,
 		timeout,
 		proxiesRaw,
-		maxConns,
+		workers,
 		requestURL,
 		skipCertVerify,
 	)

internal/sarin/template.go

@@ -286,7 +286,7 @@ func NewDefaultTemplateFuncMap(randSource rand.Source, fileCache *FileCache) tem
 		"fakeit_AdverbFrequencyDefinite":   fakeit.AdverbFrequencyDefinite,
 		"fakeit_AdverbFrequencyIndefinite": fakeit.AdverbFrequencyIndefinite,
 
-		// Propositions
+		// Prepositions
 		"fakeit_Preposition":         fakeit.Preposition,
 		"fakeit_PrepositionSimple":   fakeit.PrepositionSimple,
 		"fakeit_PrepositionDouble":   fakeit.PrepositionDouble,
@@ -589,15 +589,15 @@ func NewDefaultTemplateFuncMap(randSource rand.Source, fileCache *FileCache) tem
 }
 
 type BodyTemplateFuncMapData struct {
-	formDataContenType string
+	formDataContentType string
 }
 
-func (data BodyTemplateFuncMapData) GetFormDataContenType() string {
-	return data.formDataContenType
+func (data BodyTemplateFuncMapData) GetFormDataContentType() string {
+	return data.formDataContentType
 }
 
-func (data *BodyTemplateFuncMapData) ClearFormDataContenType() {
-	data.formDataContenType = ""
+func (data *BodyTemplateFuncMapData) ClearFormDataContentType() {
+	data.formDataContentType = ""
 }
 
 func NewDefaultBodyTemplateFuncMap(
@@ -628,7 +628,7 @@ func NewDefaultBodyTemplateFuncMap(
 
 			var multipartData bytes.Buffer
 			writer := multipart.NewWriter(&multipartData)
-			data.formDataContenType = writer.FormDataContentType()
+			data.formDataContentType = writer.FormDataContentType()
 
 			for i := 0; i < len(pairs); i += 2 {
 				key := pairs[i]