feat: limit response body sizes

fasthttp allocates the body from the declared Content-Length before
reading it, so a server could exhaust memory by declaring a size it
never sends. The new max response body config bounds every response,
defaults to 10MiB and accepts sizes like 10MiB or 2MB. Scripts get the
same as a maxBodySize option on http().
This commit is contained in:
aykhans committed 2026-09-28 22:49:39 +04:00
1 parent 412a02fea3
commit f1d91e3d77
17 files changed
+452 -147

No files matched your search

+123
View File
@@ -0,0 +1,123 @@
package config
import (
"flag"
"fmt"
"math"
"strconv"
"strings"
"go.aykhans.me/sarin/internal/types"
"go.yaml.in/yaml/v4"
)
var (
_ flag.Value = (*byteSize)(nil)
_ yaml.Unmarshaler = (*byteSize)(nil)
)
// byteSize is a size in bytes that reads and prints unit suffixes such as 10MiB.
type byteSize uint64
// byteSizeUnits run from the largest scale down, so formatting picks the largest unit
// that fits. A bare "K" is out: the common libraries disagree on whether it counts
// 1000 or 1024 bytes.
var byteSizeUnits = []struct {
suffix string
scale uint64
}{
{"TiB", 1 << 40},
{"TB", 1000 * 1000 * 1000 * 1000},
{"GiB", 1 << 30},
{"GB", 1000 * 1000 * 1000},
{"MiB", 1 << 20},
{"MB", 1000 * 1000},
{"KiB", 1 << 10},
{"KB", 1000},
{"B", 1},
}
// parseByteSize reads a byte count, with or without a unit suffix.
func parseByteSize(value string) (uint64, error) {
text := strings.TrimSpace(value)
// The suffix is split off by index, so no case mapping can shift the digits.
digitsEnd := len(text)
for digitsEnd > 0 && isASCIILetter(text[digitsEnd-1]) {
digitsEnd--
}
scale := uint64(1)
if suffix := text[digitsEnd:]; suffix != "" {
unitScale, ok := byteSizeScale(suffix)
if !ok {
return 0, types.ErrByteSizeInvalid
}
scale = unitScale
}
size, err := strconv.ParseUint(strings.TrimSpace(text[:digitsEnd]), 10, 64)
if err != nil {
return 0, types.ErrByteSizeInvalid
}
if size > math.MaxUint64/scale {
return 0, types.ErrByteSizeInvalid
}
return size * scale, nil
}
// byteSizeScale looks up a unit suffix, ignoring case.
func byteSizeScale(suffix string) (uint64, bool) {
for _, unit := range byteSizeUnits {
if strings.EqualFold(suffix, unit.suffix) {
return unit.scale, true
}
}
return 0, false
}
func isASCIILetter(char byte) bool {
return (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z')
}
// formatByteSize renders a byte count so that parseByteSize reads back the same value.
func formatByteSize(size uint64) string {
for _, unit := range byteSizeUnits {
if unit.scale > 1 && size >= unit.scale && size%unit.scale == 0 {
return strconv.FormatUint(size/unit.scale, 10) + unit.suffix
}
}
return strconv.FormatUint(size, 10)
}
func (size byteSize) String() string {
return formatByteSize(uint64(size))
}
// Set fills the size from a command line value.
func (size *byteSize) Set(value string) error {
parsed, err := parseByteSize(value)
if err != nil {
return err
}
*size = byteSize(parsed)
return nil
}
// UnmarshalYAML accepts both a YAML integer and a string with a unit.
func (size *byteSize) UnmarshalYAML(node *yaml.Node) error {
if node.Kind != yaml.ScalarNode {
return fmt.Errorf("expected a size, but got %s", node.Tag)
}
if node.Tag == "!!int" {
var parsed uint64
if err := node.Decode(&parsed); err != nil {
return types.ErrByteSizeInvalid
}
*size = byteSize(parsed)
return nil
}
return size.Set(node.Value)
}
+44 -36
View File
@@ -20,32 +20,33 @@ Simple usage:
Flags:
General Config:
-h, -help Help for sarin
-v, -version Version for sarin
-s, -show-config bool Show the final config after parsing all sources (default %v)
-f, -config-file []string Path to the config file (local file / http URL)
-c, -concurrency uint Number of concurrent requests (default %d)
-r, -requests uint Number of total requests
-d, -duration time Maximum duration for the test (e.g. 30s, 1m, 5h)
-l, -log-level string Runtime log levels to emit, comma-separated (possible values: info, error) (default %s)
-w, -log-file string Write runtime logs to this file instead of the terminal/stderr
-p, -progress string Progress display (possible values: bar, none) (default '%v')
-o, -output string Output format (possible values: table, json, yaml, none) (default '%v')
-z, -dry-run bool Run without sending requests (default %v)
-h, -help Help for sarin
-v, -version Version for sarin
-s, -show-config bool Show the final config after parsing all sources (default %v)
-f, -config-file []string Path to the config file (local file / http URL)
-c, -concurrency uint Number of concurrent requests (default %d)
-r, -requests uint Number of total requests
-d, -duration time Maximum duration for the test (e.g. 30s, 1m, 5h)
-l, -log-level string Runtime log levels to emit, comma-separated (possible values: info, error) (default %s)
-w, -log-file string Write runtime logs to this file instead of the terminal/stderr
-p, -progress string Progress display (possible values: bar, none) (default '%v')
-o, -output string Output format (possible values: table, json, yaml, none) (default '%v')
-z, -dry-run bool Run without sending requests (default %v)
Request Config:
-U, -url string Target URL for the request
-M, -method []string HTTP method for the request (default %s)
-B, -body []string Body for the request (e.g. "body text")
-P, -param []string URL parameter for the request (e.g. "key1=value1")
-H, -header []string Header for the request (e.g. "key1: value1")
-C, -cookie []string Cookie for the request (e.g. "key1=value1")
-X, -proxy []string Proxy for the request (e.g. "http://proxy.example.com:8080")
-V, -values []string List of values for templating (e.g. "key1=value1")
-T, -timeout time Timeout for the request (e.g. 400ms, 3s, 1m10s) (default %v)
-I, -insecure bool Skip SSL/TLS certificate verification (default %v)
-lua []string Lua script for request transformation (inline or @file/@url)
-js []string JavaScript script for request transformation (inline or @file/@url)`
-U, -url string Target URL for the request
-M, -method []string HTTP method for the request (default %s)
-B, -body []string Body for the request (e.g. "body text")
-P, -param []string URL parameter for the request (e.g. "key1=value1")
-H, -header []string Header for the request (e.g. "key1: value1")
-C, -cookie []string Cookie for the request (e.g. "key1=value1")
-X, -proxy []string Proxy for the request (e.g. "http://proxy.example.com:8080")
-V, -values []string List of values for templating (e.g. "key1=value1")
-T, -timeout time Timeout for the request (e.g. 400ms, 3s, 1m10s) (default %v)
-S, -max-response-body size Largest response body to read (e.g. 10MiB, 2MB), 0 for no limit (default %s)
-I, -insecure bool Skip SSL/TLS certificate verification (default %v)
-lua []string Lua script for request transformation (inline or @file/@url)
-js []string JavaScript script for request transformation (inline or @file/@url)`
var _ IParser = ConfigCLIParser{}
@@ -97,18 +98,19 @@ func (parser ConfigCLIParser) Parse() (*Config, error) {
dryRun bool
// Request config
urlInput string
methods = stringSliceArg{}
bodies = stringSliceArg{}
params = stringSliceArg{}
headers = stringSliceArg{}
cookies = stringSliceArg{}
proxies = stringSliceArg{}
values = stringSliceArg{}
timeout time.Duration
insecure bool
luaScripts = stringSliceArg{}
jsScripts = stringSliceArg{}
urlInput string
methods = stringSliceArg{}
bodies = stringSliceArg{}
params = stringSliceArg{}
headers = stringSliceArg{}
cookies = stringSliceArg{}
proxies = stringSliceArg{}
values = stringSliceArg{}
timeout time.Duration
maxResponseBody byteSize
insecure bool
luaScripts = stringSliceArg{}
jsScripts = stringSliceArg{}
)
{
@@ -174,6 +176,9 @@ func (parser ConfigCLIParser) Parse() (*Config, error) {
flagSet.DurationVar(&timeout, "timeout", 0, "Timeout for the request (e.g. 400ms, 15s, 1m10s)")
flagSet.DurationVar(&timeout, "T", 0, "Timeout for the request (e.g. 400ms, 15s, 1m10s)")
flagSet.Var(&maxResponseBody, "max-response-body", "Largest response body to read (e.g. 1048576, 10MiB, 2MB), 0 for no limit")
flagSet.Var(&maxResponseBody, "S", "Largest response body to read (e.g. 1048576, 10MiB, 2MB), 0 for no limit")
flagSet.BoolVar(&insecure, "insecure", false, "Skip SSL/TLS certificate verification")
flagSet.BoolVar(&insecure, "I", false, "Skip SSL/TLS certificate verification")
@@ -258,6 +263,8 @@ func (parser ConfigCLIParser) Parse() (*Config, error) {
config.Values = append(config.Values, values...)
case "timeout", "T":
config.Timeout = new(timeout)
case "max-response-body", "S":
config.MaxResponseBody = new(uint64(maxResponseBody))
case "insecure", "I":
config.Insecure = new(insecure)
case "lua":
@@ -286,6 +293,7 @@ func (parser ConfigCLIParser) PrintHelp() {
Defaults.Method,
Defaults.RequestTimeout,
formatByteSize(Defaults.MaxResponseBody),
Defaults.Insecure,
)
}
+65 -42
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"math"
"net/url"
"os"
"path/filepath"
@@ -27,27 +28,29 @@ import (
)
var Defaults = struct {
UserAgent string
Method string
RequestTimeout time.Duration
Concurrency uint
ShowConfig bool
Progress ConfigProgressType
Insecure bool
Output ConfigOutputType
DryRun bool
LogLevel string
UserAgent string
Method string
RequestTimeout time.Duration
MaxResponseBody uint64
Concurrency uint
ShowConfig bool
Progress ConfigProgressType
Insecure bool
Output ConfigOutputType
DryRun bool
LogLevel string
}{
UserAgent: "Sarin/" + version.Version,
Method: "GET",
RequestTimeout: time.Second * 10,
Concurrency: 1,
ShowConfig: false,
Progress: ConfigProgressTypeBar,
Insecure: false,
Output: ConfigOutputTypeTable,
DryRun: false,
LogLevel: "error",
UserAgent: "Sarin/" + version.Version,
Method: "GET",
RequestTimeout: time.Second * 10,
MaxResponseBody: 10 << 20, // 10 MiB
Concurrency: 1,
ShowConfig: false,
Progress: ConfigProgressTypeBar,
Insecure: false,
Output: ConfigOutputTypeTable,
DryRun: false,
LogLevel: "error",
}
var (
@@ -56,6 +59,8 @@ var (
ValidLogLevels = []string{"info", "error"}
)
const maxResponseBodyLimit = min(1<<40, math.MaxInt) // 1 TiB on 64 bit, 2 GiB - 1 on 32 bit
var (
StyleYellow = lipgloss.NewStyle().Foreground(lipgloss.Color("220"))
StyleRed = lipgloss.NewStyle().Foreground(lipgloss.Color("196"))
@@ -82,28 +87,29 @@ var (
)
type Config struct {
ShowConfig *bool `yaml:"showConfig,omitempty"`
Files []types.ConfigFile `yaml:"files,omitempty"`
Methods []string `yaml:"methods,omitempty"`
URL *url.URL `yaml:"url,omitempty"`
Timeout *time.Duration `yaml:"timeout,omitempty"`
Concurrency *uint `yaml:"concurrency,omitempty"`
Requests *uint64 `yaml:"requests,omitempty"`
Duration *time.Duration `yaml:"duration,omitempty"`
Progress *ConfigProgressType `yaml:"progress,omitempty"`
Output *ConfigOutputType `yaml:"output,omitempty"`
Insecure *bool `yaml:"insecure,omitempty"`
DryRun *bool `yaml:"dryRun,omitempty"`
Params types.Params `yaml:"params,omitempty"`
Headers types.Headers `yaml:"headers,omitempty"`
Cookies types.Cookies `yaml:"cookies,omitempty"`
Bodies []string `yaml:"bodies,omitempty"`
Proxies types.Proxies `yaml:"proxies,omitempty"`
Values []string `yaml:"values,omitempty"`
Lua []string `yaml:"lua,omitempty"`
Js []string `yaml:"js,omitempty"`
LogLevel *string `yaml:"logLevel,omitempty"`
LogFile *string `yaml:"logFile,omitempty"`
ShowConfig *bool `yaml:"showConfig,omitempty"`
Files []types.ConfigFile `yaml:"files,omitempty"`
Methods []string `yaml:"methods,omitempty"`
URL *url.URL `yaml:"url,omitempty"`
Timeout *time.Duration `yaml:"timeout,omitempty"`
MaxResponseBody *uint64 `yaml:"maxResponseBody,omitempty"`
Concurrency *uint `yaml:"concurrency,omitempty"`
Requests *uint64 `yaml:"requests,omitempty"`
Duration *time.Duration `yaml:"duration,omitempty"`
Progress *ConfigProgressType `yaml:"progress,omitempty"`
Output *ConfigOutputType `yaml:"output,omitempty"`
Insecure *bool `yaml:"insecure,omitempty"`
DryRun *bool `yaml:"dryRun,omitempty"`
Params types.Params `yaml:"params,omitempty"`
Headers types.Headers `yaml:"headers,omitempty"`
Cookies types.Cookies `yaml:"cookies,omitempty"`
Bodies []string `yaml:"bodies,omitempty"`
Proxies types.Proxies `yaml:"proxies,omitempty"`
Values []string `yaml:"values,omitempty"`
Lua []string `yaml:"lua,omitempty"`
Js []string `yaml:"js,omitempty"`
LogLevel *string `yaml:"logLevel,omitempty"`
LogFile *string `yaml:"logFile,omitempty"`
}
func (config Config) MarshalYAML() (any, error) {
@@ -177,6 +183,9 @@ func (config Config) MarshalYAML() (any, error) {
if config.Timeout != nil {
addField(content, "timeout", toNode(*config.Timeout), "")
}
if config.MaxResponseBody != nil {
addField(content, "maxResponseBody", toNode(formatByteSize(*config.MaxResponseBody)), "")
}
if config.Concurrency != nil {
addField(content, "concurrency", toNode(*config.Concurrency), "")
}
@@ -301,6 +310,9 @@ func (config *Config) Merge(newConfig *Config) {
if newConfig.Timeout != nil {
config.Timeout = newConfig.Timeout
}
if newConfig.MaxResponseBody != nil {
config.MaxResponseBody = newConfig.MaxResponseBody
}
if newConfig.Concurrency != nil {
config.Concurrency = newConfig.Concurrency
}
@@ -379,6 +391,9 @@ func (config *Config) SetDefaults() {
if config.Timeout == nil {
config.Timeout = &Defaults.RequestTimeout
}
if config.MaxResponseBody == nil {
config.MaxResponseBody = &Defaults.MaxResponseBody
}
if config.Concurrency == nil {
config.Concurrency = new(Defaults.Concurrency)
}
@@ -454,6 +469,14 @@ func (config Config) Validate() error {
validationErrors = append(validationErrors, types.NewFieldValidationError("Timeout", "0", errors.New("timeout must be greater than 0")))
}
if config.MaxResponseBody != nil && *config.MaxResponseBody > maxResponseBodyLimit {
validationErrors = append(validationErrors, types.NewFieldValidationError(
"MaxResponseBody",
formatByteSize(*config.MaxResponseBody),
errors.New("max response body must not exceed "+formatByteSize(maxResponseBodyLimit)),
))
}
if config.ShowConfig == nil {
validationErrors = append(validationErrors, types.NewFieldValidationError("ShowConfig", "", errors.New("showConfig field is required")))
}
+16
View File
@@ -195,6 +195,22 @@ func (parser ConfigENVParser) Parse() (*Config, error) {
}
}
if maxResponseBody := parser.getEnv("MAX_RESPONSE_BODY"); maxResponseBody != "" {
maxResponseBodyParsed, err := parseByteSize(maxResponseBody)
if err != nil {
fieldParseErrors = append(
fieldParseErrors,
types.NewFieldParseError(
parser.getFullEnvName("MAX_RESPONSE_BODY"),
maxResponseBody,
err,
),
)
} else {
config.MaxResponseBody = &maxResponseBodyParsed
}
}
if insecure := parser.getEnv("INSECURE"); insecure != "" {
insecureParsed, err := utilsParse.ParseString[bool](insecure)
if err != nil {
+26 -22
View File
@@ -192,28 +192,29 @@ func (kv *keyValuesField) unmarshalMapping(node *yaml.Node) error {
}
type configYAML struct {
ShowConfig *bool `yaml:"showConfig"`
ConfigFiles stringOrSliceField `yaml:"configFile"`
Concurrency *uint `yaml:"concurrency"`
RequestCount *uint64 `yaml:"requests"`
Duration *time.Duration `yaml:"duration"`
LogLevel *string `yaml:"logLevel"`
LogFile *string `yaml:"logFile"`
Progress *string `yaml:"progress"`
Output *string `yaml:"output"`
DryRun *bool `yaml:"dryRun"`
URL *string `yaml:"url"`
Method stringOrSliceField `yaml:"method"`
Bodies stringOrSliceField `yaml:"body"`
Params keyValuesField `yaml:"params"`
Headers keyValuesField `yaml:"headers"`
Cookies keyValuesField `yaml:"cookies"`
Proxies stringOrSliceField `yaml:"proxy"`
Values stringOrSliceField `yaml:"values"`
Timeout *time.Duration `yaml:"timeout"`
Insecure *bool `yaml:"insecure"`
Lua stringOrSliceField `yaml:"lua"`
Js stringOrSliceField `yaml:"js"`
ShowConfig *bool `yaml:"showConfig"`
ConfigFiles stringOrSliceField `yaml:"configFile"`
Concurrency *uint `yaml:"concurrency"`
RequestCount *uint64 `yaml:"requests"`
Duration *time.Duration `yaml:"duration"`
LogLevel *string `yaml:"logLevel"`
LogFile *string `yaml:"logFile"`
Progress *string `yaml:"progress"`
Output *string `yaml:"output"`
DryRun *bool `yaml:"dryRun"`
URL *string `yaml:"url"`
Method stringOrSliceField `yaml:"method"`
Bodies stringOrSliceField `yaml:"body"`
Params keyValuesField `yaml:"params"`
Headers keyValuesField `yaml:"headers"`
Cookies keyValuesField `yaml:"cookies"`
Proxies stringOrSliceField `yaml:"proxy"`
Values stringOrSliceField `yaml:"values"`
Timeout *time.Duration `yaml:"timeout"`
MaxResponseBody *byteSize `yaml:"maxResponseBody"`
Insecure *bool `yaml:"insecure"`
Lua stringOrSliceField `yaml:"lua"`
Js stringOrSliceField `yaml:"js"`
}
// ParseYAML parses YAML config file arguments into a Config object.
@@ -290,6 +291,9 @@ func (parser ConfigFileParser) ParseYAML(data []byte) (*Config, error) {
config.Values = append(config.Values, parsedData.Values...)
config.Timeout = parsedData.Timeout
if parsedData.MaxResponseBody != nil {
config.MaxResponseBody = new(uint64(*parsedData.MaxResponseBody))
}
config.Insecure = parsedData.Insecure
config.Lua = append(config.Lua, parsedData.Lua...)
config.Js = append(config.Js, parsedData.Js...)
+3 -8
View File
@@ -5,7 +5,6 @@ import (
"context"
"crypto/tls"
"encoding/base64"
"math"
"net"
"net/http"
"net/url"
@@ -20,13 +19,6 @@ import (
// clientIndexGenerator returns the index of the host client (and proxy) to use next.
type clientIndexGenerator func() int
func safeUintToInt(u uint) int {
if u > math.MaxInt {
return math.MaxInt
}
return int(u)
}
// NewHostClients creates a list of fasthttp.HostClient instances for the given proxies.
// If no proxies are provided, a single client without a proxy is returned.
// It can return the following errors:
@@ -38,6 +30,7 @@ func NewHostClients(
maxConns uint,
requestURL *url.URL,
skipVerify bool,
maxResponseBody uint64,
) ([]*fasthttp.HostClient, error) {
isTLS := requestURL.Scheme == "https"
@@ -66,6 +59,7 @@ func NewHostClients(
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
MaxResponseBodySize: safeUint64ToInt(maxResponseBody),
DisableHeaderNamesNormalizing: true,
DisablePathNormalizing: true,
NoDefaultUserAgentHeader: true,
@@ -88,6 +82,7 @@ func NewHostClients(
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
MaxResponseBodySize: safeUint64ToInt(maxResponseBody),
DisableHeaderNamesNormalizing: true,
DisablePathNormalizing: true,
NoDefaultUserAgentHeader: true,
+15
View File
@@ -1,6 +1,7 @@
package sarin
import (
"math"
"math/rand/v2"
"time"
)
@@ -19,3 +20,17 @@ func firstOrEmpty(values []string) string {
}
return values[0]
}
func safeUintToInt(u uint) int {
if u > math.MaxInt {
return math.MaxInt
}
return int(u)
}
func safeUint64ToInt(u uint64) int {
if u > math.MaxInt {
return math.MaxInt
}
return int(u)
}
+3 -2
View File
@@ -225,6 +225,7 @@ func NewSarin(
totalDuration *time.Duration,
showProgress bool,
skipCertVerify bool,
maxResponseBody uint64,
params types.Params,
headers types.Headers,
cookies types.Cookies,
@@ -258,7 +259,7 @@ func NewSarin(
proxyURLs[i] = url.URL(proxy)
}
hostClients, err := NewHostClients(ctx, timeout, proxyURLs, workers, requestURL, skipCertVerify)
hostClients, err := NewHostClients(ctx, timeout, proxyURLs, workers, requestURL, skipCertVerify, maxResponseBody)
if err != nil {
return nil, err
}
@@ -280,7 +281,7 @@ func NewSarin(
var scriptHTTPClients []*scriptHTTPClient
if !scriptChain.IsEmpty() {
scriptHTTPClients, err = newScriptHTTPClients(jobsCtx, proxyURLs, workers)
scriptHTTPClients, err = newScriptHTTPClients(jobsCtx, proxyURLs, workers, maxResponseBody)
if err != nil {
jobsCancel()
return nil, err
+51 -7
View File
@@ -1,8 +1,11 @@
package sarin
import (
"bytes"
"context"
"crypto/tls"
"errors"
"io"
"net"
"net/url"
"time"
@@ -28,9 +31,9 @@ var _ script.HTTPDoer = (*scriptHTTPClient)(nil)
// newScriptHTTPClients creates one client per proxy, in the same order as NewHostClients.
// It can return the following errors:
// - types.ProxyDialError
func newScriptHTTPClients(ctx context.Context, proxies []url.URL, maxConns uint) ([]*scriptHTTPClient, error) {
func newScriptHTTPClients(ctx context.Context, proxies []url.URL, maxConns uint, maxResponseBody uint64) ([]*scriptHTTPClient, error) {
if len(proxies) == 0 {
return []*scriptHTTPClient{newScriptHTTPClient(ctx, fasthttp.DialDualStackTimeout, scriptHTTPDefaultTimeout, maxConns)}, nil
return []*scriptHTTPClient{newScriptHTTPClient(ctx, fasthttp.DialDualStackTimeout, scriptHTTPDefaultTimeout, maxConns, maxResponseBody)}, nil
}
clients := make([]*scriptHTTPClient, 0, len(proxies))
@@ -39,18 +42,19 @@ func newScriptHTTPClients(ctx context.Context, proxies []url.URL, maxConns uint)
if err != nil {
return nil, types.NewProxyDialError(proxy.String(), err)
}
clients = append(clients, newScriptHTTPClient(ctx, dial, scriptHTTPDefaultTimeout, maxConns))
clients = append(clients, newScriptHTTPClient(ctx, dial, scriptHTTPDefaultTimeout, maxConns, maxResponseBody))
}
return clients, nil
}
func newScriptHTTPClient(ctx context.Context, dial fasthttp.DialFuncWithTimeout, defaultTimeout time.Duration, maxConns uint) *scriptHTTPClient {
func newScriptHTTPClient(ctx context.Context, dial fasthttp.DialFuncWithTimeout, defaultTimeout time.Duration, maxConns uint, maxResponseBody uint64) *scriptHTTPClient {
newClient := func(insecure bool) *fasthttp.Client {
// Read and write timeouts stay unset because they would cap the per-call timeout,
// so dialWithDeadline is what bounds the TLS handshake.
return &fasthttp.Client{
DialTimeout: dialWithDeadline(dial),
MaxConnsPerHost: safeUintToInt(maxConns),
DialTimeout: dialWithDeadline(dial),
MaxConnsPerHost: safeUintToInt(maxConns),
MaxResponseBodySize: safeUint64ToInt(maxResponseBody),
TLSConfig: &tls.Config{
InsecureSkipVerify: insecure, //nolint:gosec
},
@@ -144,6 +148,8 @@ func (c *scriptHTTPClient) Do(r *script.HTTPRequest) (*script.HTTPResponse, erro
}
req.SetTimeout(timeout)
// A body limit streams the response, so a declared Content-Length is never allocated.
resp.StreamBody = r.MaxBodySize > 0
if r.MaxRedirects > 0 {
err = client.DoRedirects(req, resp, r.MaxRedirects)
} else {
@@ -153,9 +159,47 @@ func (c *scriptHTTPClient) Do(r *script.HTTPRequest) (*script.HTTPResponse, erro
return nil, types.NewScriptHTTPRequestError(r.Method, r.URL, err)
}
body, err := readScriptBody(resp, r.MaxBodySize)
if err != nil {
return nil, types.NewScriptHTTPRequestError(r.Method, r.URL, err)
}
return &script.HTTPResponse{
Status: resp.StatusCode(),
Headers: collectRespHeaders(resp),
Body: string(resp.Body()),
Body: body,
}, nil
}
// readScriptBody reads the streamed body, refusing it once it passes maxBodySize.
// It can return the following errors:
// - types.ErrScriptHTTPBodyTooLarge
func readScriptBody(resp *fasthttp.Response, maxBodySize int64) (string, error) {
stream := resp.BodyStream()
if stream == nil {
return string(resp.Body()), nil
}
var (
body bytes.Buffer
err error
)
if maxBodySize > 0 {
// One byte past the limit tells an exact fit from an overflow.
_, err = io.CopyN(&body, stream, maxBodySize+1)
if err == nil {
err = types.ErrScriptHTTPBodyTooLarge
} else if errors.Is(err, io.EOF) {
err = nil
}
} else {
_, err = io.Copy(&body, stream)
}
// An unfinished stream makes fasthttp drop the connection instead of pooling it.
_ = resp.CloseBodyStream()
if err != nil {
return "", err
}
return body.String(), nil
}
+18 -2
View File
@@ -2,7 +2,9 @@ package sarin
import (
"errors"
"net"
"strconv"
"strings"
"sync/atomic"
"time"
@@ -31,6 +33,20 @@ func statusCodeToString(code int) string {
return strconv.Itoa(code)
}
// responseErrorKey drops the local address, whose port would mint a stats entry per connection.
func responseErrorKey(err error) string {
message := err.Error()
var opErr *net.OpError
if !errors.As(err, &opErr) || opErr.Source == nil {
return message
}
withoutSource := *opErr
withoutSource.Source = nil
return strings.Replace(message, opErr.Error(), withoutSource.Error(), 1)
}
// requestErrorKey keys a generation failure by its source instead of its message.
// A script picks its own URLs and error texts, so the message would mint a new
// stats entry per request. The full message still goes to the runtime log.
@@ -135,7 +151,7 @@ func (s sarin) workerStatsWithDynamic(
respDuration := time.Since(startTime)
if err != nil {
s.responses.Add(err.Error(), respDuration)
s.responses.Add(responseErrorKey(err), respDuration)
} else {
s.responses.Add(statusCodeToString(resp.StatusCode()), respDuration)
sendRespLog(respDuration, resp)
@@ -170,7 +186,7 @@ func (s sarin) workerStatsWithStatic(
err := s.hostClients[nextClientIndex()].DoTimeout(req, resp, s.timeout)
respDuration := time.Since(startTime)
if err != nil {
s.responses.Add(err.Error(), respDuration)
s.responses.Add(responseErrorKey(err), respDuration)
} else {
s.responses.Add(statusCodeToString(resp.StatusCode()), respDuration)
sendRespLog(respDuration, resp)
+21
View File
@@ -21,6 +21,7 @@ type HTTPRequest struct {
Timeout time.Duration // zero means the doer's default timeout
Insecure bool
MaxRedirects int
MaxBodySize int64 // zero means no limit
}
// HTTPResponse is the response returned to a script by the http.* functions.
@@ -66,6 +67,9 @@ const httpDefaultMethod = http.MethodGet
// httpMaxRedirectsLimit is the highest maxRedirects a script may ask for.
const httpMaxRedirectsLimit = 100
// httpMaxBodySizeLimit is the highest maxBodySize a script may ask for, 1 TiB.
const httpMaxBodySizeLimit int64 = 1 << 40
// Option keys accepted by the http function.
const (
httpOptionMethod = "method"
@@ -76,6 +80,7 @@ const (
httpOptionTimeout = "timeout"
httpOptionInsecure = "insecure"
httpOptionMaxRedirects = "maxRedirects"
httpOptionMaxBodySize = "maxBodySize"
)
// httpBridge connects http to the doer. Calls only work while transform runs.
@@ -118,6 +123,22 @@ func parseHTTPMaxRedirects(count float64) (int, error) {
return int(count), nil
}
// parseHTTPMaxBodySize checks that the body limit is a non-negative whole number.
// It can return the following errors:
// - types.ScriptHTTPOptionError
func parseHTTPMaxBodySize(size float64) (int64, error) {
if math.IsNaN(size) || size < 0 || size > float64(httpMaxBodySizeLimit) || size != math.Trunc(size) {
return 0, types.NewScriptHTTPOptionError(
httpOptionMaxBodySize,
types.NewScriptTypeError(
"a whole number of bytes between 0 and "+strconv.FormatInt(httpMaxBodySizeLimit, 10),
strconv.FormatFloat(size, 'g', -1, 64),
),
)
}
return int64(size), nil
}
// parseHTTPTimeout parses a Go duration string such as "500ms" or "2s".
// It can return the following errors:
// - types.ScriptHTTPOptionError
+10
View File
@@ -117,6 +117,16 @@ func (e *JsEngine) parseHTTPOptions(opts *goja.Object, req *HTTPRequest) error {
if req.MaxRedirects, err = parseHTTPMaxRedirects(count); err != nil {
return err
}
case httpOptionMaxBodySize:
size, ok := numberValue(value)
if !ok {
return types.NewScriptHTTPOptionError(name, types.NewScriptTypeError("number", jsTypeName(value)))
}
var err error
if req.MaxBodySize, err = parseHTTPMaxBodySize(size); err != nil {
return err
}
default:
return types.NewScriptHTTPOptionError(name, types.ErrScriptHTTPUnknownOption)
}
+7
View File
@@ -106,6 +106,13 @@ func (e *LuaEngine) parseHTTPOptions(opts *lua.LTable, req *HTTPRequest) error {
return
}
req.MaxRedirects, err = parseHTTPMaxRedirects(float64(count))
case httpOptionMaxBodySize:
size, ok := v.(lua.LNumber)
if !ok {
err = types.NewScriptHTTPOptionError(name, types.NewScriptTypeError("number", v.Type().String()))
return
}
req.MaxBodySize, err = parseHTTPMaxBodySize(float64(size))
default:
err = types.NewScriptHTTPOptionError(name, types.ErrScriptHTTPUnknownOption)
}
+6
View File
@@ -10,6 +10,11 @@ import (
var (
errNoError = errors.New("no error (internal)")
ErrByteSizeInvalid = errors.New(
"invalid value for size, expected a whole number of bytes or a whole number with a unit: " +
"B, KB, MB, GB, TB (1000 based) or KiB, MiB, GiB, TiB (1024 based), e.g. '1048576', '10MiB', '2MB'",
)
)
type FieldParseError struct {
@@ -408,6 +413,7 @@ var (
ErrScriptHTTPMethodEmpty = errors.New("http method cannot be empty")
ErrScriptHTTPURLInvalid = errors.New("URL must be an absolute http or https URL")
ErrScriptHTTPUnknownOption = errors.New("unknown option")
ErrScriptHTTPBodyTooLarge = errors.New("response body exceeds maxBodySize")
ErrScriptJSONCycle = errors.New("cannot encode a table that contains itself")
)