mirror of
https://github.com/aykhans/sarin.git
synced 2026-10-16 14:43:53 +00:00
feat: limit response body sizes
fasthttp allocates the body from the declared Content-Length before reading it, so a server could exhaust memory by declaring a size it never sends. The new max response body config bounds every response, defaults to 10MiB and accepts sizes like 10MiB or 2MB. Scripts get the same as a maxBodySize option on http().
This commit is contained in:
17 files changed
+452
-147
No files matched your search
@@ -0,0 +1,123 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"go.aykhans.me/sarin/internal/types"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
var (
|
||||
_ flag.Value = (*byteSize)(nil)
|
||||
_ yaml.Unmarshaler = (*byteSize)(nil)
|
||||
)
|
||||
|
||||
// byteSize is a size in bytes that reads and prints unit suffixes such as 10MiB.
|
||||
type byteSize uint64
|
||||
|
||||
// byteSizeUnits run from the largest scale down, so formatting picks the largest unit
|
||||
// that fits. A bare "K" is out: the common libraries disagree on whether it counts
|
||||
// 1000 or 1024 bytes.
|
||||
var byteSizeUnits = []struct {
|
||||
suffix string
|
||||
scale uint64
|
||||
}{
|
||||
{"TiB", 1 << 40},
|
||||
{"TB", 1000 * 1000 * 1000 * 1000},
|
||||
{"GiB", 1 << 30},
|
||||
{"GB", 1000 * 1000 * 1000},
|
||||
{"MiB", 1 << 20},
|
||||
{"MB", 1000 * 1000},
|
||||
{"KiB", 1 << 10},
|
||||
{"KB", 1000},
|
||||
{"B", 1},
|
||||
}
|
||||
|
||||
// parseByteSize reads a byte count, with or without a unit suffix.
|
||||
func parseByteSize(value string) (uint64, error) {
|
||||
text := strings.TrimSpace(value)
|
||||
|
||||
// The suffix is split off by index, so no case mapping can shift the digits.
|
||||
digitsEnd := len(text)
|
||||
for digitsEnd > 0 && isASCIILetter(text[digitsEnd-1]) {
|
||||
digitsEnd--
|
||||
}
|
||||
|
||||
scale := uint64(1)
|
||||
if suffix := text[digitsEnd:]; suffix != "" {
|
||||
unitScale, ok := byteSizeScale(suffix)
|
||||
if !ok {
|
||||
return 0, types.ErrByteSizeInvalid
|
||||
}
|
||||
scale = unitScale
|
||||
}
|
||||
|
||||
size, err := strconv.ParseUint(strings.TrimSpace(text[:digitsEnd]), 10, 64)
|
||||
if err != nil {
|
||||
return 0, types.ErrByteSizeInvalid
|
||||
}
|
||||
if size > math.MaxUint64/scale {
|
||||
return 0, types.ErrByteSizeInvalid
|
||||
}
|
||||
return size * scale, nil
|
||||
}
|
||||
|
||||
// byteSizeScale looks up a unit suffix, ignoring case.
|
||||
func byteSizeScale(suffix string) (uint64, bool) {
|
||||
for _, unit := range byteSizeUnits {
|
||||
if strings.EqualFold(suffix, unit.suffix) {
|
||||
return unit.scale, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func isASCIILetter(char byte) bool {
|
||||
return (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z')
|
||||
}
|
||||
|
||||
// formatByteSize renders a byte count so that parseByteSize reads back the same value.
|
||||
func formatByteSize(size uint64) string {
|
||||
for _, unit := range byteSizeUnits {
|
||||
if unit.scale > 1 && size >= unit.scale && size%unit.scale == 0 {
|
||||
return strconv.FormatUint(size/unit.scale, 10) + unit.suffix
|
||||
}
|
||||
}
|
||||
return strconv.FormatUint(size, 10)
|
||||
}
|
||||
|
||||
func (size byteSize) String() string {
|
||||
return formatByteSize(uint64(size))
|
||||
}
|
||||
|
||||
// Set fills the size from a command line value.
|
||||
func (size *byteSize) Set(value string) error {
|
||||
parsed, err := parseByteSize(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*size = byteSize(parsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UnmarshalYAML accepts both a YAML integer and a string with a unit.
|
||||
func (size *byteSize) UnmarshalYAML(node *yaml.Node) error {
|
||||
if node.Kind != yaml.ScalarNode {
|
||||
return fmt.Errorf("expected a size, but got %s", node.Tag)
|
||||
}
|
||||
|
||||
if node.Tag == "!!int" {
|
||||
var parsed uint64
|
||||
if err := node.Decode(&parsed); err != nil {
|
||||
return types.ErrByteSizeInvalid
|
||||
}
|
||||
*size = byteSize(parsed)
|
||||
return nil
|
||||
}
|
||||
|
||||
return size.Set(node.Value)
|
||||
}
|
||||
+44
-36
@@ -20,32 +20,33 @@ Simple usage:
|
||||
|
||||
Flags:
|
||||
General Config:
|
||||
-h, -help Help for sarin
|
||||
-v, -version Version for sarin
|
||||
-s, -show-config bool Show the final config after parsing all sources (default %v)
|
||||
-f, -config-file []string Path to the config file (local file / http URL)
|
||||
-c, -concurrency uint Number of concurrent requests (default %d)
|
||||
-r, -requests uint Number of total requests
|
||||
-d, -duration time Maximum duration for the test (e.g. 30s, 1m, 5h)
|
||||
-l, -log-level string Runtime log levels to emit, comma-separated (possible values: info, error) (default %s)
|
||||
-w, -log-file string Write runtime logs to this file instead of the terminal/stderr
|
||||
-p, -progress string Progress display (possible values: bar, none) (default '%v')
|
||||
-o, -output string Output format (possible values: table, json, yaml, none) (default '%v')
|
||||
-z, -dry-run bool Run without sending requests (default %v)
|
||||
-h, -help Help for sarin
|
||||
-v, -version Version for sarin
|
||||
-s, -show-config bool Show the final config after parsing all sources (default %v)
|
||||
-f, -config-file []string Path to the config file (local file / http URL)
|
||||
-c, -concurrency uint Number of concurrent requests (default %d)
|
||||
-r, -requests uint Number of total requests
|
||||
-d, -duration time Maximum duration for the test (e.g. 30s, 1m, 5h)
|
||||
-l, -log-level string Runtime log levels to emit, comma-separated (possible values: info, error) (default %s)
|
||||
-w, -log-file string Write runtime logs to this file instead of the terminal/stderr
|
||||
-p, -progress string Progress display (possible values: bar, none) (default '%v')
|
||||
-o, -output string Output format (possible values: table, json, yaml, none) (default '%v')
|
||||
-z, -dry-run bool Run without sending requests (default %v)
|
||||
|
||||
Request Config:
|
||||
-U, -url string Target URL for the request
|
||||
-M, -method []string HTTP method for the request (default %s)
|
||||
-B, -body []string Body for the request (e.g. "body text")
|
||||
-P, -param []string URL parameter for the request (e.g. "key1=value1")
|
||||
-H, -header []string Header for the request (e.g. "key1: value1")
|
||||
-C, -cookie []string Cookie for the request (e.g. "key1=value1")
|
||||
-X, -proxy []string Proxy for the request (e.g. "http://proxy.example.com:8080")
|
||||
-V, -values []string List of values for templating (e.g. "key1=value1")
|
||||
-T, -timeout time Timeout for the request (e.g. 400ms, 3s, 1m10s) (default %v)
|
||||
-I, -insecure bool Skip SSL/TLS certificate verification (default %v)
|
||||
-lua []string Lua script for request transformation (inline or @file/@url)
|
||||
-js []string JavaScript script for request transformation (inline or @file/@url)`
|
||||
-U, -url string Target URL for the request
|
||||
-M, -method []string HTTP method for the request (default %s)
|
||||
-B, -body []string Body for the request (e.g. "body text")
|
||||
-P, -param []string URL parameter for the request (e.g. "key1=value1")
|
||||
-H, -header []string Header for the request (e.g. "key1: value1")
|
||||
-C, -cookie []string Cookie for the request (e.g. "key1=value1")
|
||||
-X, -proxy []string Proxy for the request (e.g. "http://proxy.example.com:8080")
|
||||
-V, -values []string List of values for templating (e.g. "key1=value1")
|
||||
-T, -timeout time Timeout for the request (e.g. 400ms, 3s, 1m10s) (default %v)
|
||||
-S, -max-response-body size Largest response body to read (e.g. 10MiB, 2MB), 0 for no limit (default %s)
|
||||
-I, -insecure bool Skip SSL/TLS certificate verification (default %v)
|
||||
-lua []string Lua script for request transformation (inline or @file/@url)
|
||||
-js []string JavaScript script for request transformation (inline or @file/@url)`
|
||||
|
||||
var _ IParser = ConfigCLIParser{}
|
||||
|
||||
@@ -97,18 +98,19 @@ func (parser ConfigCLIParser) Parse() (*Config, error) {
|
||||
dryRun bool
|
||||
|
||||
// Request config
|
||||
urlInput string
|
||||
methods = stringSliceArg{}
|
||||
bodies = stringSliceArg{}
|
||||
params = stringSliceArg{}
|
||||
headers = stringSliceArg{}
|
||||
cookies = stringSliceArg{}
|
||||
proxies = stringSliceArg{}
|
||||
values = stringSliceArg{}
|
||||
timeout time.Duration
|
||||
insecure bool
|
||||
luaScripts = stringSliceArg{}
|
||||
jsScripts = stringSliceArg{}
|
||||
urlInput string
|
||||
methods = stringSliceArg{}
|
||||
bodies = stringSliceArg{}
|
||||
params = stringSliceArg{}
|
||||
headers = stringSliceArg{}
|
||||
cookies = stringSliceArg{}
|
||||
proxies = stringSliceArg{}
|
||||
values = stringSliceArg{}
|
||||
timeout time.Duration
|
||||
maxResponseBody byteSize
|
||||
insecure bool
|
||||
luaScripts = stringSliceArg{}
|
||||
jsScripts = stringSliceArg{}
|
||||
)
|
||||
|
||||
{
|
||||
@@ -174,6 +176,9 @@ func (parser ConfigCLIParser) Parse() (*Config, error) {
|
||||
flagSet.DurationVar(&timeout, "timeout", 0, "Timeout for the request (e.g. 400ms, 15s, 1m10s)")
|
||||
flagSet.DurationVar(&timeout, "T", 0, "Timeout for the request (e.g. 400ms, 15s, 1m10s)")
|
||||
|
||||
flagSet.Var(&maxResponseBody, "max-response-body", "Largest response body to read (e.g. 1048576, 10MiB, 2MB), 0 for no limit")
|
||||
flagSet.Var(&maxResponseBody, "S", "Largest response body to read (e.g. 1048576, 10MiB, 2MB), 0 for no limit")
|
||||
|
||||
flagSet.BoolVar(&insecure, "insecure", false, "Skip SSL/TLS certificate verification")
|
||||
flagSet.BoolVar(&insecure, "I", false, "Skip SSL/TLS certificate verification")
|
||||
|
||||
@@ -258,6 +263,8 @@ func (parser ConfigCLIParser) Parse() (*Config, error) {
|
||||
config.Values = append(config.Values, values...)
|
||||
case "timeout", "T":
|
||||
config.Timeout = new(timeout)
|
||||
case "max-response-body", "S":
|
||||
config.MaxResponseBody = new(uint64(maxResponseBody))
|
||||
case "insecure", "I":
|
||||
config.Insecure = new(insecure)
|
||||
case "lua":
|
||||
@@ -286,6 +293,7 @@ func (parser ConfigCLIParser) PrintHelp() {
|
||||
|
||||
Defaults.Method,
|
||||
Defaults.RequestTimeout,
|
||||
formatByteSize(Defaults.MaxResponseBody),
|
||||
Defaults.Insecure,
|
||||
)
|
||||
}
|
||||
+65
-42
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -27,27 +28,29 @@ import (
|
||||
)
|
||||
|
||||
var Defaults = struct {
|
||||
UserAgent string
|
||||
Method string
|
||||
RequestTimeout time.Duration
|
||||
Concurrency uint
|
||||
ShowConfig bool
|
||||
Progress ConfigProgressType
|
||||
Insecure bool
|
||||
Output ConfigOutputType
|
||||
DryRun bool
|
||||
LogLevel string
|
||||
UserAgent string
|
||||
Method string
|
||||
RequestTimeout time.Duration
|
||||
MaxResponseBody uint64
|
||||
Concurrency uint
|
||||
ShowConfig bool
|
||||
Progress ConfigProgressType
|
||||
Insecure bool
|
||||
Output ConfigOutputType
|
||||
DryRun bool
|
||||
LogLevel string
|
||||
}{
|
||||
UserAgent: "Sarin/" + version.Version,
|
||||
Method: "GET",
|
||||
RequestTimeout: time.Second * 10,
|
||||
Concurrency: 1,
|
||||
ShowConfig: false,
|
||||
Progress: ConfigProgressTypeBar,
|
||||
Insecure: false,
|
||||
Output: ConfigOutputTypeTable,
|
||||
DryRun: false,
|
||||
LogLevel: "error",
|
||||
UserAgent: "Sarin/" + version.Version,
|
||||
Method: "GET",
|
||||
RequestTimeout: time.Second * 10,
|
||||
MaxResponseBody: 10 << 20, // 10 MiB
|
||||
Concurrency: 1,
|
||||
ShowConfig: false,
|
||||
Progress: ConfigProgressTypeBar,
|
||||
Insecure: false,
|
||||
Output: ConfigOutputTypeTable,
|
||||
DryRun: false,
|
||||
LogLevel: "error",
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -56,6 +59,8 @@ var (
|
||||
ValidLogLevels = []string{"info", "error"}
|
||||
)
|
||||
|
||||
const maxResponseBodyLimit = min(1<<40, math.MaxInt) // 1 TiB on 64 bit, 2 GiB - 1 on 32 bit
|
||||
|
||||
var (
|
||||
StyleYellow = lipgloss.NewStyle().Foreground(lipgloss.Color("220"))
|
||||
StyleRed = lipgloss.NewStyle().Foreground(lipgloss.Color("196"))
|
||||
@@ -82,28 +87,29 @@ var (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ShowConfig *bool `yaml:"showConfig,omitempty"`
|
||||
Files []types.ConfigFile `yaml:"files,omitempty"`
|
||||
Methods []string `yaml:"methods,omitempty"`
|
||||
URL *url.URL `yaml:"url,omitempty"`
|
||||
Timeout *time.Duration `yaml:"timeout,omitempty"`
|
||||
Concurrency *uint `yaml:"concurrency,omitempty"`
|
||||
Requests *uint64 `yaml:"requests,omitempty"`
|
||||
Duration *time.Duration `yaml:"duration,omitempty"`
|
||||
Progress *ConfigProgressType `yaml:"progress,omitempty"`
|
||||
Output *ConfigOutputType `yaml:"output,omitempty"`
|
||||
Insecure *bool `yaml:"insecure,omitempty"`
|
||||
DryRun *bool `yaml:"dryRun,omitempty"`
|
||||
Params types.Params `yaml:"params,omitempty"`
|
||||
Headers types.Headers `yaml:"headers,omitempty"`
|
||||
Cookies types.Cookies `yaml:"cookies,omitempty"`
|
||||
Bodies []string `yaml:"bodies,omitempty"`
|
||||
Proxies types.Proxies `yaml:"proxies,omitempty"`
|
||||
Values []string `yaml:"values,omitempty"`
|
||||
Lua []string `yaml:"lua,omitempty"`
|
||||
Js []string `yaml:"js,omitempty"`
|
||||
LogLevel *string `yaml:"logLevel,omitempty"`
|
||||
LogFile *string `yaml:"logFile,omitempty"`
|
||||
ShowConfig *bool `yaml:"showConfig,omitempty"`
|
||||
Files []types.ConfigFile `yaml:"files,omitempty"`
|
||||
Methods []string `yaml:"methods,omitempty"`
|
||||
URL *url.URL `yaml:"url,omitempty"`
|
||||
Timeout *time.Duration `yaml:"timeout,omitempty"`
|
||||
MaxResponseBody *uint64 `yaml:"maxResponseBody,omitempty"`
|
||||
Concurrency *uint `yaml:"concurrency,omitempty"`
|
||||
Requests *uint64 `yaml:"requests,omitempty"`
|
||||
Duration *time.Duration `yaml:"duration,omitempty"`
|
||||
Progress *ConfigProgressType `yaml:"progress,omitempty"`
|
||||
Output *ConfigOutputType `yaml:"output,omitempty"`
|
||||
Insecure *bool `yaml:"insecure,omitempty"`
|
||||
DryRun *bool `yaml:"dryRun,omitempty"`
|
||||
Params types.Params `yaml:"params,omitempty"`
|
||||
Headers types.Headers `yaml:"headers,omitempty"`
|
||||
Cookies types.Cookies `yaml:"cookies,omitempty"`
|
||||
Bodies []string `yaml:"bodies,omitempty"`
|
||||
Proxies types.Proxies `yaml:"proxies,omitempty"`
|
||||
Values []string `yaml:"values,omitempty"`
|
||||
Lua []string `yaml:"lua,omitempty"`
|
||||
Js []string `yaml:"js,omitempty"`
|
||||
LogLevel *string `yaml:"logLevel,omitempty"`
|
||||
LogFile *string `yaml:"logFile,omitempty"`
|
||||
}
|
||||
|
||||
func (config Config) MarshalYAML() (any, error) {
|
||||
@@ -177,6 +183,9 @@ func (config Config) MarshalYAML() (any, error) {
|
||||
if config.Timeout != nil {
|
||||
addField(content, "timeout", toNode(*config.Timeout), "")
|
||||
}
|
||||
if config.MaxResponseBody != nil {
|
||||
addField(content, "maxResponseBody", toNode(formatByteSize(*config.MaxResponseBody)), "")
|
||||
}
|
||||
if config.Concurrency != nil {
|
||||
addField(content, "concurrency", toNode(*config.Concurrency), "")
|
||||
}
|
||||
@@ -301,6 +310,9 @@ func (config *Config) Merge(newConfig *Config) {
|
||||
if newConfig.Timeout != nil {
|
||||
config.Timeout = newConfig.Timeout
|
||||
}
|
||||
if newConfig.MaxResponseBody != nil {
|
||||
config.MaxResponseBody = newConfig.MaxResponseBody
|
||||
}
|
||||
if newConfig.Concurrency != nil {
|
||||
config.Concurrency = newConfig.Concurrency
|
||||
}
|
||||
@@ -379,6 +391,9 @@ func (config *Config) SetDefaults() {
|
||||
if config.Timeout == nil {
|
||||
config.Timeout = &Defaults.RequestTimeout
|
||||
}
|
||||
if config.MaxResponseBody == nil {
|
||||
config.MaxResponseBody = &Defaults.MaxResponseBody
|
||||
}
|
||||
if config.Concurrency == nil {
|
||||
config.Concurrency = new(Defaults.Concurrency)
|
||||
}
|
||||
@@ -454,6 +469,14 @@ func (config Config) Validate() error {
|
||||
validationErrors = append(validationErrors, types.NewFieldValidationError("Timeout", "0", errors.New("timeout must be greater than 0")))
|
||||
}
|
||||
|
||||
if config.MaxResponseBody != nil && *config.MaxResponseBody > maxResponseBodyLimit {
|
||||
validationErrors = append(validationErrors, types.NewFieldValidationError(
|
||||
"MaxResponseBody",
|
||||
formatByteSize(*config.MaxResponseBody),
|
||||
errors.New("max response body must not exceed "+formatByteSize(maxResponseBodyLimit)),
|
||||
))
|
||||
}
|
||||
|
||||
if config.ShowConfig == nil {
|
||||
validationErrors = append(validationErrors, types.NewFieldValidationError("ShowConfig", "", errors.New("showConfig field is required")))
|
||||
}
|
||||
|
||||
@@ -195,6 +195,22 @@ func (parser ConfigENVParser) Parse() (*Config, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if maxResponseBody := parser.getEnv("MAX_RESPONSE_BODY"); maxResponseBody != "" {
|
||||
maxResponseBodyParsed, err := parseByteSize(maxResponseBody)
|
||||
if err != nil {
|
||||
fieldParseErrors = append(
|
||||
fieldParseErrors,
|
||||
types.NewFieldParseError(
|
||||
parser.getFullEnvName("MAX_RESPONSE_BODY"),
|
||||
maxResponseBody,
|
||||
err,
|
||||
),
|
||||
)
|
||||
} else {
|
||||
config.MaxResponseBody = &maxResponseBodyParsed
|
||||
}
|
||||
}
|
||||
|
||||
if insecure := parser.getEnv("INSECURE"); insecure != "" {
|
||||
insecureParsed, err := utilsParse.ParseString[bool](insecure)
|
||||
if err != nil {
|
||||
|
||||
+26
-22
@@ -192,28 +192,29 @@ func (kv *keyValuesField) unmarshalMapping(node *yaml.Node) error {
|
||||
}
|
||||
|
||||
type configYAML struct {
|
||||
ShowConfig *bool `yaml:"showConfig"`
|
||||
ConfigFiles stringOrSliceField `yaml:"configFile"`
|
||||
Concurrency *uint `yaml:"concurrency"`
|
||||
RequestCount *uint64 `yaml:"requests"`
|
||||
Duration *time.Duration `yaml:"duration"`
|
||||
LogLevel *string `yaml:"logLevel"`
|
||||
LogFile *string `yaml:"logFile"`
|
||||
Progress *string `yaml:"progress"`
|
||||
Output *string `yaml:"output"`
|
||||
DryRun *bool `yaml:"dryRun"`
|
||||
URL *string `yaml:"url"`
|
||||
Method stringOrSliceField `yaml:"method"`
|
||||
Bodies stringOrSliceField `yaml:"body"`
|
||||
Params keyValuesField `yaml:"params"`
|
||||
Headers keyValuesField `yaml:"headers"`
|
||||
Cookies keyValuesField `yaml:"cookies"`
|
||||
Proxies stringOrSliceField `yaml:"proxy"`
|
||||
Values stringOrSliceField `yaml:"values"`
|
||||
Timeout *time.Duration `yaml:"timeout"`
|
||||
Insecure *bool `yaml:"insecure"`
|
||||
Lua stringOrSliceField `yaml:"lua"`
|
||||
Js stringOrSliceField `yaml:"js"`
|
||||
ShowConfig *bool `yaml:"showConfig"`
|
||||
ConfigFiles stringOrSliceField `yaml:"configFile"`
|
||||
Concurrency *uint `yaml:"concurrency"`
|
||||
RequestCount *uint64 `yaml:"requests"`
|
||||
Duration *time.Duration `yaml:"duration"`
|
||||
LogLevel *string `yaml:"logLevel"`
|
||||
LogFile *string `yaml:"logFile"`
|
||||
Progress *string `yaml:"progress"`
|
||||
Output *string `yaml:"output"`
|
||||
DryRun *bool `yaml:"dryRun"`
|
||||
URL *string `yaml:"url"`
|
||||
Method stringOrSliceField `yaml:"method"`
|
||||
Bodies stringOrSliceField `yaml:"body"`
|
||||
Params keyValuesField `yaml:"params"`
|
||||
Headers keyValuesField `yaml:"headers"`
|
||||
Cookies keyValuesField `yaml:"cookies"`
|
||||
Proxies stringOrSliceField `yaml:"proxy"`
|
||||
Values stringOrSliceField `yaml:"values"`
|
||||
Timeout *time.Duration `yaml:"timeout"`
|
||||
MaxResponseBody *byteSize `yaml:"maxResponseBody"`
|
||||
Insecure *bool `yaml:"insecure"`
|
||||
Lua stringOrSliceField `yaml:"lua"`
|
||||
Js stringOrSliceField `yaml:"js"`
|
||||
}
|
||||
|
||||
// ParseYAML parses YAML config file arguments into a Config object.
|
||||
@@ -290,6 +291,9 @@ func (parser ConfigFileParser) ParseYAML(data []byte) (*Config, error) {
|
||||
|
||||
config.Values = append(config.Values, parsedData.Values...)
|
||||
config.Timeout = parsedData.Timeout
|
||||
if parsedData.MaxResponseBody != nil {
|
||||
config.MaxResponseBody = new(uint64(*parsedData.MaxResponseBody))
|
||||
}
|
||||
config.Insecure = parsedData.Insecure
|
||||
config.Lua = append(config.Lua, parsedData.Lua...)
|
||||
config.Js = append(config.Js, parsedData.Js...)
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -20,13 +19,6 @@ import (
|
||||
// clientIndexGenerator returns the index of the host client (and proxy) to use next.
|
||||
type clientIndexGenerator func() int
|
||||
|
||||
func safeUintToInt(u uint) int {
|
||||
if u > math.MaxInt {
|
||||
return math.MaxInt
|
||||
}
|
||||
return int(u)
|
||||
}
|
||||
|
||||
// NewHostClients creates a list of fasthttp.HostClient instances for the given proxies.
|
||||
// If no proxies are provided, a single client without a proxy is returned.
|
||||
// It can return the following errors:
|
||||
@@ -38,6 +30,7 @@ func NewHostClients(
|
||||
maxConns uint,
|
||||
requestURL *url.URL,
|
||||
skipVerify bool,
|
||||
maxResponseBody uint64,
|
||||
) ([]*fasthttp.HostClient, error) {
|
||||
isTLS := requestURL.Scheme == "https"
|
||||
|
||||
@@ -66,6 +59,7 @@ func NewHostClients(
|
||||
MaxConnDuration: timeout,
|
||||
WriteTimeout: timeout,
|
||||
ReadTimeout: timeout,
|
||||
MaxResponseBodySize: safeUint64ToInt(maxResponseBody),
|
||||
DisableHeaderNamesNormalizing: true,
|
||||
DisablePathNormalizing: true,
|
||||
NoDefaultUserAgentHeader: true,
|
||||
@@ -88,6 +82,7 @@ func NewHostClients(
|
||||
MaxConnDuration: timeout,
|
||||
WriteTimeout: timeout,
|
||||
ReadTimeout: timeout,
|
||||
MaxResponseBodySize: safeUint64ToInt(maxResponseBody),
|
||||
DisableHeaderNamesNormalizing: true,
|
||||
DisablePathNormalizing: true,
|
||||
NoDefaultUserAgentHeader: true,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package sarin
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
)
|
||||
@@ -19,3 +20,17 @@ func firstOrEmpty(values []string) string {
|
||||
}
|
||||
return values[0]
|
||||
}
|
||||
|
||||
func safeUintToInt(u uint) int {
|
||||
if u > math.MaxInt {
|
||||
return math.MaxInt
|
||||
}
|
||||
return int(u)
|
||||
}
|
||||
|
||||
func safeUint64ToInt(u uint64) int {
|
||||
if u > math.MaxInt {
|
||||
return math.MaxInt
|
||||
}
|
||||
return int(u)
|
||||
}
|
||||
@@ -225,6 +225,7 @@ func NewSarin(
|
||||
totalDuration *time.Duration,
|
||||
showProgress bool,
|
||||
skipCertVerify bool,
|
||||
maxResponseBody uint64,
|
||||
params types.Params,
|
||||
headers types.Headers,
|
||||
cookies types.Cookies,
|
||||
@@ -258,7 +259,7 @@ func NewSarin(
|
||||
proxyURLs[i] = url.URL(proxy)
|
||||
}
|
||||
|
||||
hostClients, err := NewHostClients(ctx, timeout, proxyURLs, workers, requestURL, skipCertVerify)
|
||||
hostClients, err := NewHostClients(ctx, timeout, proxyURLs, workers, requestURL, skipCertVerify, maxResponseBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -280,7 +281,7 @@ func NewSarin(
|
||||
|
||||
var scriptHTTPClients []*scriptHTTPClient
|
||||
if !scriptChain.IsEmpty() {
|
||||
scriptHTTPClients, err = newScriptHTTPClients(jobsCtx, proxyURLs, workers)
|
||||
scriptHTTPClients, err = newScriptHTTPClients(jobsCtx, proxyURLs, workers, maxResponseBody)
|
||||
if err != nil {
|
||||
jobsCancel()
|
||||
return nil, err
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package sarin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"time"
|
||||
@@ -28,9 +31,9 @@ var _ script.HTTPDoer = (*scriptHTTPClient)(nil)
|
||||
// newScriptHTTPClients creates one client per proxy, in the same order as NewHostClients.
|
||||
// It can return the following errors:
|
||||
// - types.ProxyDialError
|
||||
func newScriptHTTPClients(ctx context.Context, proxies []url.URL, maxConns uint) ([]*scriptHTTPClient, error) {
|
||||
func newScriptHTTPClients(ctx context.Context, proxies []url.URL, maxConns uint, maxResponseBody uint64) ([]*scriptHTTPClient, error) {
|
||||
if len(proxies) == 0 {
|
||||
return []*scriptHTTPClient{newScriptHTTPClient(ctx, fasthttp.DialDualStackTimeout, scriptHTTPDefaultTimeout, maxConns)}, nil
|
||||
return []*scriptHTTPClient{newScriptHTTPClient(ctx, fasthttp.DialDualStackTimeout, scriptHTTPDefaultTimeout, maxConns, maxResponseBody)}, nil
|
||||
}
|
||||
|
||||
clients := make([]*scriptHTTPClient, 0, len(proxies))
|
||||
@@ -39,18 +42,19 @@ func newScriptHTTPClients(ctx context.Context, proxies []url.URL, maxConns uint)
|
||||
if err != nil {
|
||||
return nil, types.NewProxyDialError(proxy.String(), err)
|
||||
}
|
||||
clients = append(clients, newScriptHTTPClient(ctx, dial, scriptHTTPDefaultTimeout, maxConns))
|
||||
clients = append(clients, newScriptHTTPClient(ctx, dial, scriptHTTPDefaultTimeout, maxConns, maxResponseBody))
|
||||
}
|
||||
return clients, nil
|
||||
}
|
||||
|
||||
func newScriptHTTPClient(ctx context.Context, dial fasthttp.DialFuncWithTimeout, defaultTimeout time.Duration, maxConns uint) *scriptHTTPClient {
|
||||
func newScriptHTTPClient(ctx context.Context, dial fasthttp.DialFuncWithTimeout, defaultTimeout time.Duration, maxConns uint, maxResponseBody uint64) *scriptHTTPClient {
|
||||
newClient := func(insecure bool) *fasthttp.Client {
|
||||
// Read and write timeouts stay unset because they would cap the per-call timeout,
|
||||
// so dialWithDeadline is what bounds the TLS handshake.
|
||||
return &fasthttp.Client{
|
||||
DialTimeout: dialWithDeadline(dial),
|
||||
MaxConnsPerHost: safeUintToInt(maxConns),
|
||||
DialTimeout: dialWithDeadline(dial),
|
||||
MaxConnsPerHost: safeUintToInt(maxConns),
|
||||
MaxResponseBodySize: safeUint64ToInt(maxResponseBody),
|
||||
TLSConfig: &tls.Config{
|
||||
InsecureSkipVerify: insecure, //nolint:gosec
|
||||
},
|
||||
@@ -144,6 +148,8 @@ func (c *scriptHTTPClient) Do(r *script.HTTPRequest) (*script.HTTPResponse, erro
|
||||
}
|
||||
|
||||
req.SetTimeout(timeout)
|
||||
// A body limit streams the response, so a declared Content-Length is never allocated.
|
||||
resp.StreamBody = r.MaxBodySize > 0
|
||||
if r.MaxRedirects > 0 {
|
||||
err = client.DoRedirects(req, resp, r.MaxRedirects)
|
||||
} else {
|
||||
@@ -153,9 +159,47 @@ func (c *scriptHTTPClient) Do(r *script.HTTPRequest) (*script.HTTPResponse, erro
|
||||
return nil, types.NewScriptHTTPRequestError(r.Method, r.URL, err)
|
||||
}
|
||||
|
||||
body, err := readScriptBody(resp, r.MaxBodySize)
|
||||
if err != nil {
|
||||
return nil, types.NewScriptHTTPRequestError(r.Method, r.URL, err)
|
||||
}
|
||||
|
||||
return &script.HTTPResponse{
|
||||
Status: resp.StatusCode(),
|
||||
Headers: collectRespHeaders(resp),
|
||||
Body: string(resp.Body()),
|
||||
Body: body,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// readScriptBody reads the streamed body, refusing it once it passes maxBodySize.
|
||||
// It can return the following errors:
|
||||
// - types.ErrScriptHTTPBodyTooLarge
|
||||
func readScriptBody(resp *fasthttp.Response, maxBodySize int64) (string, error) {
|
||||
stream := resp.BodyStream()
|
||||
if stream == nil {
|
||||
return string(resp.Body()), nil
|
||||
}
|
||||
|
||||
var (
|
||||
body bytes.Buffer
|
||||
err error
|
||||
)
|
||||
if maxBodySize > 0 {
|
||||
// One byte past the limit tells an exact fit from an overflow.
|
||||
_, err = io.CopyN(&body, stream, maxBodySize+1)
|
||||
if err == nil {
|
||||
err = types.ErrScriptHTTPBodyTooLarge
|
||||
} else if errors.Is(err, io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
} else {
|
||||
_, err = io.Copy(&body, stream)
|
||||
}
|
||||
|
||||
// An unfinished stream makes fasthttp drop the connection instead of pooling it.
|
||||
_ = resp.CloseBodyStream()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return body.String(), nil
|
||||
}
|
||||
@@ -2,7 +2,9 @@ package sarin
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -31,6 +33,20 @@ func statusCodeToString(code int) string {
|
||||
return strconv.Itoa(code)
|
||||
}
|
||||
|
||||
// responseErrorKey drops the local address, whose port would mint a stats entry per connection.
|
||||
func responseErrorKey(err error) string {
|
||||
message := err.Error()
|
||||
|
||||
var opErr *net.OpError
|
||||
if !errors.As(err, &opErr) || opErr.Source == nil {
|
||||
return message
|
||||
}
|
||||
|
||||
withoutSource := *opErr
|
||||
withoutSource.Source = nil
|
||||
return strings.Replace(message, opErr.Error(), withoutSource.Error(), 1)
|
||||
}
|
||||
|
||||
// requestErrorKey keys a generation failure by its source instead of its message.
|
||||
// A script picks its own URLs and error texts, so the message would mint a new
|
||||
// stats entry per request. The full message still goes to the runtime log.
|
||||
@@ -135,7 +151,7 @@ func (s sarin) workerStatsWithDynamic(
|
||||
respDuration := time.Since(startTime)
|
||||
|
||||
if err != nil {
|
||||
s.responses.Add(err.Error(), respDuration)
|
||||
s.responses.Add(responseErrorKey(err), respDuration)
|
||||
} else {
|
||||
s.responses.Add(statusCodeToString(resp.StatusCode()), respDuration)
|
||||
sendRespLog(respDuration, resp)
|
||||
@@ -170,7 +186,7 @@ func (s sarin) workerStatsWithStatic(
|
||||
err := s.hostClients[nextClientIndex()].DoTimeout(req, resp, s.timeout)
|
||||
respDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
s.responses.Add(err.Error(), respDuration)
|
||||
s.responses.Add(responseErrorKey(err), respDuration)
|
||||
} else {
|
||||
s.responses.Add(statusCodeToString(resp.StatusCode()), respDuration)
|
||||
sendRespLog(respDuration, resp)
|
||||
|
||||
@@ -21,6 +21,7 @@ type HTTPRequest struct {
|
||||
Timeout time.Duration // zero means the doer's default timeout
|
||||
Insecure bool
|
||||
MaxRedirects int
|
||||
MaxBodySize int64 // zero means no limit
|
||||
}
|
||||
|
||||
// HTTPResponse is the response returned to a script by the http.* functions.
|
||||
@@ -66,6 +67,9 @@ const httpDefaultMethod = http.MethodGet
|
||||
// httpMaxRedirectsLimit is the highest maxRedirects a script may ask for.
|
||||
const httpMaxRedirectsLimit = 100
|
||||
|
||||
// httpMaxBodySizeLimit is the highest maxBodySize a script may ask for, 1 TiB.
|
||||
const httpMaxBodySizeLimit int64 = 1 << 40
|
||||
|
||||
// Option keys accepted by the http function.
|
||||
const (
|
||||
httpOptionMethod = "method"
|
||||
@@ -76,6 +80,7 @@ const (
|
||||
httpOptionTimeout = "timeout"
|
||||
httpOptionInsecure = "insecure"
|
||||
httpOptionMaxRedirects = "maxRedirects"
|
||||
httpOptionMaxBodySize = "maxBodySize"
|
||||
)
|
||||
|
||||
// httpBridge connects http to the doer. Calls only work while transform runs.
|
||||
@@ -118,6 +123,22 @@ func parseHTTPMaxRedirects(count float64) (int, error) {
|
||||
return int(count), nil
|
||||
}
|
||||
|
||||
// parseHTTPMaxBodySize checks that the body limit is a non-negative whole number.
|
||||
// It can return the following errors:
|
||||
// - types.ScriptHTTPOptionError
|
||||
func parseHTTPMaxBodySize(size float64) (int64, error) {
|
||||
if math.IsNaN(size) || size < 0 || size > float64(httpMaxBodySizeLimit) || size != math.Trunc(size) {
|
||||
return 0, types.NewScriptHTTPOptionError(
|
||||
httpOptionMaxBodySize,
|
||||
types.NewScriptTypeError(
|
||||
"a whole number of bytes between 0 and "+strconv.FormatInt(httpMaxBodySizeLimit, 10),
|
||||
strconv.FormatFloat(size, 'g', -1, 64),
|
||||
),
|
||||
)
|
||||
}
|
||||
return int64(size), nil
|
||||
}
|
||||
|
||||
// parseHTTPTimeout parses a Go duration string such as "500ms" or "2s".
|
||||
// It can return the following errors:
|
||||
// - types.ScriptHTTPOptionError
|
||||
|
||||
@@ -117,6 +117,16 @@ func (e *JsEngine) parseHTTPOptions(opts *goja.Object, req *HTTPRequest) error {
|
||||
if req.MaxRedirects, err = parseHTTPMaxRedirects(count); err != nil {
|
||||
return err
|
||||
}
|
||||
case httpOptionMaxBodySize:
|
||||
size, ok := numberValue(value)
|
||||
if !ok {
|
||||
return types.NewScriptHTTPOptionError(name, types.NewScriptTypeError("number", jsTypeName(value)))
|
||||
}
|
||||
|
||||
var err error
|
||||
if req.MaxBodySize, err = parseHTTPMaxBodySize(size); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return types.NewScriptHTTPOptionError(name, types.ErrScriptHTTPUnknownOption)
|
||||
}
|
||||
|
||||
@@ -106,6 +106,13 @@ func (e *LuaEngine) parseHTTPOptions(opts *lua.LTable, req *HTTPRequest) error {
|
||||
return
|
||||
}
|
||||
req.MaxRedirects, err = parseHTTPMaxRedirects(float64(count))
|
||||
case httpOptionMaxBodySize:
|
||||
size, ok := v.(lua.LNumber)
|
||||
if !ok {
|
||||
err = types.NewScriptHTTPOptionError(name, types.NewScriptTypeError("number", v.Type().String()))
|
||||
return
|
||||
}
|
||||
req.MaxBodySize, err = parseHTTPMaxBodySize(float64(size))
|
||||
default:
|
||||
err = types.NewScriptHTTPOptionError(name, types.ErrScriptHTTPUnknownOption)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,11 @@ import (
|
||||
|
||||
var (
|
||||
errNoError = errors.New("no error (internal)")
|
||||
|
||||
ErrByteSizeInvalid = errors.New(
|
||||
"invalid value for size, expected a whole number of bytes or a whole number with a unit: " +
|
||||
"B, KB, MB, GB, TB (1000 based) or KiB, MiB, GiB, TiB (1024 based), e.g. '1048576', '10MiB', '2MB'",
|
||||
)
|
||||
)
|
||||
|
||||
type FieldParseError struct {
|
||||
@@ -408,6 +413,7 @@ var (
|
||||
ErrScriptHTTPMethodEmpty = errors.New("http method cannot be empty")
|
||||
ErrScriptHTTPURLInvalid = errors.New("URL must be an absolute http or https URL")
|
||||
ErrScriptHTTPUnknownOption = errors.New("unknown option")
|
||||
ErrScriptHTTPBodyTooLarge = errors.New("response body exceeds maxBodySize")
|
||||
ErrScriptJSONCycle = errors.New("cannot encode a table that contains itself")
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user