feat: limit response body sizes

fasthttp allocates the body from the declared Content-Length before
reading it, so a server could exhaust memory by declaring a size it
never sends. The new max response body config bounds every response,
defaults to 10MiB and accepts sizes like 10MiB or 2MB. Scripts get the
same as a maxBodySize option on http().
This commit is contained in:
aykhans committed 2026-09-28 22:49:39 +04:00
1 parent 412a02fea3
commit f1d91e3d77
17 files changed
+452 -147

No files matched your search

+1 -1
View File
@@ -67,7 +67,7 @@ nix run github:aykhans/sarin/release -- -U http://example.com -r 100 -c 10
Install into your profile:
```sh
nix profile install github:aykhans/sarin/release
nix profile add github:aykhans/sarin/release
```
Or add it to your own flake via the overlay:
+2 -1
View File
@@ -51,7 +51,8 @@ func main() {
ctx,
combinedConfig.Methods, combinedConfig.URL, *combinedConfig.Timeout,
*combinedConfig.Concurrency, combinedConfig.Requests, combinedConfig.Duration,
*combinedConfig.Progress == config.ConfigProgressTypeBar, *combinedConfig.Insecure, combinedConfig.Params, combinedConfig.Headers,
*combinedConfig.Progress == config.ConfigProgressTypeBar, *combinedConfig.Insecure, *combinedConfig.MaxResponseBody,
combinedConfig.Params, combinedConfig.Headers,
combinedConfig.Cookies, combinedConfig.Bodies, combinedConfig.Proxies, combinedConfig.Values,
*combinedConfig.Output != config.ConfigOutputTypeNone,
*combinedConfig.DryRun, *combinedConfig.LogLevel, *combinedConfig.LogFile,
+41 -26
View File
@@ -14,32 +14,33 @@ Use `-s` or `--show-config` to see the final merged configuration before sending
> **Note:** For CLI flags with `string / []string` type, the flag can be used once with a single value or multiple times to provide multiple values.
| Name | YAML | CLI | ENV | Default | Description |
| --------------------------- | ----------------------------------- | -------------------------------------------- | -------------------------------- | ------- | ---------------------------- |
| [Help](#help) | - | `-help` / `-h` | - | - | Show help message |
| [Version](#version) | - | `-version` / `-v` | - | - | Show version and build info |
| [Show Config](#show-config) | `showConfig`<br>(boolean) | `-show-config` / `-s`<br>(boolean) | `SARIN_SHOW_CONFIG`<br>(boolean) | `false` | Show merged configuration |
| [Config File](#config-file) | `configFile`<br>(string / []string) | `-config-file` / `-f`<br>(string / []string) | `SARIN_CONFIG_FILE`<br>(string) | - | Path to config file(s) |
| [URL](#url) | `url`<br>(string) | `-url` / `-U`<br>(string) | `SARIN_URL`<br>(string) | - | Target URL (HTTP/HTTPS) |
| [Method](#method) | `method`<br>(string / []string) | `-method` / `-M`<br>(string / []string) | `SARIN_METHOD`<br>(string) | `GET` | HTTP method(s) |
| [Timeout](#timeout) | `timeout`<br>(duration) | `-timeout` / `-T`<br>(duration) | `SARIN_TIMEOUT`<br>(duration) | `10s` | Request timeout |
| [Concurrency](#concurrency) | `concurrency`<br>(number) | `-concurrency` / `-c`<br>(number) | `SARIN_CONCURRENCY`<br>(number) | `1` | Number of concurrent workers |
| [Requests](#requests) | `requests`<br>(number) | `-requests` / `-r`<br>(number) | `SARIN_REQUESTS`<br>(number) | - | Total requests to send |
| [Duration](#duration) | `duration`<br>(duration) | `-duration` / `-d`<br>(duration) | `SARIN_DURATION`<br>(duration) | - | Test duration |
| [Log Level](#log-level) | `logLevel`<br>(string) | `-log-level` / `-l`<br>(string) | `SARIN_LOG_LEVEL`<br>(string) | `error` | Runtime log levels to emit |
| [Log File](#log-file) | `logFile`<br>(string) | `-log-file` / `-w`<br>(string) | `SARIN_LOG_FILE`<br>(string) | - | Write runtime logs to a file |
| [Progress](#progress) | `progress`<br>(string) | `-progress` / `-p`<br>(string) | `SARIN_PROGRESS`<br>(string) | `bar` | Progress display (bar/none) |
| [Output](#output) | `output`<br>(string) | `-output` / `-o`<br>(string) | `SARIN_OUTPUT`<br>(string) | `table` | Output format for stats |
| [Dry Run](#dry-run) | `dryRun`<br>(boolean) | `-dry-run` / `-z`<br>(boolean) | `SARIN_DRY_RUN`<br>(boolean) | `false` | Generate without sending |
| [Insecure](#insecure) | `insecure`<br>(boolean) | `-insecure` / `-I`<br>(boolean) | `SARIN_INSECURE`<br>(boolean) | `false` | Skip TLS verification |
| [Body](#body) | `body`<br>(string / []string) | `-body` / `-B`<br>(string / []string) | `SARIN_BODY`<br>(string) | - | Request body |
| [Params](#params) | `params`<br>(object) | `-param` / `-P`<br>(string / []string) | `SARIN_PARAM`<br>(string) | - | URL query parameters |
| [Headers](#headers) | `headers`<br>(object) | `-header` / `-H`<br>(string / []string) | `SARIN_HEADER`<br>(string) | - | HTTP headers |
| [Cookies](#cookies) | `cookies`<br>(object) | `-cookie` / `-C`<br>(string / []string) | `SARIN_COOKIE`<br>(string) | - | HTTP cookies |
| [Proxy](#proxy) | `proxy`<br>(string / []string) | `-proxy` / `-X`<br>(string / []string) | `SARIN_PROXY`<br>(string) | - | Proxy URL(s) |
| [Values](#values) | `values`<br>(string / []string) | `-values` / `-V`<br>(string / []string) | `SARIN_VALUES`<br>(string) | - | Template values (key=value) |
| [Lua](#lua) | `lua`<br>(string / []string) | `-lua`<br>(string / []string) | `SARIN_LUA`<br>(string) | - | Lua script(s) |
| [Js](#js) | `js`<br>(string / []string) | `-js`<br>(string / []string) | `SARIN_JS`<br>(string) | - | JavaScript script(s) |
| Name | YAML | CLI | ENV | Default | Description |
| --------------------------------------- | ----------------------------------- | -------------------------------------------- | ----------------------------------- | ------- | ----------------------------- |
| [Help](#help) | - | `-help` / `-h` | - | - | Show help message |
| [Version](#version) | - | `-version` / `-v` | - | - | Show version and build info |
| [Show Config](#show-config) | `showConfig`<br>(boolean) | `-show-config` / `-s`<br>(boolean) | `SARIN_SHOW_CONFIG`<br>(boolean) | `false` | Show merged configuration |
| [Config File](#config-file) | `configFile`<br>(string / []string) | `-config-file` / `-f`<br>(string / []string) | `SARIN_CONFIG_FILE`<br>(string) | - | Path to config file(s) |
| [URL](#url) | `url`<br>(string) | `-url` / `-U`<br>(string) | `SARIN_URL`<br>(string) | - | Target URL (HTTP/HTTPS) |
| [Method](#method) | `method`<br>(string / []string) | `-method` / `-M`<br>(string / []string) | `SARIN_METHOD`<br>(string) | `GET` | HTTP method(s) |
| [Timeout](#timeout) | `timeout`<br>(duration) | `-timeout` / `-T`<br>(duration) | `SARIN_TIMEOUT`<br>(duration) | `10s` | Request timeout |
| [Max Response Body](#max-response-body) | `maxResponseBody`<br>(size) | `-max-response-body` / `-S`<br>(size) | `SARIN_MAX_RESPONSE_BODY`<br>(size) | `10MiB` | Largest response body to read |
| [Concurrency](#concurrency) | `concurrency`<br>(number) | `-concurrency` / `-c`<br>(number) | `SARIN_CONCURRENCY`<br>(number) | `1` | Number of concurrent workers |
| [Requests](#requests) | `requests`<br>(number) | `-requests` / `-r`<br>(number) | `SARIN_REQUESTS`<br>(number) | - | Total requests to send |
| [Duration](#duration) | `duration`<br>(duration) | `-duration` / `-d`<br>(duration) | `SARIN_DURATION`<br>(duration) | - | Test duration |
| [Log Level](#log-level) | `logLevel`<br>(string) | `-log-level` / `-l`<br>(string) | `SARIN_LOG_LEVEL`<br>(string) | `error` | Runtime log levels to emit |
| [Log File](#log-file) | `logFile`<br>(string) | `-log-file` / `-w`<br>(string) | `SARIN_LOG_FILE`<br>(string) | - | Write runtime logs to a file |
| [Progress](#progress) | `progress`<br>(string) | `-progress` / `-p`<br>(string) | `SARIN_PROGRESS`<br>(string) | `bar` | Progress display (bar/none) |
| [Output](#output) | `output`<br>(string) | `-output` / `-o`<br>(string) | `SARIN_OUTPUT`<br>(string) | `table` | Output format for stats |
| [Dry Run](#dry-run) | `dryRun`<br>(boolean) | `-dry-run` / `-z`<br>(boolean) | `SARIN_DRY_RUN`<br>(boolean) | `false` | Generate without sending |
| [Insecure](#insecure) | `insecure`<br>(boolean) | `-insecure` / `-I`<br>(boolean) | `SARIN_INSECURE`<br>(boolean) | `false` | Skip TLS verification |
| [Body](#body) | `body`<br>(string / []string) | `-body` / `-B`<br>(string / []string) | `SARIN_BODY`<br>(string) | - | Request body |
| [Params](#params) | `params`<br>(object) | `-param` / `-P`<br>(string / []string) | `SARIN_PARAM`<br>(string) | - | URL query parameters |
| [Headers](#headers) | `headers`<br>(object) | `-header` / `-H`<br>(string / []string) | `SARIN_HEADER`<br>(string) | - | HTTP headers |
| [Cookies](#cookies) | `cookies`<br>(object) | `-cookie` / `-C`<br>(string / []string) | `SARIN_COOKIE`<br>(string) | - | HTTP cookies |
| [Proxy](#proxy) | `proxy`<br>(string / []string) | `-proxy` / `-X`<br>(string / []string) | `SARIN_PROXY`<br>(string) | - | Proxy URL(s) |
| [Values](#values) | `values`<br>(string / []string) | `-values` / `-V`<br>(string / []string) | `SARIN_VALUES`<br>(string) | - | Template values (key=value) |
| [Lua](#lua) | `lua`<br>(string / []string) | `-lua`<br>(string / []string) | `SARIN_LUA`<br>(string) | - | Lua script(s) |
| [Js](#js) | `js`<br>(string / []string) | `-js`<br>(string / []string) | `SARIN_JS`<br>(string) | - | JavaScript script(s) |
---
@@ -168,6 +169,18 @@ Valid time units: `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`
**Examples:** `5s`, `300ms`, `1m20s`
## Max Response Body
Largest response body sarin reads. Defaults to `10MiB`, and `0` removes the limit.
The value is a whole number of bytes, with or without a unit: `B`, `KB`, `MB`, `GB`, `TB` are powers of 1000, `KiB`, `MiB`, `GiB`, `TiB` are powers of 1024. Case is ignored and a space before the unit is allowed. The largest accepted value is `1TiB`, or `2147483647` on a 32-bit platform.
**Examples:** `1048576`, `10MiB`, `2MB`, `512KiB`
A server can declare a `Content-Length` far larger than it sends, and the body is allocated from that declared length, so without a limit a single response can exhaust memory. A response above the limit fails with `body size exceeds the given limit` and is counted like any other error.
The limit applies per response in flight, so the worst case is roughly the limit times the concurrency. It also applies to the requests scripts make, unless the script passes its own `maxBodySize` (see [HTTP Requests in Scripts](#http-requests-in-scripts)).
## Concurrency
Number of concurrent workers. Must be between 1 and 100,000,000. Defaults to `1`.
@@ -583,6 +596,7 @@ http(url, opts)
| `timeout` | duration string (`"500ms"`) | `30s` | Request timeout (the `timeout` config does not apply), applied to each hop when redirects are followed |
| `insecure` | boolean | `false` | Skip TLS verification (the `insecure` config does not apply) |
| `maxRedirects` | number | `0` | How many redirects to follow, `0` follows none, maximum 100 |
| `maxBodySize` | number | - | Largest response body to accept in bytes, maximum 1099511627776 |
Unknown options and values of the wrong type raise an error.
@@ -605,6 +619,7 @@ In Lua, call the helpers as `res:header("name")` or `res.header("name")`.
- **Same proxy:** when proxies are configured, a script's requests go through the same proxy as the main request they belong to.
- **Status codes:** non-2xx responses are returned normally; check `status` in the script.
- **Errors:** network failures and timeouts raise a script error. The main request is not sent and the error is counted in the results. Scripts can catch it with `pcall` (Lua) or `try`/`catch` (JavaScript).
- **Response bodies:** without `maxBodySize` the [Max Response Body](#max-response-body) config applies, as it does to the main request. With `maxBodySize` the body is read as it arrives, so a `Content-Length` the server never sends costs nothing, the limit is exact, and it may be larger than the config. Passing the limit raises an error instead of truncating the body.
- **Only inside `transform`:** `http` works anywhere while `transform` runs, including in helper functions it calls. Calling it at the top level of a script raises an error, because that code also runs during config validation and when each worker starts.
- **Dry run:** script requests are still sent in dry-run mode.
- **Not measured:** script requests are not included in the results.
+123
View File
@@ -0,0 +1,123 @@
package config
import (
"flag"
"fmt"
"math"
"strconv"
"strings"
"go.aykhans.me/sarin/internal/types"
"go.yaml.in/yaml/v4"
)
var (
_ flag.Value = (*byteSize)(nil)
_ yaml.Unmarshaler = (*byteSize)(nil)
)
// byteSize is a size in bytes that reads and prints unit suffixes such as 10MiB.
type byteSize uint64
// byteSizeUnits run from the largest scale down, so formatting picks the largest unit
// that fits. A bare "K" is out: the common libraries disagree on whether it counts
// 1000 or 1024 bytes.
var byteSizeUnits = []struct {
suffix string
scale uint64
}{
{"TiB", 1 << 40},
{"TB", 1000 * 1000 * 1000 * 1000},
{"GiB", 1 << 30},
{"GB", 1000 * 1000 * 1000},
{"MiB", 1 << 20},
{"MB", 1000 * 1000},
{"KiB", 1 << 10},
{"KB", 1000},
{"B", 1},
}
// parseByteSize reads a byte count, with or without a unit suffix.
func parseByteSize(value string) (uint64, error) {
text := strings.TrimSpace(value)
// The suffix is split off by index, so no case mapping can shift the digits.
digitsEnd := len(text)
for digitsEnd > 0 && isASCIILetter(text[digitsEnd-1]) {
digitsEnd--
}
scale := uint64(1)
if suffix := text[digitsEnd:]; suffix != "" {
unitScale, ok := byteSizeScale(suffix)
if !ok {
return 0, types.ErrByteSizeInvalid
}
scale = unitScale
}
size, err := strconv.ParseUint(strings.TrimSpace(text[:digitsEnd]), 10, 64)
if err != nil {
return 0, types.ErrByteSizeInvalid
}
if size > math.MaxUint64/scale {
return 0, types.ErrByteSizeInvalid
}
return size * scale, nil
}
// byteSizeScale looks up a unit suffix, ignoring case.
func byteSizeScale(suffix string) (uint64, bool) {
for _, unit := range byteSizeUnits {
if strings.EqualFold(suffix, unit.suffix) {
return unit.scale, true
}
}
return 0, false
}
func isASCIILetter(char byte) bool {
return (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z')
}
// formatByteSize renders a byte count so that parseByteSize reads back the same value.
func formatByteSize(size uint64) string {
for _, unit := range byteSizeUnits {
if unit.scale > 1 && size >= unit.scale && size%unit.scale == 0 {
return strconv.FormatUint(size/unit.scale, 10) + unit.suffix
}
}
return strconv.FormatUint(size, 10)
}
func (size byteSize) String() string {
return formatByteSize(uint64(size))
}
// Set fills the size from a command line value.
func (size *byteSize) Set(value string) error {
parsed, err := parseByteSize(value)
if err != nil {
return err
}
*size = byteSize(parsed)
return nil
}
// UnmarshalYAML accepts both a YAML integer and a string with a unit.
func (size *byteSize) UnmarshalYAML(node *yaml.Node) error {
if node.Kind != yaml.ScalarNode {
return fmt.Errorf("expected a size, but got %s", node.Tag)
}
if node.Tag == "!!int" {
var parsed uint64
if err := node.Decode(&parsed); err != nil {
return types.ErrByteSizeInvalid
}
*size = byteSize(parsed)
return nil
}
return size.Set(node.Value)
}
+44 -36
View File
@@ -20,32 +20,33 @@ Simple usage:
Flags:
General Config:
-h, -help Help for sarin
-v, -version Version for sarin
-s, -show-config bool Show the final config after parsing all sources (default %v)
-f, -config-file []string Path to the config file (local file / http URL)
-c, -concurrency uint Number of concurrent requests (default %d)
-r, -requests uint Number of total requests
-d, -duration time Maximum duration for the test (e.g. 30s, 1m, 5h)
-l, -log-level string Runtime log levels to emit, comma-separated (possible values: info, error) (default %s)
-w, -log-file string Write runtime logs to this file instead of the terminal/stderr
-p, -progress string Progress display (possible values: bar, none) (default '%v')
-o, -output string Output format (possible values: table, json, yaml, none) (default '%v')
-z, -dry-run bool Run without sending requests (default %v)
-h, -help Help for sarin
-v, -version Version for sarin
-s, -show-config bool Show the final config after parsing all sources (default %v)
-f, -config-file []string Path to the config file (local file / http URL)
-c, -concurrency uint Number of concurrent requests (default %d)
-r, -requests uint Number of total requests
-d, -duration time Maximum duration for the test (e.g. 30s, 1m, 5h)
-l, -log-level string Runtime log levels to emit, comma-separated (possible values: info, error) (default %s)
-w, -log-file string Write runtime logs to this file instead of the terminal/stderr
-p, -progress string Progress display (possible values: bar, none) (default '%v')
-o, -output string Output format (possible values: table, json, yaml, none) (default '%v')
-z, -dry-run bool Run without sending requests (default %v)
Request Config:
-U, -url string Target URL for the request
-M, -method []string HTTP method for the request (default %s)
-B, -body []string Body for the request (e.g. "body text")
-P, -param []string URL parameter for the request (e.g. "key1=value1")
-H, -header []string Header for the request (e.g. "key1: value1")
-C, -cookie []string Cookie for the request (e.g. "key1=value1")
-X, -proxy []string Proxy for the request (e.g. "http://proxy.example.com:8080")
-V, -values []string List of values for templating (e.g. "key1=value1")
-T, -timeout time Timeout for the request (e.g. 400ms, 3s, 1m10s) (default %v)
-I, -insecure bool Skip SSL/TLS certificate verification (default %v)
-lua []string Lua script for request transformation (inline or @file/@url)
-js []string JavaScript script for request transformation (inline or @file/@url)`
-U, -url string Target URL for the request
-M, -method []string HTTP method for the request (default %s)
-B, -body []string Body for the request (e.g. "body text")
-P, -param []string URL parameter for the request (e.g. "key1=value1")
-H, -header []string Header for the request (e.g. "key1: value1")
-C, -cookie []string Cookie for the request (e.g. "key1=value1")
-X, -proxy []string Proxy for the request (e.g. "http://proxy.example.com:8080")
-V, -values []string List of values for templating (e.g. "key1=value1")
-T, -timeout time Timeout for the request (e.g. 400ms, 3s, 1m10s) (default %v)
-S, -max-response-body size Largest response body to read (e.g. 10MiB, 2MB), 0 for no limit (default %s)
-I, -insecure bool Skip SSL/TLS certificate verification (default %v)
-lua []string Lua script for request transformation (inline or @file/@url)
-js []string JavaScript script for request transformation (inline or @file/@url)`
var _ IParser = ConfigCLIParser{}
@@ -97,18 +98,19 @@ func (parser ConfigCLIParser) Parse() (*Config, error) {
dryRun bool
// Request config
urlInput string
methods = stringSliceArg{}
bodies = stringSliceArg{}
params = stringSliceArg{}
headers = stringSliceArg{}
cookies = stringSliceArg{}
proxies = stringSliceArg{}
values = stringSliceArg{}
timeout time.Duration
insecure bool
luaScripts = stringSliceArg{}
jsScripts = stringSliceArg{}
urlInput string
methods = stringSliceArg{}
bodies = stringSliceArg{}
params = stringSliceArg{}
headers = stringSliceArg{}
cookies = stringSliceArg{}
proxies = stringSliceArg{}
values = stringSliceArg{}
timeout time.Duration
maxResponseBody byteSize
insecure bool
luaScripts = stringSliceArg{}
jsScripts = stringSliceArg{}
)
{
@@ -174,6 +176,9 @@ func (parser ConfigCLIParser) Parse() (*Config, error) {
flagSet.DurationVar(&timeout, "timeout", 0, "Timeout for the request (e.g. 400ms, 15s, 1m10s)")
flagSet.DurationVar(&timeout, "T", 0, "Timeout for the request (e.g. 400ms, 15s, 1m10s)")
flagSet.Var(&maxResponseBody, "max-response-body", "Largest response body to read (e.g. 1048576, 10MiB, 2MB), 0 for no limit")
flagSet.Var(&maxResponseBody, "S", "Largest response body to read (e.g. 1048576, 10MiB, 2MB), 0 for no limit")
flagSet.BoolVar(&insecure, "insecure", false, "Skip SSL/TLS certificate verification")
flagSet.BoolVar(&insecure, "I", false, "Skip SSL/TLS certificate verification")
@@ -258,6 +263,8 @@ func (parser ConfigCLIParser) Parse() (*Config, error) {
config.Values = append(config.Values, values...)
case "timeout", "T":
config.Timeout = new(timeout)
case "max-response-body", "S":
config.MaxResponseBody = new(uint64(maxResponseBody))
case "insecure", "I":
config.Insecure = new(insecure)
case "lua":
@@ -286,6 +293,7 @@ func (parser ConfigCLIParser) PrintHelp() {
Defaults.Method,
Defaults.RequestTimeout,
formatByteSize(Defaults.MaxResponseBody),
Defaults.Insecure,
)
}
+65 -42
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"math"
"net/url"
"os"
"path/filepath"
@@ -27,27 +28,29 @@ import (
)
var Defaults = struct {
UserAgent string
Method string
RequestTimeout time.Duration
Concurrency uint
ShowConfig bool
Progress ConfigProgressType
Insecure bool
Output ConfigOutputType
DryRun bool
LogLevel string
UserAgent string
Method string
RequestTimeout time.Duration
MaxResponseBody uint64
Concurrency uint
ShowConfig bool
Progress ConfigProgressType
Insecure bool
Output ConfigOutputType
DryRun bool
LogLevel string
}{
UserAgent: "Sarin/" + version.Version,
Method: "GET",
RequestTimeout: time.Second * 10,
Concurrency: 1,
ShowConfig: false,
Progress: ConfigProgressTypeBar,
Insecure: false,
Output: ConfigOutputTypeTable,
DryRun: false,
LogLevel: "error",
UserAgent: "Sarin/" + version.Version,
Method: "GET",
RequestTimeout: time.Second * 10,
MaxResponseBody: 10 << 20, // 10 MiB
Concurrency: 1,
ShowConfig: false,
Progress: ConfigProgressTypeBar,
Insecure: false,
Output: ConfigOutputTypeTable,
DryRun: false,
LogLevel: "error",
}
var (
@@ -56,6 +59,8 @@ var (
ValidLogLevels = []string{"info", "error"}
)
const maxResponseBodyLimit = min(1<<40, math.MaxInt) // 1 TiB on 64 bit, 2 GiB - 1 on 32 bit
var (
StyleYellow = lipgloss.NewStyle().Foreground(lipgloss.Color("220"))
StyleRed = lipgloss.NewStyle().Foreground(lipgloss.Color("196"))
@@ -82,28 +87,29 @@ var (
)
type Config struct {
ShowConfig *bool `yaml:"showConfig,omitempty"`
Files []types.ConfigFile `yaml:"files,omitempty"`
Methods []string `yaml:"methods,omitempty"`
URL *url.URL `yaml:"url,omitempty"`
Timeout *time.Duration `yaml:"timeout,omitempty"`
Concurrency *uint `yaml:"concurrency,omitempty"`
Requests *uint64 `yaml:"requests,omitempty"`
Duration *time.Duration `yaml:"duration,omitempty"`
Progress *ConfigProgressType `yaml:"progress,omitempty"`
Output *ConfigOutputType `yaml:"output,omitempty"`
Insecure *bool `yaml:"insecure,omitempty"`
DryRun *bool `yaml:"dryRun,omitempty"`
Params types.Params `yaml:"params,omitempty"`
Headers types.Headers `yaml:"headers,omitempty"`
Cookies types.Cookies `yaml:"cookies,omitempty"`
Bodies []string `yaml:"bodies,omitempty"`
Proxies types.Proxies `yaml:"proxies,omitempty"`
Values []string `yaml:"values,omitempty"`
Lua []string `yaml:"lua,omitempty"`
Js []string `yaml:"js,omitempty"`
LogLevel *string `yaml:"logLevel,omitempty"`
LogFile *string `yaml:"logFile,omitempty"`
ShowConfig *bool `yaml:"showConfig,omitempty"`
Files []types.ConfigFile `yaml:"files,omitempty"`
Methods []string `yaml:"methods,omitempty"`
URL *url.URL `yaml:"url,omitempty"`
Timeout *time.Duration `yaml:"timeout,omitempty"`
MaxResponseBody *uint64 `yaml:"maxResponseBody,omitempty"`
Concurrency *uint `yaml:"concurrency,omitempty"`
Requests *uint64 `yaml:"requests,omitempty"`
Duration *time.Duration `yaml:"duration,omitempty"`
Progress *ConfigProgressType `yaml:"progress,omitempty"`
Output *ConfigOutputType `yaml:"output,omitempty"`
Insecure *bool `yaml:"insecure,omitempty"`
DryRun *bool `yaml:"dryRun,omitempty"`
Params types.Params `yaml:"params,omitempty"`
Headers types.Headers `yaml:"headers,omitempty"`
Cookies types.Cookies `yaml:"cookies,omitempty"`
Bodies []string `yaml:"bodies,omitempty"`
Proxies types.Proxies `yaml:"proxies,omitempty"`
Values []string `yaml:"values,omitempty"`
Lua []string `yaml:"lua,omitempty"`
Js []string `yaml:"js,omitempty"`
LogLevel *string `yaml:"logLevel,omitempty"`
LogFile *string `yaml:"logFile,omitempty"`
}
func (config Config) MarshalYAML() (any, error) {
@@ -177,6 +183,9 @@ func (config Config) MarshalYAML() (any, error) {
if config.Timeout != nil {
addField(content, "timeout", toNode(*config.Timeout), "")
}
if config.MaxResponseBody != nil {
addField(content, "maxResponseBody", toNode(formatByteSize(*config.MaxResponseBody)), "")
}
if config.Concurrency != nil {
addField(content, "concurrency", toNode(*config.Concurrency), "")
}
@@ -301,6 +310,9 @@ func (config *Config) Merge(newConfig *Config) {
if newConfig.Timeout != nil {
config.Timeout = newConfig.Timeout
}
if newConfig.MaxResponseBody != nil {
config.MaxResponseBody = newConfig.MaxResponseBody
}
if newConfig.Concurrency != nil {
config.Concurrency = newConfig.Concurrency
}
@@ -379,6 +391,9 @@ func (config *Config) SetDefaults() {
if config.Timeout == nil {
config.Timeout = &Defaults.RequestTimeout
}
if config.MaxResponseBody == nil {
config.MaxResponseBody = &Defaults.MaxResponseBody
}
if config.Concurrency == nil {
config.Concurrency = new(Defaults.Concurrency)
}
@@ -454,6 +469,14 @@ func (config Config) Validate() error {
validationErrors = append(validationErrors, types.NewFieldValidationError("Timeout", "0", errors.New("timeout must be greater than 0")))
}
if config.MaxResponseBody != nil && *config.MaxResponseBody > maxResponseBodyLimit {
validationErrors = append(validationErrors, types.NewFieldValidationError(
"MaxResponseBody",
formatByteSize(*config.MaxResponseBody),
errors.New("max response body must not exceed "+formatByteSize(maxResponseBodyLimit)),
))
}
if config.ShowConfig == nil {
validationErrors = append(validationErrors, types.NewFieldValidationError("ShowConfig", "", errors.New("showConfig field is required")))
}
+16
View File
@@ -195,6 +195,22 @@ func (parser ConfigENVParser) Parse() (*Config, error) {
}
}
if maxResponseBody := parser.getEnv("MAX_RESPONSE_BODY"); maxResponseBody != "" {
maxResponseBodyParsed, err := parseByteSize(maxResponseBody)
if err != nil {
fieldParseErrors = append(
fieldParseErrors,
types.NewFieldParseError(
parser.getFullEnvName("MAX_RESPONSE_BODY"),
maxResponseBody,
err,
),
)
} else {
config.MaxResponseBody = &maxResponseBodyParsed
}
}
if insecure := parser.getEnv("INSECURE"); insecure != "" {
insecureParsed, err := utilsParse.ParseString[bool](insecure)
if err != nil {
+26 -22
View File
@@ -192,28 +192,29 @@ func (kv *keyValuesField) unmarshalMapping(node *yaml.Node) error {
}
type configYAML struct {
ShowConfig *bool `yaml:"showConfig"`
ConfigFiles stringOrSliceField `yaml:"configFile"`
Concurrency *uint `yaml:"concurrency"`
RequestCount *uint64 `yaml:"requests"`
Duration *time.Duration `yaml:"duration"`
LogLevel *string `yaml:"logLevel"`
LogFile *string `yaml:"logFile"`
Progress *string `yaml:"progress"`
Output *string `yaml:"output"`
DryRun *bool `yaml:"dryRun"`
URL *string `yaml:"url"`
Method stringOrSliceField `yaml:"method"`
Bodies stringOrSliceField `yaml:"body"`
Params keyValuesField `yaml:"params"`
Headers keyValuesField `yaml:"headers"`
Cookies keyValuesField `yaml:"cookies"`
Proxies stringOrSliceField `yaml:"proxy"`
Values stringOrSliceField `yaml:"values"`
Timeout *time.Duration `yaml:"timeout"`
Insecure *bool `yaml:"insecure"`
Lua stringOrSliceField `yaml:"lua"`
Js stringOrSliceField `yaml:"js"`
ShowConfig *bool `yaml:"showConfig"`
ConfigFiles stringOrSliceField `yaml:"configFile"`
Concurrency *uint `yaml:"concurrency"`
RequestCount *uint64 `yaml:"requests"`
Duration *time.Duration `yaml:"duration"`
LogLevel *string `yaml:"logLevel"`
LogFile *string `yaml:"logFile"`
Progress *string `yaml:"progress"`
Output *string `yaml:"output"`
DryRun *bool `yaml:"dryRun"`
URL *string `yaml:"url"`
Method stringOrSliceField `yaml:"method"`
Bodies stringOrSliceField `yaml:"body"`
Params keyValuesField `yaml:"params"`
Headers keyValuesField `yaml:"headers"`
Cookies keyValuesField `yaml:"cookies"`
Proxies stringOrSliceField `yaml:"proxy"`
Values stringOrSliceField `yaml:"values"`
Timeout *time.Duration `yaml:"timeout"`
MaxResponseBody *byteSize `yaml:"maxResponseBody"`
Insecure *bool `yaml:"insecure"`
Lua stringOrSliceField `yaml:"lua"`
Js stringOrSliceField `yaml:"js"`
}
// ParseYAML parses YAML config file arguments into a Config object.
@@ -290,6 +291,9 @@ func (parser ConfigFileParser) ParseYAML(data []byte) (*Config, error) {
config.Values = append(config.Values, parsedData.Values...)
config.Timeout = parsedData.Timeout
if parsedData.MaxResponseBody != nil {
config.MaxResponseBody = new(uint64(*parsedData.MaxResponseBody))
}
config.Insecure = parsedData.Insecure
config.Lua = append(config.Lua, parsedData.Lua...)
config.Js = append(config.Js, parsedData.Js...)
+3 -8
View File
@@ -5,7 +5,6 @@ import (
"context"
"crypto/tls"
"encoding/base64"
"math"
"net"
"net/http"
"net/url"
@@ -20,13 +19,6 @@ import (
// clientIndexGenerator returns the index of the host client (and proxy) to use next.
type clientIndexGenerator func() int
func safeUintToInt(u uint) int {
if u > math.MaxInt {
return math.MaxInt
}
return int(u)
}
// NewHostClients creates a list of fasthttp.HostClient instances for the given proxies.
// If no proxies are provided, a single client without a proxy is returned.
// It can return the following errors:
@@ -38,6 +30,7 @@ func NewHostClients(
maxConns uint,
requestURL *url.URL,
skipVerify bool,
maxResponseBody uint64,
) ([]*fasthttp.HostClient, error) {
isTLS := requestURL.Scheme == "https"
@@ -66,6 +59,7 @@ func NewHostClients(
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
MaxResponseBodySize: safeUint64ToInt(maxResponseBody),
DisableHeaderNamesNormalizing: true,
DisablePathNormalizing: true,
NoDefaultUserAgentHeader: true,
@@ -88,6 +82,7 @@ func NewHostClients(
MaxConnDuration: timeout,
WriteTimeout: timeout,
ReadTimeout: timeout,
MaxResponseBodySize: safeUint64ToInt(maxResponseBody),
DisableHeaderNamesNormalizing: true,
DisablePathNormalizing: true,
NoDefaultUserAgentHeader: true,
+15
View File
@@ -1,6 +1,7 @@
package sarin
import (
"math"
"math/rand/v2"
"time"
)
@@ -19,3 +20,17 @@ func firstOrEmpty(values []string) string {
}
return values[0]
}
func safeUintToInt(u uint) int {
if u > math.MaxInt {
return math.MaxInt
}
return int(u)
}
func safeUint64ToInt(u uint64) int {
if u > math.MaxInt {
return math.MaxInt
}
return int(u)
}
+3 -2
View File
@@ -225,6 +225,7 @@ func NewSarin(
totalDuration *time.Duration,
showProgress bool,
skipCertVerify bool,
maxResponseBody uint64,
params types.Params,
headers types.Headers,
cookies types.Cookies,
@@ -258,7 +259,7 @@ func NewSarin(
proxyURLs[i] = url.URL(proxy)
}
hostClients, err := NewHostClients(ctx, timeout, proxyURLs, workers, requestURL, skipCertVerify)
hostClients, err := NewHostClients(ctx, timeout, proxyURLs, workers, requestURL, skipCertVerify, maxResponseBody)
if err != nil {
return nil, err
}
@@ -280,7 +281,7 @@ func NewSarin(
var scriptHTTPClients []*scriptHTTPClient
if !scriptChain.IsEmpty() {
scriptHTTPClients, err = newScriptHTTPClients(jobsCtx, proxyURLs, workers)
scriptHTTPClients, err = newScriptHTTPClients(jobsCtx, proxyURLs, workers, maxResponseBody)
if err != nil {
jobsCancel()
return nil, err
+51 -7
View File
@@ -1,8 +1,11 @@
package sarin
import (
"bytes"
"context"
"crypto/tls"
"errors"
"io"
"net"
"net/url"
"time"
@@ -28,9 +31,9 @@ var _ script.HTTPDoer = (*scriptHTTPClient)(nil)
// newScriptHTTPClients creates one client per proxy, in the same order as NewHostClients.
// It can return the following errors:
// - types.ProxyDialError
func newScriptHTTPClients(ctx context.Context, proxies []url.URL, maxConns uint) ([]*scriptHTTPClient, error) {
func newScriptHTTPClients(ctx context.Context, proxies []url.URL, maxConns uint, maxResponseBody uint64) ([]*scriptHTTPClient, error) {
if len(proxies) == 0 {
return []*scriptHTTPClient{newScriptHTTPClient(ctx, fasthttp.DialDualStackTimeout, scriptHTTPDefaultTimeout, maxConns)}, nil
return []*scriptHTTPClient{newScriptHTTPClient(ctx, fasthttp.DialDualStackTimeout, scriptHTTPDefaultTimeout, maxConns, maxResponseBody)}, nil
}
clients := make([]*scriptHTTPClient, 0, len(proxies))
@@ -39,18 +42,19 @@ func newScriptHTTPClients(ctx context.Context, proxies []url.URL, maxConns uint)
if err != nil {
return nil, types.NewProxyDialError(proxy.String(), err)
}
clients = append(clients, newScriptHTTPClient(ctx, dial, scriptHTTPDefaultTimeout, maxConns))
clients = append(clients, newScriptHTTPClient(ctx, dial, scriptHTTPDefaultTimeout, maxConns, maxResponseBody))
}
return clients, nil
}
func newScriptHTTPClient(ctx context.Context, dial fasthttp.DialFuncWithTimeout, defaultTimeout time.Duration, maxConns uint) *scriptHTTPClient {
func newScriptHTTPClient(ctx context.Context, dial fasthttp.DialFuncWithTimeout, defaultTimeout time.Duration, maxConns uint, maxResponseBody uint64) *scriptHTTPClient {
newClient := func(insecure bool) *fasthttp.Client {
// Read and write timeouts stay unset because they would cap the per-call timeout,
// so dialWithDeadline is what bounds the TLS handshake.
return &fasthttp.Client{
DialTimeout: dialWithDeadline(dial),
MaxConnsPerHost: safeUintToInt(maxConns),
DialTimeout: dialWithDeadline(dial),
MaxConnsPerHost: safeUintToInt(maxConns),
MaxResponseBodySize: safeUint64ToInt(maxResponseBody),
TLSConfig: &tls.Config{
InsecureSkipVerify: insecure, //nolint:gosec
},
@@ -144,6 +148,8 @@ func (c *scriptHTTPClient) Do(r *script.HTTPRequest) (*script.HTTPResponse, erro
}
req.SetTimeout(timeout)
// A body limit streams the response, so a declared Content-Length is never allocated.
resp.StreamBody = r.MaxBodySize > 0
if r.MaxRedirects > 0 {
err = client.DoRedirects(req, resp, r.MaxRedirects)
} else {
@@ -153,9 +159,47 @@ func (c *scriptHTTPClient) Do(r *script.HTTPRequest) (*script.HTTPResponse, erro
return nil, types.NewScriptHTTPRequestError(r.Method, r.URL, err)
}
body, err := readScriptBody(resp, r.MaxBodySize)
if err != nil {
return nil, types.NewScriptHTTPRequestError(r.Method, r.URL, err)
}
return &script.HTTPResponse{
Status: resp.StatusCode(),
Headers: collectRespHeaders(resp),
Body: string(resp.Body()),
Body: body,
}, nil
}
// readScriptBody reads the streamed body, refusing it once it passes maxBodySize.
// It can return the following errors:
// - types.ErrScriptHTTPBodyTooLarge
func readScriptBody(resp *fasthttp.Response, maxBodySize int64) (string, error) {
stream := resp.BodyStream()
if stream == nil {
return string(resp.Body()), nil
}
var (
body bytes.Buffer
err error
)
if maxBodySize > 0 {
// One byte past the limit tells an exact fit from an overflow.
_, err = io.CopyN(&body, stream, maxBodySize+1)
if err == nil {
err = types.ErrScriptHTTPBodyTooLarge
} else if errors.Is(err, io.EOF) {
err = nil
}
} else {
_, err = io.Copy(&body, stream)
}
// An unfinished stream makes fasthttp drop the connection instead of pooling it.
_ = resp.CloseBodyStream()
if err != nil {
return "", err
}
return body.String(), nil
}
+18 -2
View File
@@ -2,7 +2,9 @@ package sarin
import (
"errors"
"net"
"strconv"
"strings"
"sync/atomic"
"time"
@@ -31,6 +33,20 @@ func statusCodeToString(code int) string {
return strconv.Itoa(code)
}
// responseErrorKey drops the local address, whose port would mint a stats entry per connection.
func responseErrorKey(err error) string {
message := err.Error()
var opErr *net.OpError
if !errors.As(err, &opErr) || opErr.Source == nil {
return message
}
withoutSource := *opErr
withoutSource.Source = nil
return strings.Replace(message, opErr.Error(), withoutSource.Error(), 1)
}
// requestErrorKey keys a generation failure by its source instead of its message.
// A script picks its own URLs and error texts, so the message would mint a new
// stats entry per request. The full message still goes to the runtime log.
@@ -135,7 +151,7 @@ func (s sarin) workerStatsWithDynamic(
respDuration := time.Since(startTime)
if err != nil {
s.responses.Add(err.Error(), respDuration)
s.responses.Add(responseErrorKey(err), respDuration)
} else {
s.responses.Add(statusCodeToString(resp.StatusCode()), respDuration)
sendRespLog(respDuration, resp)
@@ -170,7 +186,7 @@ func (s sarin) workerStatsWithStatic(
err := s.hostClients[nextClientIndex()].DoTimeout(req, resp, s.timeout)
respDuration := time.Since(startTime)
if err != nil {
s.responses.Add(err.Error(), respDuration)
s.responses.Add(responseErrorKey(err), respDuration)
} else {
s.responses.Add(statusCodeToString(resp.StatusCode()), respDuration)
sendRespLog(respDuration, resp)
+21
View File
@@ -21,6 +21,7 @@ type HTTPRequest struct {
Timeout time.Duration // zero means the doer's default timeout
Insecure bool
MaxRedirects int
MaxBodySize int64 // zero means no limit
}
// HTTPResponse is the response returned to a script by the http.* functions.
@@ -66,6 +67,9 @@ const httpDefaultMethod = http.MethodGet
// httpMaxRedirectsLimit is the highest maxRedirects a script may ask for.
const httpMaxRedirectsLimit = 100
// httpMaxBodySizeLimit is the highest maxBodySize a script may ask for, 1 TiB.
const httpMaxBodySizeLimit int64 = 1 << 40
// Option keys accepted by the http function.
const (
httpOptionMethod = "method"
@@ -76,6 +80,7 @@ const (
httpOptionTimeout = "timeout"
httpOptionInsecure = "insecure"
httpOptionMaxRedirects = "maxRedirects"
httpOptionMaxBodySize = "maxBodySize"
)
// httpBridge connects http to the doer. Calls only work while transform runs.
@@ -118,6 +123,22 @@ func parseHTTPMaxRedirects(count float64) (int, error) {
return int(count), nil
}
// parseHTTPMaxBodySize checks that the body limit is a non-negative whole number.
// It can return the following errors:
// - types.ScriptHTTPOptionError
func parseHTTPMaxBodySize(size float64) (int64, error) {
if math.IsNaN(size) || size < 0 || size > float64(httpMaxBodySizeLimit) || size != math.Trunc(size) {
return 0, types.NewScriptHTTPOptionError(
httpOptionMaxBodySize,
types.NewScriptTypeError(
"a whole number of bytes between 0 and "+strconv.FormatInt(httpMaxBodySizeLimit, 10),
strconv.FormatFloat(size, 'g', -1, 64),
),
)
}
return int64(size), nil
}
// parseHTTPTimeout parses a Go duration string such as "500ms" or "2s".
// It can return the following errors:
// - types.ScriptHTTPOptionError
+10
View File
@@ -117,6 +117,16 @@ func (e *JsEngine) parseHTTPOptions(opts *goja.Object, req *HTTPRequest) error {
if req.MaxRedirects, err = parseHTTPMaxRedirects(count); err != nil {
return err
}
case httpOptionMaxBodySize:
size, ok := numberValue(value)
if !ok {
return types.NewScriptHTTPOptionError(name, types.NewScriptTypeError("number", jsTypeName(value)))
}
var err error
if req.MaxBodySize, err = parseHTTPMaxBodySize(size); err != nil {
return err
}
default:
return types.NewScriptHTTPOptionError(name, types.ErrScriptHTTPUnknownOption)
}
+7
View File
@@ -106,6 +106,13 @@ func (e *LuaEngine) parseHTTPOptions(opts *lua.LTable, req *HTTPRequest) error {
return
}
req.MaxRedirects, err = parseHTTPMaxRedirects(float64(count))
case httpOptionMaxBodySize:
size, ok := v.(lua.LNumber)
if !ok {
err = types.NewScriptHTTPOptionError(name, types.NewScriptTypeError("number", v.Type().String()))
return
}
req.MaxBodySize, err = parseHTTPMaxBodySize(float64(size))
default:
err = types.NewScriptHTTPOptionError(name, types.ErrScriptHTTPUnknownOption)
}
+6
View File
@@ -10,6 +10,11 @@ import (
var (
errNoError = errors.New("no error (internal)")
ErrByteSizeInvalid = errors.New(
"invalid value for size, expected a whole number of bytes or a whole number with a unit: " +
"B, KB, MB, GB, TB (1000 based) or KiB, MiB, GiB, TiB (1024 based), e.g. '1048576', '10MiB', '2MB'",
)
)
type FieldParseError struct {
@@ -408,6 +413,7 @@ var (
ErrScriptHTTPMethodEmpty = errors.New("http method cannot be empty")
ErrScriptHTTPURLInvalid = errors.New("URL must be an absolute http or https URL")
ErrScriptHTTPUnknownOption = errors.New("unknown option")
ErrScriptHTTPBodyTooLarge = errors.New("response body exceeds maxBodySize")
ErrScriptJSONCycle = errors.New("cannot encode a table that contains itself")
)