Add benchmark suite comparing sarin, wrk, and bombardier

2026-04-14 20:19:37 +00:00 · 2026-03-22 22:30:47 +04:00
parent 304fb160f8
commit a7b5f73069
3 changed files with 491 additions and 0 deletions
--- a/benchmark/README.md
+++ b/benchmark/README.md
@@ -0,0 +1,42 @@
+# Benchmark
+
+Compares [sarin](https://github.com/aykhans/sarin), [wrk](https://github.com/wg/wrk), and [bombardier](https://github.com/codesenberg/bombardier) against a minimal C HTTP server using epoll.
+
+## Requirements
+
+- `sarin`, `wrk`, `bombardier` in PATH
+- `gcc`
+
+## Usage
+
+```bash
+./benchmark/run.sh
+```
+
+Configuration is at the top of `run.sh`:
+
+```bash
+DURATION="30s"
+CONNECTIONS=(50 100 200)
+ITERATIONS=3
+```
+
+## Structure
+
+```
+benchmark/
+  run.sh          - benchmark script
+  server/
+    server.c      - C epoll HTTP server (returns "ok")
+  results/        - output directory (auto-created)
+```
+
+## Output
+
+Each run produces per-tool files:
+
+- `*.out` - tool stdout (throughput, latency)
+- `*.time` - `/usr/bin/time -v` output (peak memory, CPU time)
+- `*_resources.csv` - sampled CPU/memory during run
+
+A summary table is printed at the end with requests/sec, total requests, elapsed time, and peak memory.
--- a/benchmark/run.sh
+++ b/benchmark/run.sh
@@ -0,0 +1,335 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# ─── Configuration ───────────────────────────────────────────────────────────
+SERVER_PORT=8080
+SERVER_URL="http://127.0.0.1:${SERVER_PORT}/"
+DURATION="30s"
+CONNECTIONS=(50 100 200)
+ITERATIONS=3
+WARMUP_DURATION="5s"
+RESULTS_DIR="benchmark/results/$(date +%Y%m%d_%H%M%S)"
+
+# ─── Colors ──────────────────────────────────────────────────────────────────
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+BOLD='\033[1m'
+NC='\033[0m'
+
+log()  { echo -e "${GREEN}[+]${NC} $*"; }
+warn() { echo -e "${YELLOW}[!]${NC} $*"; }
+err()  { echo -e "${RED}[✗]${NC} $*" >&2; }
+header() { echo -e "\n${BOLD}${CYAN}═══ $* ═══${NC}\n"; }
+
+# ─── Dependency checks ──────────────────────────────────────────────────────
+check_deps() {
+    local missing=()
+    for cmd in wrk bombardier sarin gcc; do
+        if ! command -v "$cmd" &>/dev/null; then
+            missing+=("$cmd")
+        fi
+    done
+    if [[ ${#missing[@]} -gt 0 ]]; then
+        err "Missing dependencies: ${missing[*]}"
+        echo "Install them before running this benchmark."
+        exit 1
+    fi
+    log "All dependencies found"
+}
+
+# ─── Build & manage the C server ────────────────────────────────────────────
+build_server() {
+    header "Building C HTTP server"
+    gcc -O3 -o benchmark/server/bench-server benchmark/server/server.c
+    log "Server built successfully"
+}
+
+start_server() {
+    log "Starting server on port ${SERVER_PORT}..."
+    benchmark/server/bench-server &
+    SERVER_PID=$!
+    sleep 1
+
+    if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+        err "Server failed to start"
+        exit 1
+    fi
+    log "Server running (PID: ${SERVER_PID})"
+}
+
+stop_server() {
+    if [[ -n "${SERVER_PID:-}" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
+        kill "$SERVER_PID" 2>/dev/null || true
+        wait "$SERVER_PID" 2>/dev/null || true
+        log "Server stopped"
+    fi
+}
+
+trap stop_server EXIT
+
+# ─── Resource monitoring ────────────────────────────────────────────────────
+start_monitor() {
+    local tool_name=$1
+    local conns=$2
+    local iter=$3
+    local monitor_file="${RESULTS_DIR}/${tool_name}_c${conns}_i${iter}_resources.csv"
+
+    echo "timestamp,cpu%,mem_kb" > "$monitor_file"
+
+    (
+        while true; do
+            # Find the PID of the tool by name (exclude monitor itself)
+            local pid
+            pid=$(pgrep -x "$tool_name" 2>/dev/null | head -1) || true
+            if [[ -n "$pid" ]]; then
+                local stats
+                stats=$(ps -p "$pid" -o %cpu=,%mem=,rss= 2>/dev/null) || true
+                if [[ -n "$stats" ]]; then
+                    local cpu mem_kb
+                    cpu=$(echo "$stats" | awk '{print $1}')
+                    mem_kb=$(echo "$stats" | awk '{print $3}')
+                    echo "$(date +%s),$cpu,$mem_kb" >> "$monitor_file"
+                fi
+            fi
+            sleep 0.5
+        done
+    ) &
+    MONITOR_PID=$!
+}
+
+stop_monitor() {
+    if [[ -n "${MONITOR_PID:-}" ]] && kill -0 "$MONITOR_PID" 2>/dev/null; then
+        kill "$MONITOR_PID" 2>/dev/null || true
+        wait "$MONITOR_PID" 2>/dev/null || true
+    fi
+}
+
+# ─── Benchmark runners ─────────────────────────────────────────────────────
+run_wrk() {
+    local conns=$1
+    local dur=$2
+    local out_file=$3
+    local threads=$((conns < 10 ? conns : 10))
+
+    /usr/bin/time -v wrk -t"${threads}" -c"${conns}" -d"${dur}" "${SERVER_URL}" \
+        2>"${out_file}.time" | tee "${out_file}.out"
+}
+
+run_bombardier() {
+    local conns=$1
+    local dur=$2
+    local out_file=$3
+
+    /usr/bin/time -v bombardier -c "${conns}" -d "${dur}" --print result "${SERVER_URL}" \
+        2>"${out_file}.time" | tee "${out_file}.out"
+}
+
+run_sarin() {
+    local conns=$1
+    local dur=$2
+    local out_file=$3
+
+    /usr/bin/time -v sarin -U "${SERVER_URL}" -c "${conns}" -d "${dur}" -q \
+        2>"${out_file}.time" | tee "${out_file}.out"
+}
+
+# ─── Warmup ──────────────────────────────────────────────────────────────────
+warmup() {
+    header "Warming up server"
+    wrk -t4 -c50 -d"${WARMUP_DURATION}" "${SERVER_URL}" > /dev/null 2>&1
+    log "Warmup complete"
+    sleep 2
+}
+
+# ─── Extract peak memory from /usr/bin/time -v output ────────────────────────
+extract_peak_mem() {
+    local time_file=$1
+    grep "Maximum resident set size" "$time_file" 2>/dev/null | awk '{print $NF}' || echo "N/A"
+}
+
+# ─── Extract total requests from tool output ─────────────────────────────────
+extract_requests() {
+    local tool=$1
+    local out_file=$2
+
+    case "$tool" in
+        wrk)
+            # wrk: "312513 requests in 2.10s, ..."
+            grep "requests in" "$out_file" 2>/dev/null | awk '{print $1}' || echo "N/A"
+            ;;
+        bombardier)
+            # bombardier: "1xx - 0, 2xx - 100000, 3xx - 0, 4xx - 0, 5xx - 0"
+            # Sum all HTTP code counts
+            grep -E "^\s+1xx" "$out_file" 2>/dev/null | \
+                awk -F'[,-]' '{sum=0; for(i=1;i<=NF;i++){gsub(/[^0-9]/,"",$i); if($i+0>0) sum+=$i} print sum}' || echo "N/A"
+            ;;
+        sarin)
+            # sarin table: "│ Total │ 1556177 │ ..."
+            grep -i "total" "$out_file" 2>/dev/null | awk -F'│' '{gsub(/[[:space:]]/, "", $3); print $3}' || echo "N/A"
+            ;;
+    esac
+}
+
+extract_elapsed() {
+    local time_file=$1
+    grep "wall clock" "$time_file" 2>/dev/null | awk '{print $NF}' || echo "N/A"
+}
+
+extract_rps() {
+    local tool=$1
+    local out_file=$2
+
+    case "$tool" in
+        wrk)
+            # wrk: "Requests/sec:  12345.67"
+            grep "Requests/sec" "$out_file" 2>/dev/null | awk '{print $2}' || echo "N/A"
+            ;;
+        bombardier)
+            # bombardier: "Reqs/sec  12345.67  ..."
+            grep -i "reqs/sec" "$out_file" 2>/dev/null | awk '{print $2}' || echo "N/A"
+            ;;
+        sarin)
+            # sarin doesn't output rps - calculate from total requests and duration
+            local total
+            total=$(extract_requests "sarin" "$out_file")
+            if [[ "$total" != "N/A" && -n "$total" ]]; then
+                local dur_secs
+                dur_secs=$(echo "$DURATION" | sed 's/s$//')
+                awk "BEGIN {printf \"%.2f\", $total / $dur_secs}"
+            else
+                echo "N/A"
+            fi
+            ;;
+    esac
+}
+
+# ─── Print comparison table ──────────────────────────────────────────────────
+print_table() {
+    local title=$1
+    local extract_fn=$2
+    shift 2
+    local columns=("$@")
+
+    echo -e "${BOLD}${title}:${NC}"
+    printf "%-12s" ""
+    for col in "${columns[@]}"; do
+        printf "%-18s" "$col"
+    done
+    echo ""
+
+    local tools=("wrk" "bombardier" "sarin")
+    for tool in "${tools[@]}"; do
+        printf "%-12s" "$tool"
+        for col in "${columns[@]}"; do
+            local val
+            val=$($extract_fn "$tool" "$col")
+            printf "%-18s" "${val}"
+        done
+        echo ""
+    done
+    echo ""
+}
+
+# ─── Main ────────────────────────────────────────────────────────────────────
+main() {
+    header "HTTP Load Testing Tool Benchmark"
+    echo "Tools:       wrk, bombardier, sarin"
+    echo "Duration:    ${DURATION} per run"
+    echo "Connections: ${CONNECTIONS[*]}"
+    echo "Iterations:  ${ITERATIONS} per configuration"
+    echo ""
+
+    check_deps
+
+    mkdir -p "${RESULTS_DIR}"
+    log "Results will be saved to ${RESULTS_DIR}/"
+
+    build_server
+    start_server
+    warmup
+
+    local tools=("wrk" "bombardier" "sarin")
+
+    for conns in "${CONNECTIONS[@]}"; do
+        header "Testing with ${conns} connections"
+
+        for tool in "${tools[@]}"; do
+            echo -e "${BOLD}--- ${tool} (${conns} connections) ---${NC}"
+
+            for iter in $(seq 1 "$ITERATIONS"); do
+                local out_file="${RESULTS_DIR}/${tool}_c${conns}_i${iter}"
+                echo -n "  Run ${iter}/${ITERATIONS}... "
+
+                start_monitor "$tool" "$conns" "$iter"
+
+                case "$tool" in
+                    wrk)         run_wrk "$conns" "$DURATION" "$out_file" > /dev/null 2>&1 ;;
+                    bombardier)  run_bombardier "$conns" "$DURATION" "$out_file" > /dev/null 2>&1 ;;
+                    sarin)       run_sarin "$conns" "$DURATION" "$out_file" > /dev/null 2>&1 ;;
+                esac
+
+                stop_monitor
+
+                local peak_mem rps elapsed
+                peak_mem=$(extract_peak_mem "${out_file}.time")
+                rps=$(extract_rps "$tool" "${out_file}.out")
+                elapsed=$(extract_elapsed "${out_file}.time")
+                echo -e "done (elapsed: ${elapsed}, rps: ${rps}, peak mem: ${peak_mem} KB)"
+
+                sleep 2
+            done
+            echo ""
+        done
+    done
+
+    # ─── Summary ─────────────────────────────────────────────────────────
+    header "Summary"
+    echo "Raw results saved to: ${RESULTS_DIR}/"
+    echo ""
+    echo "Files per run:"
+    echo "  *.out       - tool stdout (throughput, latency stats)"
+    echo "  *.time      - /usr/bin/time output (peak memory, CPU time)"
+    echo "  *_resources.csv - sampled CPU/memory during run"
+    echo ""
+
+    local columns=()
+    for conns in "${CONNECTIONS[@]}"; do
+        columns+=("c=${conns}")
+    done
+
+    _get_rps() {
+        local c=${2#c=}
+        extract_rps "$1" "${RESULTS_DIR}/${1}_c${c}_i${ITERATIONS}.out"
+    }
+
+    _get_total() {
+        local c=${2#c=}
+        extract_requests "$1" "${RESULTS_DIR}/${1}_c${c}_i${ITERATIONS}.out"
+    }
+
+    _get_mem() {
+        local c=${2#c=}
+        extract_peak_mem "${RESULTS_DIR}/${1}_c${c}_i${ITERATIONS}.time"
+    }
+
+    _get_elapsed() {
+        local c=${2#c=}
+        extract_elapsed "${RESULTS_DIR}/${1}_c${c}_i${ITERATIONS}.time"
+    }
+
+    print_table "Requests/sec" _get_rps "${columns[@]}"
+    print_table "Total Requests" _get_total "${columns[@]}"
+    print_table "Elapsed Time" _get_elapsed "${columns[@]}"
+    print_table "Peak Memory (KB)" _get_mem "${columns[@]}"
+
+    log "Benchmark complete!"
+    echo ""
+    echo "To inspect individual results:"
+    echo "  cat ${RESULTS_DIR}/wrk_c200_i1.out"
+    echo "  cat ${RESULTS_DIR}/sarin_c200_i1.out"
+    echo "  cat ${RESULTS_DIR}/bombardier_c200_i1.out"
+}
+
+main "$@"
--- a/benchmark/server/server.c
+++ b/benchmark/server/server.c
@@ -0,0 +1,114 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/epoll.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#define PORT 8080
+#define MAX_EVENTS 1024
+#define BUF_SIZE 4096
+
+static const char RESPONSE[] =
+    "HTTP/1.1 200 OK\r\n"
+    "Content-Length: 2\r\n"
+    "Content-Type: text/plain\r\n"
+    "Connection: keep-alive\r\n"
+    "\r\n"
+    "ok";
+
+static const int RESPONSE_LEN = sizeof(RESPONSE) - 1;
+
+static void set_nonblocking(int fd) {
+    int flags = fcntl(fd, F_GETFL, 0);
+    fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+}
+
+int main(void) {
+    signal(SIGPIPE, SIG_IGN);
+
+    int server_fd = socket(AF_INET, SOCK_STREAM, 0);
+    if (server_fd < 0) {
+        perror("socket");
+        return 1;
+    }
+
+    int opt = 1;
+    setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
+    setsockopt(server_fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt));
+    setsockopt(server_fd, IPPROTO_TCP, TCP_NODELAY, &opt, sizeof(opt));
+
+    struct sockaddr_in addr = {
+        .sin_family = AF_INET,
+        .sin_port = htons(PORT),
+        .sin_addr.s_addr = htonl(INADDR_LOOPBACK),
+    };
+
+    if (bind(server_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
+        perror("bind");
+        return 1;
+    }
+
+    if (listen(server_fd, SOMAXCONN) < 0) {
+        perror("listen");
+        return 1;
+    }
+
+    set_nonblocking(server_fd);
+
+    int epoll_fd = epoll_create1(0);
+    struct epoll_event ev = {.events = EPOLLIN, .data.fd = server_fd};
+    epoll_ctl(epoll_fd, EPOLL_CTL_ADD, server_fd, &ev);
+
+    struct epoll_event events[MAX_EVENTS];
+    char buf[BUF_SIZE];
+
+    fprintf(stderr, "Listening on http://127.0.0.1:%d\n", PORT);
+
+    for (;;) {
+        int n = epoll_wait(epoll_fd, events, MAX_EVENTS, -1);
+        for (int i = 0; i < n; i++) {
+            if (events[i].data.fd == server_fd) {
+                /* Accept all pending connections */
+                for (;;) {
+                    int client_fd = accept(server_fd, NULL, NULL);
+                    if (client_fd < 0) {
+                        if (errno == EAGAIN || errno == EWOULDBLOCK)
+                            break;
+                        continue;
+                    }
+                    set_nonblocking(client_fd);
+                    int tcp_opt = 1;
+                    setsockopt(client_fd, IPPROTO_TCP, TCP_NODELAY, &tcp_opt, sizeof(tcp_opt));
+                    struct epoll_event cev = {.events = EPOLLIN | EPOLLET, .data.fd = client_fd};
+                    epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &cev);
+                }
+            } else {
+                int fd = events[i].data.fd;
+                /* Read all available data and respond to each request */
+                for (;;) {
+                    ssize_t nread = read(fd, buf, BUF_SIZE);
+                    if (nread <= 0) {
+                        if (nread == 0 || (errno != EAGAIN && errno != EWOULDBLOCK)) {
+                            close(fd);
+                        }
+                        break;
+                    }
+                    if (write(fd, RESPONSE, RESPONSE_LEN) < 0) {
+                        close(fd);
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    close(server_fd);
+    close(epoll_fd);
+    return 0;
+}