diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..4bdd038 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,42 @@ +# Benchmark + +Compares [sarin](https://github.com/aykhans/sarin), [wrk](https://github.com/wg/wrk), and [bombardier](https://github.com/codesenberg/bombardier) against a minimal C HTTP server using epoll. + +## Requirements + +- `sarin`, `wrk`, `bombardier` in PATH +- `gcc` + +## Usage + +```bash +./benchmark/run.sh +``` + +Configuration is at the top of `run.sh`: + +```bash +DURATION="30s" +CONNECTIONS=(50 100 200) +ITERATIONS=3 +``` + +## Structure + +``` +benchmark/ + run.sh - benchmark script + server/ + server.c - C epoll HTTP server (returns "ok") + results/ - output directory (auto-created) +``` + +## Output + +Each run produces per-tool files: + +- `*.out` - tool stdout (throughput, latency) +- `*.time` - `/usr/bin/time -v` output (peak memory, CPU time) +- `*_resources.csv` - sampled CPU/memory during run + +A summary table is printed at the end with requests/sec, total requests, elapsed time, and peak memory. diff --git a/benchmark/run.sh b/benchmark/run.sh new file mode 100755 index 0000000..4507ea0 --- /dev/null +++ b/benchmark/run.sh @@ -0,0 +1,335 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ─── Configuration ─────────────────────────────────────────────────────────── +SERVER_PORT=8080 +SERVER_URL="http://127.0.0.1:${SERVER_PORT}/" +DURATION="30s" +CONNECTIONS=(50 100 200) +ITERATIONS=3 +WARMUP_DURATION="5s" +RESULTS_DIR="benchmark/results/$(date +%Y%m%d_%H%M%S)" + +# ─── Colors ────────────────────────────────────────────────────────────────── +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +BOLD='\033[1m' +NC='\033[0m' + +log() { echo -e "${GREEN}[+]${NC} $*"; } +warn() { echo -e "${YELLOW}[!]${NC} $*"; } +err() { echo -e "${RED}[✗]${NC} $*" >&2; } +header() { echo -e "\n${BOLD}${CYAN}═══ $* ═══${NC}\n"; } + +# ─── Dependency checks ────────────────────────────────────────────────────── +check_deps() { + local missing=() + for cmd in wrk bombardier sarin gcc; do + if ! command -v "$cmd" &>/dev/null; then + missing+=("$cmd") + fi + done + if [[ ${#missing[@]} -gt 0 ]]; then + err "Missing dependencies: ${missing[*]}" + echo "Install them before running this benchmark." + exit 1 + fi + log "All dependencies found" +} + +# ─── Build & manage the C server ──────────────────────────────────────────── +build_server() { + header "Building C HTTP server" + gcc -O3 -o benchmark/server/bench-server benchmark/server/server.c + log "Server built successfully" +} + +start_server() { + log "Starting server on port ${SERVER_PORT}..." + benchmark/server/bench-server & + SERVER_PID=$! + sleep 1 + + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + err "Server failed to start" + exit 1 + fi + log "Server running (PID: ${SERVER_PID})" +} + +stop_server() { + if [[ -n "${SERVER_PID:-}" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + log "Server stopped" + fi +} + +trap stop_server EXIT + +# ─── Resource monitoring ──────────────────────────────────────────────────── +start_monitor() { + local tool_name=$1 + local conns=$2 + local iter=$3 + local monitor_file="${RESULTS_DIR}/${tool_name}_c${conns}_i${iter}_resources.csv" + + echo "timestamp,cpu%,mem_kb" > "$monitor_file" + + ( + while true; do + # Find the PID of the tool by name (exclude monitor itself) + local pid + pid=$(pgrep -x "$tool_name" 2>/dev/null | head -1) || true + if [[ -n "$pid" ]]; then + local stats + stats=$(ps -p "$pid" -o %cpu=,%mem=,rss= 2>/dev/null) || true + if [[ -n "$stats" ]]; then + local cpu mem_kb + cpu=$(echo "$stats" | awk '{print $1}') + mem_kb=$(echo "$stats" | awk '{print $3}') + echo "$(date +%s),$cpu,$mem_kb" >> "$monitor_file" + fi + fi + sleep 0.5 + done + ) & + MONITOR_PID=$! +} + +stop_monitor() { + if [[ -n "${MONITOR_PID:-}" ]] && kill -0 "$MONITOR_PID" 2>/dev/null; then + kill "$MONITOR_PID" 2>/dev/null || true + wait "$MONITOR_PID" 2>/dev/null || true + fi +} + +# ─── Benchmark runners ───────────────────────────────────────────────────── +run_wrk() { + local conns=$1 + local dur=$2 + local out_file=$3 + local threads=$((conns < 10 ? conns : 10)) + + /usr/bin/time -v wrk -t"${threads}" -c"${conns}" -d"${dur}" "${SERVER_URL}" \ + 2>"${out_file}.time" | tee "${out_file}.out" +} + +run_bombardier() { + local conns=$1 + local dur=$2 + local out_file=$3 + + /usr/bin/time -v bombardier -c "${conns}" -d "${dur}" --print result "${SERVER_URL}" \ + 2>"${out_file}.time" | tee "${out_file}.out" +} + +run_sarin() { + local conns=$1 + local dur=$2 + local out_file=$3 + + /usr/bin/time -v sarin -U "${SERVER_URL}" -c "${conns}" -d "${dur}" -q \ + 2>"${out_file}.time" | tee "${out_file}.out" +} + +# ─── Warmup ────────────────────────────────────────────────────────────────── +warmup() { + header "Warming up server" + wrk -t4 -c50 -d"${WARMUP_DURATION}" "${SERVER_URL}" > /dev/null 2>&1 + log "Warmup complete" + sleep 2 +} + +# ─── Extract peak memory from /usr/bin/time -v output ──────────────────────── +extract_peak_mem() { + local time_file=$1 + grep "Maximum resident set size" "$time_file" 2>/dev/null | awk '{print $NF}' || echo "N/A" +} + +# ─── Extract total requests from tool output ───────────────────────────────── +extract_requests() { + local tool=$1 + local out_file=$2 + + case "$tool" in + wrk) + # wrk: "312513 requests in 2.10s, ..." + grep "requests in" "$out_file" 2>/dev/null | awk '{print $1}' || echo "N/A" + ;; + bombardier) + # bombardier: "1xx - 0, 2xx - 100000, 3xx - 0, 4xx - 0, 5xx - 0" + # Sum all HTTP code counts + grep -E "^\s+1xx" "$out_file" 2>/dev/null | \ + awk -F'[,-]' '{sum=0; for(i=1;i<=NF;i++){gsub(/[^0-9]/,"",$i); if($i+0>0) sum+=$i} print sum}' || echo "N/A" + ;; + sarin) + # sarin table: "│ Total │ 1556177 │ ..." + grep -i "total" "$out_file" 2>/dev/null | awk -F'│' '{gsub(/[[:space:]]/, "", $3); print $3}' || echo "N/A" + ;; + esac +} + +extract_elapsed() { + local time_file=$1 + grep "wall clock" "$time_file" 2>/dev/null | awk '{print $NF}' || echo "N/A" +} + +extract_rps() { + local tool=$1 + local out_file=$2 + + case "$tool" in + wrk) + # wrk: "Requests/sec: 12345.67" + grep "Requests/sec" "$out_file" 2>/dev/null | awk '{print $2}' || echo "N/A" + ;; + bombardier) + # bombardier: "Reqs/sec 12345.67 ..." + grep -i "reqs/sec" "$out_file" 2>/dev/null | awk '{print $2}' || echo "N/A" + ;; + sarin) + # sarin doesn't output rps - calculate from total requests and duration + local total + total=$(extract_requests "sarin" "$out_file") + if [[ "$total" != "N/A" && -n "$total" ]]; then + local dur_secs + dur_secs=$(echo "$DURATION" | sed 's/s$//') + awk "BEGIN {printf \"%.2f\", $total / $dur_secs}" + else + echo "N/A" + fi + ;; + esac +} + +# ─── Print comparison table ────────────────────────────────────────────────── +print_table() { + local title=$1 + local extract_fn=$2 + shift 2 + local columns=("$@") + + echo -e "${BOLD}${title}:${NC}" + printf "%-12s" "" + for col in "${columns[@]}"; do + printf "%-18s" "$col" + done + echo "" + + local tools=("wrk" "bombardier" "sarin") + for tool in "${tools[@]}"; do + printf "%-12s" "$tool" + for col in "${columns[@]}"; do + local val + val=$($extract_fn "$tool" "$col") + printf "%-18s" "${val}" + done + echo "" + done + echo "" +} + +# ─── Main ──────────────────────────────────────────────────────────────────── +main() { + header "HTTP Load Testing Tool Benchmark" + echo "Tools: wrk, bombardier, sarin" + echo "Duration: ${DURATION} per run" + echo "Connections: ${CONNECTIONS[*]}" + echo "Iterations: ${ITERATIONS} per configuration" + echo "" + + check_deps + + mkdir -p "${RESULTS_DIR}" + log "Results will be saved to ${RESULTS_DIR}/" + + build_server + start_server + warmup + + local tools=("wrk" "bombardier" "sarin") + + for conns in "${CONNECTIONS[@]}"; do + header "Testing with ${conns} connections" + + for tool in "${tools[@]}"; do + echo -e "${BOLD}--- ${tool} (${conns} connections) ---${NC}" + + for iter in $(seq 1 "$ITERATIONS"); do + local out_file="${RESULTS_DIR}/${tool}_c${conns}_i${iter}" + echo -n " Run ${iter}/${ITERATIONS}... " + + start_monitor "$tool" "$conns" "$iter" + + case "$tool" in + wrk) run_wrk "$conns" "$DURATION" "$out_file" > /dev/null 2>&1 ;; + bombardier) run_bombardier "$conns" "$DURATION" "$out_file" > /dev/null 2>&1 ;; + sarin) run_sarin "$conns" "$DURATION" "$out_file" > /dev/null 2>&1 ;; + esac + + stop_monitor + + local peak_mem rps elapsed + peak_mem=$(extract_peak_mem "${out_file}.time") + rps=$(extract_rps "$tool" "${out_file}.out") + elapsed=$(extract_elapsed "${out_file}.time") + echo -e "done (elapsed: ${elapsed}, rps: ${rps}, peak mem: ${peak_mem} KB)" + + sleep 2 + done + echo "" + done + done + + # ─── Summary ───────────────────────────────────────────────────────── + header "Summary" + echo "Raw results saved to: ${RESULTS_DIR}/" + echo "" + echo "Files per run:" + echo " *.out - tool stdout (throughput, latency stats)" + echo " *.time - /usr/bin/time output (peak memory, CPU time)" + echo " *_resources.csv - sampled CPU/memory during run" + echo "" + + local columns=() + for conns in "${CONNECTIONS[@]}"; do + columns+=("c=${conns}") + done + + _get_rps() { + local c=${2#c=} + extract_rps "$1" "${RESULTS_DIR}/${1}_c${c}_i${ITERATIONS}.out" + } + + _get_total() { + local c=${2#c=} + extract_requests "$1" "${RESULTS_DIR}/${1}_c${c}_i${ITERATIONS}.out" + } + + _get_mem() { + local c=${2#c=} + extract_peak_mem "${RESULTS_DIR}/${1}_c${c}_i${ITERATIONS}.time" + } + + _get_elapsed() { + local c=${2#c=} + extract_elapsed "${RESULTS_DIR}/${1}_c${c}_i${ITERATIONS}.time" + } + + print_table "Requests/sec" _get_rps "${columns[@]}" + print_table "Total Requests" _get_total "${columns[@]}" + print_table "Elapsed Time" _get_elapsed "${columns[@]}" + print_table "Peak Memory (KB)" _get_mem "${columns[@]}" + + log "Benchmark complete!" + echo "" + echo "To inspect individual results:" + echo " cat ${RESULTS_DIR}/wrk_c200_i1.out" + echo " cat ${RESULTS_DIR}/sarin_c200_i1.out" + echo " cat ${RESULTS_DIR}/bombardier_c200_i1.out" +} + +main "$@" diff --git a/benchmark/server/server.c b/benchmark/server/server.c new file mode 100644 index 0000000..d9e1b23 --- /dev/null +++ b/benchmark/server/server.c @@ -0,0 +1,114 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PORT 8080 +#define MAX_EVENTS 1024 +#define BUF_SIZE 4096 + +static const char RESPONSE[] = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 2\r\n" + "Content-Type: text/plain\r\n" + "Connection: keep-alive\r\n" + "\r\n" + "ok"; + +static const int RESPONSE_LEN = sizeof(RESPONSE) - 1; + +static void set_nonblocking(int fd) { + int flags = fcntl(fd, F_GETFL, 0); + fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + +int main(void) { + signal(SIGPIPE, SIG_IGN); + + int server_fd = socket(AF_INET, SOCK_STREAM, 0); + if (server_fd < 0) { + perror("socket"); + return 1; + } + + int opt = 1; + setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); + setsockopt(server_fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)); + setsockopt(server_fd, IPPROTO_TCP, TCP_NODELAY, &opt, sizeof(opt)); + + struct sockaddr_in addr = { + .sin_family = AF_INET, + .sin_port = htons(PORT), + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + }; + + if (bind(server_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + perror("bind"); + return 1; + } + + if (listen(server_fd, SOMAXCONN) < 0) { + perror("listen"); + return 1; + } + + set_nonblocking(server_fd); + + int epoll_fd = epoll_create1(0); + struct epoll_event ev = {.events = EPOLLIN, .data.fd = server_fd}; + epoll_ctl(epoll_fd, EPOLL_CTL_ADD, server_fd, &ev); + + struct epoll_event events[MAX_EVENTS]; + char buf[BUF_SIZE]; + + fprintf(stderr, "Listening on http://127.0.0.1:%d\n", PORT); + + for (;;) { + int n = epoll_wait(epoll_fd, events, MAX_EVENTS, -1); + for (int i = 0; i < n; i++) { + if (events[i].data.fd == server_fd) { + /* Accept all pending connections */ + for (;;) { + int client_fd = accept(server_fd, NULL, NULL); + if (client_fd < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) + break; + continue; + } + set_nonblocking(client_fd); + int tcp_opt = 1; + setsockopt(client_fd, IPPROTO_TCP, TCP_NODELAY, &tcp_opt, sizeof(tcp_opt)); + struct epoll_event cev = {.events = EPOLLIN | EPOLLET, .data.fd = client_fd}; + epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &cev); + } + } else { + int fd = events[i].data.fd; + /* Read all available data and respond to each request */ + for (;;) { + ssize_t nread = read(fd, buf, BUF_SIZE); + if (nread <= 0) { + if (nread == 0 || (errno != EAGAIN && errno != EWOULDBLOCK)) { + close(fd); + } + break; + } + if (write(fd, RESPONSE, RESPONSE_LEN) < 0) { + close(fd); + break; + } + } + } + } + } + + close(server_fd); + close(epoll_fd); + return 0; +}