#!/usr/bin/env bash
# startup is measured from spawn to the first stdout byte
set -euo pipefail
[ $# -ge 1 ] || { echo "usage: $0 <knot-static binary>..." >&2; exit 2; }
rounds=${ROUNDS:-3}
runs=${RUNS:-100}
warm=${WARM:-10}
here=$(cd "$(dirname "$0")/.." && pwd)
ALBEDO_NO_MBX=1 cargo build -q --release --manifest-path "$here/bench/startup/Cargo.toml"
harness=$here/bench/startup/target/release/startup-bench
fx=$(mktemp -d)
trap 'rm -rf "$fx"' EXIT
mkdir -p "$fx/repos" "$fx/out" "$fx/secrets" "$fx/runs"
cat >"$fx/config.toml" <<EOF
[knot]
hostname = "knot.test"
owner = "did:plc:owner"
[repo]
scan_path = "$fx/repos"
[secrets]
sealed_key_file = "$fx/secrets/sealed"
[output]
root = "$fx/out"
[writes]
enabled = true
fastcgi_socket = "$fx/fcgi.sock"
EOF
key() {
{ printf '\0\0\0\013ssh-ed25519\0\0\0\040'; head -c 32 /dev/urandom; } | base64 | tr -d '\n'
}
{
printf '{"version": 1'
for i in $(seq 0 39); do
printf ', "did:plc:acct%020d": [' "$i"
for k in $(seq 0 $((i % 3))); do
[ "$k" = 0 ] || printf ', '
printf '"ssh-ed25519 %s"' "$(key)"
done
printf ']'
done
printf '}\n'
} >"$fx/out/ssh-keys.json"
# init exits nonzero offline, after the meta repo is already made
GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 "$1" --config "$fx/config.toml" init >/dev/null 2>&1 || true
unknown="did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"
cgi() {
env GATEWAY_INTERFACE=CGI/1.1 KNOT_STATIC_CONFIG="$fx/config.toml" REQUEST_METHOD=GET "$@"
}
# one tsv line per run, see bench/startup/src/main.rs for the columns
case_run() {
local n=$3 bin=$2
case $1 in
cgi_405) cgi REQUEST_URI=/xrpc/com.atproto.repo.createRecord "$harness" "$n" - 0 -- "$bin" ;;
inflate_405) cgi REQUEST_URI="/$unknown/git-upload-pack" KNOT_STATIC_INFLATE=1 "$harness" "$n" - 0 -- "$bin" ;;
keys_list) "$harness" "$n" - 0 -- "$bin" --config "$fx/config.toml" keys list --command "/bin/knot-static git-ssh" ;;
git_ssh_refuse) SSH_ORIGINAL_COMMAND="git-upload-pack '$unknown'" \
"$harness" "$n" - 0 -- "$bin" --config "$fx/config.toml" git-ssh --operator ;;
esac
}
cases="cgi_405 inflate_405 keys_list git_ssh_refuse"
for round in $(seq 1 "$rounds"); do
for c in $cases; do
i=0
for bin in "$@"; do
case_run "$c" "$bin" "$warm" >/dev/null
case_run "$c" "$bin" "$runs" >>"$fx/runs/$i-$c.tsv"
i=$((i + 1))
done
done
done
pick() {
cut -f"$2" "$1" | sort -n | awk '{ v[NR] = $1 } END {
m = v[int((NR + 1) / 2)]; p = v[int(NR * 0.95 + 0.5)]
printf "%.3f\t%.3f", m / 1000, p / 1000 }'
}
cpu() {
awk -F'\t' '{ print $4 + $5 }' "$1" | sort -n | awk '{ v[NR] = $1 } END { printf "%.3f", v[int((NR + 1) / 2)] / 1000 }'
}
printf 'case\tbinary\tfirst med\tfirst p95\texit med\texit p95\tcpu med\tthreads\n'
for c in $cases; do
i=0
for bin in "$@"; do
f=$fx/runs/$i-$c.tsv
threads=$(cut -f3 "$f" | sort -n | tail -n 1)
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$c" "$(basename "$bin")" "$(pick "$f" 1)" "$(pick "$f" 2)" "$(cpu "$f")" "$threads"
i=$((i + 1))
done
done