#!/usr/bin/env bash
# times owner comment writes from spawn to exit, across the scenarios a deployed knot sees
set -euo pipefail
export LC_ALL=C
usage="usage: $0 <knot-static binary> (<knot config.toml> | --fixture <repos>)"
[ $# -ge 2 ] || { echo "$usage" >&2; exit 2; }
bin=$1
runs=${RUNS:-5}
budget_ms=${BUDGET_MS:-2000}
fx=$(mktemp -d)
trap 'kill $(cat "$fx/stub.pid" 2>/dev/null) 2>/dev/null || true; rm -rf "$fx"' EXIT
export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 KNOT_PROFILE=${KNOT_PROFILE:-0}
# a small knot of its own, with the same key the clones of site-work use
fixture() {
mkdir -p "$fx/knot/state" "$fx/knot/sources"
cat >"$fx/knot/config.toml" <<EOF
[knot]
hostname = "knot.bench.test"
owner = "did:web:owner.bench.test"
[repo]
scan_path = "$fx/knot/state"
default_branch = "main"
object_format = "sha1"
[secrets]
sealed_key_file = "$fx/knot/keys.sealed"
master_key_env = "KNOT_STATIC_PHASE0_KEY"
[atproto]
plc_directory = "https://127.0.0.1:9"
[output]
root = "$fx/knot/output"
[activate]
nginx_binary = "true"
EOF
# init and import submit to an offline plc after the work is done, so their status is ignored
"$bin" --config "$fx/knot/config.toml" init >/dev/null 2>&1 || true
for i in $(seq "$1"); do
src=$fx/knot/sources/r$i
git init -q -b main "$src"
for f in $(seq 20); do echo "file $f of repo $i" >"$src/f$f.txt"; done
git -C "$src" add -A
git -C "$src" -c user.name=b -c user.email=b@bench.test commit -q -m "repo $i"
"$bin" --config "$fx/knot/config.toml" repo import --owner did:web:owner.bench.test \
--rkey "r$i" --name "r$i" --default-branch main "$src" >/dev/null 2>&1 || true
done
"$bin" --config "$fx/knot/config.toml" generate >/dev/null 2>&1
"$bin" --config "$fx/knot/config.toml" activate "$(ls "$fx/knot/output/generations" | head -n 1)" >/dev/null 2>&1
echo "$fx/knot/config.toml"
}
if [ "$2" = --fixture ]; then
export KNOT_STATIC_PHASE0_KEY=${KNOT_STATIC_PHASE0_KEY:-$(head -c 32 /dev/zero | tr '\0' '\7' | base64)}
base_config=$(fixture "${3:?$usage}")
else
base_config=$2
fi
# a renderer that takes its time over everything, dying included
cat >"$fx/stub.sh" <<EOF
#!/usr/bin/env bash
echo \$\$ >"$fx/stub.pid"
trap 'sleep 3; exit 3' TERM
sleep 600 & wait
EOF
chmod +x "$fx/stub.sh"
# a firehose keeps the write from polling its own listRepos, as on a deployed knot
sed '/^\[firehose\]/,/^\[/d' "$base_config" >"$fx/quiet.toml"
printf '\n[firehose]\nsocket = "%s"\n' "$fx/firehose.sock" >>"$fx/quiet.toml"
cp "$fx/quiet.toml" "$fx/render.toml"
printf '\n[[hooks]]\nname = "render"\ncommand = "%s"\n' "$fx/stub.sh" >>"$fx/render.toml"
ks() { "$bin" --config "$1" "${@:2}"; }
# one issue to comment on in each of up to five repos the owner can write as
touch "$fx/issues"
# BENCH_REPOS picks them, e.g. a repo seeded with many writes first
for repo in ${BENCH_REPOS:-$(ks "$fx/quiet.toml" repo list | awk '{ print $4 }')}; do
number=$( (ks "$fx/quiet.toml" issue new "$repo" --title "write latency" --body bench 2>/dev/null || true) |
sed -n 's|.*org.tangled.track.ticket/\([^"]*\)".*|\1|p' | head -n 1)
[ -n "$number" ] && echo "$repo $number" >>"$fx/issues"
[ "$(wc -l <"$fx/issues")" -lt 5 ] || break
done
[ -s "$fx/issues" ] || { echo "no repo took an issue" >&2; exit 1; }
first=$(head -n 1 "$fx/issues" | cut -d' ' -f1)
write_ms() {
local config=$1 repo=$2 start end number
number=$(awk -v r="$repo" '$1 == r { print $2 }' "$fx/issues")
start=$EPOCHREALTIME
"${WRITER:-$bin}" --config "$config" issue comment "$repo" "$number" --body "bench $RANDOM" >>"$fx/log" 2>&1
end=$EPOCHREALTIME
awk -v s="$start" -v e="$end" 'BEGIN { printf "%d\n", (e - s) * 1000 }'
}
# gated scenarios count against the budget, the concurrent ones are reported
summary() {
sort -n | awk -v name="$1" -v budget="$budget_ms" -v gated="$2" '{ v[NR] = $1 } END {
p95 = v[int(NR * 0.95 + 0.999)]; over = gated && p95 > budget
printf "%-24s med %6d ms p95 %6d ms max %6d ms (%d writes)%s\n", name, v[int((NR + 1) / 2)], p95, v[NR], NR,
(over ? " OVER " budget " ms" : "")
exit over }'
}
# n writes started together, to n different repos or all to the first
together() {
local n=$1 spread=$2 i repo
for i in $(seq "$n"); do
repo=$first
[ "$spread" = different ] && repo=$(sed -n "${i}p" "$fx/issues" | cut -d' ' -f1)
[ -n "$repo" ] || repo=$first
write_ms "$fx/quiet.toml" "$repo" >"$fx/together.$i" &
done
wait
cat "$fx"/together.*
rm -f "$fx"/together.*
}
over=0
write_ms "$fx/quiet.toml" "$first" >/dev/null
for _ in $(seq "$runs"); do write_ms "$fx/quiet.toml" "$first"; done | summary quiet 1 || over=1
write_ms "$fx/render.toml" "$first" >/dev/null
# each write lands while the renderer the last one started is still going
for _ in $(seq "$runs"); do write_ms "$fx/render.toml" "$first"; done | summary mid-render 1 || over=1
kill "$(cat "$fx/stub.pid")" 2>/dev/null || true
for _ in $(seq "$runs"); do
ks "$fx/quiet.toml" post-activate >>"$fx/log" 2>&1 &
write_ms "$fx/quiet.toml" "$first"
wait
done | summary during-drain 1 || over=1
# a deploy: another binary, its prepare unit, then the first write
# a fresh file each time: macos kills a binary rewritten in place
for i in $(seq "$runs"); do
cp "$bin" "$fx/deployed.$i"
"$fx/deployed.$i" --config "$fx/quiet.toml" prepare >>"$fx/log" 2>&1
WRITER=$fx/deployed.$i write_ms "$fx/quiet.toml" "$first"
done | summary first-after-deploy 1 || over=1
distinct=$(wc -l <"$fx/issues" | tr -d ' ')
for n in 2 5; do
for spread in different same; do
[ "$spread" = different ] && [ "$distinct" -lt "$n" ] && continue
for _ in $(seq "$runs"); do together "$n" "$spread"; done | summary "$n concurrent, $spread" 0 || true
done
done
[ "$KNOT_PROFILE" = 1 ] && grep '\[mark\]' "$fx/log" | awk '{ ms = $(NF - 1); label = $0
sub(/^\[mark\] +/, "", label); sub(/ +[0-9.]+ ms$/, "", label); t[label] += ms; n[label]++ } END {
for (k in t) printf "%9.1f ms %s\n", t[k] / n[k], k }' | sort -rn | head -n 25
exit $over