# nixos module for knot-static (`services.knot-static`).
# `knot-static` compiles generations into ${out} and swaps them under its own
# nginx, while the host's main nginx terminates tls.
self:
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.knot-static;
inherit (lib)
mkEnableOption
mkIf
mkMerge
mkOption
types
;
tomlFormat = pkgs.formats.toml { };
state = cfg.stateDir;
out = "${state}/out";
socket = "/run/knot-static/fcgiwrap.sock";
# not under /run/knot-static: the cgi's RuntimeDirectory goes when it stops
firehoseSocket = "/run/knot-static-firehose.sock";
# generations name these paths, so they must outlive a rebuild
configPath = "/etc/knot-static/config.toml";
binPath = "/run/current-system/sw/bin/knot-static";
# what an activation spawns: reload_command through sh, git for the exports
activatePath = [
pkgs.gitMinimal
pkgs.bash
];
defaultSettings = {
knot = {
inherit (cfg) hostname owner;
};
repo = {
scan_path = "${state}/src";
default_branch = "main";
object_format = "sha1";
};
secrets.sealed_key_file = "${state}/keys.sealed";
output.root = out;
activate = {
nginx_binary = "${pkgs.nginx}/bin/nginx";
inherit (cfg) listen;
# an empty pid file means nginx isn't up yet, the path unit starts it
reload_command = "p=$(cat ${out}/runtime/nginx.pid 2>/dev/null); [ -z \"$p\" ] || kill -HUP $p";
# the tls edge below, so the write limits see each client
real_ip_from = [ "127.0.0.1" ];
};
crawl.urls = [ "https://index.tangled.network" ];
firehose.socket = firehoseSocket;
writes = {
enabled = true;
fastcgi_socket = "unix:${socket}";
cgi_binary = binPath;
};
};
settings = lib.recursiveUpdate defaultSettings cfg.settings;
# pushes run as cfg.user; the actor flags say who is pushing. a push reads the
# master key from master.env next to keys.sealed, so no session carries it in
# its environment
gitSsh = pkgs.writeShellScript "knot-static-git-ssh" ''
if [ "$(${pkgs.coreutils}/bin/id -un)" != "${cfg.user}" ]; then
exec /run/wrappers/bin/sudo -u ${cfg.user} --preserve-env=SSH_ORIGINAL_COMMAND,GIT_PROTOCOL ${binPath} --config ${configPath} git-ssh "$@"
fi
exec ${binPath} --config ${configPath} git-ssh "$@"
'';
# collaborators' keys, read from their PDSes by `knot-static keys sync`
keysList = pkgs.writeShellScript "knot-static-keys-list" ''
exec ${cfg.package}/bin/knot-static --config ${configPath} keys list --command /etc/knot-static/git-ssh
'';
operatorKeyLines = map (
key: "command=\"/etc/knot-static/git-ssh --operator\",restrict ${key}"
) cfg.operatorKeys;
extraSshUsers = lib.filter (u: u != cfg.user) cfg.sshUsers;
in
{
options.services.knot-static = {
enable = mkEnableOption "knot-static";
package = mkOption {
type = types.package;
default = self.packages.${pkgs.stdenv.hostPlatform.system}.knot-static;
description = "Package providing the knot-static binary.";
};
firehosePackage = mkOption {
type = types.package;
default = self.packages.${pkgs.stdenv.hostPlatform.system}.knot-static-firehose;
description = "Package providing the knot-static-firehose binary.";
};
hostname = mkOption {
type = types.str;
description = "Public hostname of the knot.";
};
owner = mkOption {
type = types.str;
description = "Owner DID of the knot.";
};
user = mkOption {
type = types.str;
default = "op";
description = "User that owns the knot-static state and runs its services.";
};
group = mkOption {
type = types.str;
default = cfg.user;
description = "Group that owns the knot-static state.";
};
sshUsers = mkOption {
type = types.listOf types.str;
default = [
cfg.user
"git"
];
description = "System users that accept git- over SSH for knot-static.";
};
operatorKeys = mkOption {
type = types.listOf types.str;
default = [ ];
description = "SSH public keys granted --operator access on git-ssh.";
};
stateDir = mkOption {
type = types.str;
default = "/var/lib/knot-static";
description = "Directory holding src/, out/, keys.sealed, and master.env.";
};
environmentFile = mkOption {
type = types.str;
default = "${cfg.stateDir}/master.env";
description = "Environment file holding KNOT_STATIC_MASTER_KEY.";
};
listen = mkOption {
type = types.str;
default = "127.0.0.1:8080";
description = "Loopback address knot-static's inner nginx listens on.";
};
btrfsNoCompress = mkOption {
type = types.bool;
default = true;
description = "Whether to mark stateDir uncompressed on btrfs.";
};
settings = mkOption {
type = tomlFormat.type;
default = { };
description = "Extra or override settings merged into /etc/knot-static/config.toml.";
};
nginx = {
enable = mkOption {
type = types.bool;
default = true;
description = "Whether to configure the outer services.nginx virtualHost for hostname.";
};
virtualHost = mkOption {
type = types.attrsOf types.anything;
default = {
useACMEHost = cfg.hostname;
forceSSL = true;
};
description = "Extra attributes merged into services.nginx.virtualHosts.<hostname>.";
};
};
};
config = mkIf cfg.enable (mkMerge [
{
environment.systemPackages = [ cfg.package ];
environment.etc."knot-static/config.toml".source = tomlFormat.generate "knot-static.toml" settings;
# stable path for authorized_keys, store paths go stale on every rebuild
environment.etc."knot-static/git-ssh".source = gitSsh;
# sshd refuses an AuthorizedKeysCommand under the group-writable /nix/store; a mode copies it out
environment.etc."knot-static/keys-list" = {
source = keysList;
mode = "0755";
};
users.users = mkMerge (
[
{
${cfg.user} = {
isSystemUser = true;
inherit (cfg) group;
home = state;
createHome = true;
shell = pkgs.bashInteractive;
openssh.authorizedKeys.keys = operatorKeyLines;
};
}
]
++ map (u: {
${u} = {
isSystemUser = true;
inherit (cfg) group;
home = "/var/empty";
shell = pkgs.bashInteractive;
openssh.authorizedKeys.keys = operatorKeyLines;
};
}) extraSshUsers
);
users.groups.${cfg.group} = { };
security.sudo.extraRules = mkIf (extraSshUsers != [ ]) [
{
users = extraSshUsers;
runAs = "${cfg.user}:${cfg.group}";
commands = [
{
command = binPath;
options = [
"NOPASSWD"
"SETENV"
];
}
];
}
];
services.openssh.extraConfig = ''
Match User ${lib.concatStringsSep "," cfg.sshUsers}
AuthorizedKeysCommand /etc/knot-static/keys-list
AuthorizedKeysCommandUser ${cfg.user}
# git sends its protocol version in it; without, every ssh fetch is refused
AcceptEnv GIT_PROTOCOL
Match all
'';
# activate writes active.conf, the first one starts the server
systemd.paths.knot-static-nginx = {
wantedBy = [ "multi-user.target" ];
pathConfig.PathExists = "${out}/active.conf";
};
systemd.services.knot-static-nginx = {
description = "knot-static serving nginx";
after = [ "network.target" ];
serviceConfig = {
User = cfg.user;
Group = cfg.group;
ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p ${out}/runtime";
ExecStart = "${pkgs.nginx}/bin/nginx -e stderr -p ${out}/runtime/ -c ${out}/active.conf -g 'daemon off;'";
ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
Restart = "on-failure";
RestartSec = "5s";
};
};
# git packs are already compressed: new files under the state inherit
# FS_NOCOMP_FL (+m), so btrfs stores them as they are. tmpfiles's h rule only
# knows [aAcCdDeijPsStTu] (no m), so set the btrfs property directly
systemd.services.knot-static-nocompress = mkIf cfg.btrfsNoCompress {
description = "mark knot-static's state directory uncompressed on btrfs";
wantedBy = [ "multi-user.target" ];
before = [
"knot-static-nginx.service"
"knot-static-cgi.service"
"knot-static-drain.service"
"knot-static-repack.service"
];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
ExecStart = "${pkgs.btrfs-progs}/bin/btrfs property set ${state} compression none";
};
};
systemd.services.knot-static-cgi = {
description = "fcgiwrap for knot-static's cob writes";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
path = activatePath;
environment.KNOT_STATIC_CONFIG = configPath;
unitConfig.ConditionPathExists = cfg.environmentFile;
serviceConfig = {
User = cfg.user;
Group = cfg.group;
EnvironmentFile = cfg.environmentFile;
RuntimeDirectory = "knot-static";
# one worker would let a slow push hold up every write
ExecStart = "${pkgs.fcgiwrap}/bin/fcgiwrap -f -c 4 -s unix:${socket}";
Restart = "on-failure";
RestartSec = "5s";
};
};
# subscribeRepos: activations append finished websocket frames to
# ${out}/firehose/frames, and this server only sends that file on to
# whoever knot-static's nginx hands it
systemd.sockets.knot-static-firehose = {
wantedBy = [ "sockets.target" ];
socketConfig = {
ListenStream = firehoseSocket;
SocketUser = cfg.user;
SocketGroup = cfg.group;
SocketMode = "0600";
};
};
systemd.services.knot-static-firehose = {
description = "knot-static's subscribeRepos server";
requires = [ "knot-static-firehose.socket" ];
serviceConfig = {
User = cfg.user;
Group = cfg.group;
ExecStart = "${lib.getExe cfg.firehosePackage} ${out}/firehose/frames";
Restart = "on-failure";
RestartSec = "5s";
MemoryMax = "128M";
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateNetwork = true;
RestrictAddressFamilies = "AF_UNIX";
};
};
# it follows the file through its directory, which the first activation makes
systemd.tmpfiles.rules = [ "d ${out}/firehose 0755 ${cfg.user} ${cfg.group} -" ];
# pushes deliver spindle events themselves, this retries the ones a spindle missed
systemd.services.knot-static-drain = {
description = "deliver knot-static's queued spindle events";
# post-activate publishes refreshed handles
path = activatePath;
unitConfig.ConditionPathExists = cfg.environmentFile;
serviceConfig = {
Type = "oneshot";
User = cfg.user;
Group = cfg.group;
EnvironmentFile = cfg.environmentFile;
ExecStart = "${binPath} --config ${configPath} post-activate";
};
};
systemd.timers.knot-static-drain = {
wantedBy = [ "timers.target" ];
timerConfig.OnUnitActiveSec = "5min";
timerConfig.OnBootSec = "5min";
};
# a new binary may serve from files the old one didn't write, lay them out now
# instead of in the first write after the switch, then publish what it builds
systemd.services.knot-static-prepare = {
description = "lay out what a new knot-static serves from and publish it";
wantedBy = [ "multi-user.target" ];
after = [ "knot-static-nginx.service" ];
restartTriggers = [
cfg.package
config.environment.etc."knot-static/config.toml".source
];
path = activatePath;
unitConfig.ConditionPathExists = cfg.environmentFile;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
User = cfg.user;
Group = cfg.group;
EnvironmentFile = cfg.environmentFile;
ExecStart = "${binPath} --config ${configPath} prepare";
Nice = 19;
IOSchedulingClass = "idle";
};
};
# rolls each hosted repo's pushed packs and loose objects into one, keeping
# the big pack the exports hardlink
systemd.services.knot-static-repack = {
description = "repack knot-static's hosted repos";
path = [ pkgs.gitMinimal ];
unitConfig.ConditionPathExists = cfg.environmentFile;
serviceConfig = {
Type = "oneshot";
User = cfg.user;
Group = cfg.group;
EnvironmentFile = cfg.environmentFile;
ExecStart = "${binPath} --config ${configPath} maintenance --all --command repack";
Nice = 19;
IOSchedulingClass = "idle";
# the first night of a kernel-sized repo peaks around 2 GB (the bitmap
# writer beside git's walk), later nights well under
MemoryMax = "4G";
};
};
systemd.timers.knot-static-repack = {
wantedBy = [ "timers.target" ];
# 6-7 in europe is around midnight in the us, quiet on both
timerConfig.OnCalendar = "*-*-* 06:00:00 Europe/Berlin";
timerConfig.RandomizedDelaySec = "1h";
timerConfig.Persistent = true;
};
# one access log per generation, and the active one runs until the next push
services.logrotate.settings.knot-static = {
files = "${out}/runtime/*/*.log";
su = "${cfg.user} ${cfg.group}";
frequency = "daily";
rotate = 7;
compress = true;
missingok = true;
notifempty = true;
postrotate = "${pkgs.systemd}/bin/systemctl kill -s USR1 knot-static-nginx.service";
};
}
(mkIf cfg.nginx.enable {
services.nginx.virtualHosts.${cfg.hostname} = lib.recursiveUpdate {
# a pack's rate is per request, so one http/2 connection's streams each get
# their own: sixteen is more than any client needs and bounds the multiple
extraConfig = ''
http2_max_concurrent_streams 16;
'';
locations."/" = {
proxyPass = "http://${cfg.listen}";
extraConfig = ''
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# a pack is a file knot-static's nginx already has: relay it at the
# client's speed instead of copying it into a temp file first
proxy_buffering off;
proxy_max_temp_file_size 0;
'';
};
# canned git replies are matched on bodies up to 16 MiB; everything else
# keeps the 1 MiB default, so a big POST to any other path is a 413
locations."~ ^/[^/]+/git-upload-pack$" = {
proxyPass = "http://${cfg.listen}";
extraConfig = ''
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
client_max_body_size 16m;
proxy_buffering off;
proxy_max_temp_file_size 0;
'';
};
# knot-static's own nginx limits these and streams them to the cgi
locations."~ ^/[^/]+/git-receive-pack$" = {
proxyPass = "http://${cfg.listen}";
extraConfig = ''
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
client_max_body_size 8g;
proxy_request_buffering off;
proxy_read_timeout 3600s;
'';
};
# firehose websockets stay open; the server pings every 10 s
locations."= /xrpc/com.atproto.sync.subscribeRepos" = {
proxyPass = "http://${cfg.listen}";
proxyWebsockets = true;
extraConfig = ''
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_buffering off;
proxy_read_timeout 120s;
'';
};
} cfg.nginx.virtualHost;
})
]);
}