use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::Context as _;
use knot_cobs::Registration;
use knot_types::{BranchName, KnotServiceUrl, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds};
use crate::cli::{CreateArgs, ImportArgs, ImportFrom};
use crate::config::ValidatedConfig;
use crate::registry::{
PendingGenesis, RotationSource, drop_pending_genesis, enqueue_pending_genesis,
find_rotation_signer, load_pending_genesis, open_foreign_store, verify_did_document,
};
enum ProvisionSource {
Create,
Import(PathBuf),
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Genesis {
Live,
Pending,
}
#[derive(Default)]
struct AdoptOptions {
did: Option<String>,
rotation_key_env: Option<String>,
from_sealed_store: Option<PathBuf>,
from_master_key_env: Option<String>,
from_knot_did: Option<String>,
}
struct SourceManifest {
owner: OwnerDid,
name: RepoName,
rkey: RepoRkey,
policy: knot_types::ContributionPolicy,
created_at: UnixSeconds,
}
struct Knot2Source {
scan_path: PathBuf,
repo_git_dir: PathBuf,
meta_path: Option<PathBuf>,
knot_did: Option<knot_types::KnotId>,
sealed_key_file: Option<PathBuf>,
master_key_env: Option<String>,
registry_manifest: Option<SourceManifest>,
}
fn discover_knot2_web_did(scan_path: &Path) -> Option<knot_types::KnotId> {
let web_dir = scan_path.join("web");
let mut found = Vec::new();
for shard_entry in std::fs::read_dir(&web_dir).ok()?.flatten() {
if !shard_entry.file_type().ok()?.is_dir() {
continue;
}
let shard = shard_entry.file_name().to_string_lossy().into_owned();
for rem_entry in std::fs::read_dir(shard_entry.path()).ok()?.flatten() {
if rem_entry.path().join("HEAD").is_file() {
let rem = rem_entry.file_name().to_string_lossy().into_owned();
if let Ok(id) = knot_types::KnotId::new(format!("did:web:{shard}{rem}")) {
found.push(id);
}
}
}
}
(found.len() == 1).then(|| found.pop().unwrap())
}
fn resolve_knot2_source(
input_path: &Path,
repo_did: &RepoDid,
args: &ImportArgs,
default_master_env: &str,
) -> anyhow::Result<Knot2Source> {
let mut cfg_scan_path: Option<PathBuf> = None;
let mut cfg_sealed_store: Option<PathBuf> = None;
let mut cfg_master_env: Option<String> = None;
let mut cfg_knot_did: Option<String> = None;
if input_path.is_file() {
let raw = std::fs::read_to_string(input_path)
.with_context(|| format!("read knot2 config {}", input_path.display()))?;
let table: toml::Table = toml::from_str(&raw)
.with_context(|| format!("parse knot2 config {}", input_path.display()))?;
let base_dir = input_path.parent().unwrap_or_else(|| Path::new("."));
let resolve_rel = |p: &str| -> PathBuf {
let pb = PathBuf::from(p);
if pb.is_relative() {
base_dir.join(pb)
} else {
pb
}
};
if let Some(sp) = table
.get("repo")
.and_then(|v| v.get("scan_path"))
.and_then(toml::Value::as_str)
{
cfg_scan_path = Some(resolve_rel(sp));
}
if let Some(sk) = table
.get("secrets")
.and_then(|v| v.get("sealed_key_file"))
.and_then(toml::Value::as_str)
{
cfg_sealed_store = Some(resolve_rel(sk));
}
if let Some(me) = table
.get("secrets")
.and_then(|v| v.get("master_key_env"))
.and_then(toml::Value::as_str)
{
cfg_master_env = Some(me.to_string());
}
if let Some(host) = table
.get("server")
.or_else(|| table.get("knot"))
.and_then(|v| v.get("hostname"))
.and_then(toml::Value::as_str)
{
let encoded = host.replace(':', "%3A");
cfg_knot_did = Some(format!("did:web:{encoded}"));
}
}
let scan_path = if let Some(sp) = cfg_scan_path {
sp
} else if input_path.join("repos").is_dir() {
input_path.join("repos")
} else if input_path.join("HEAD").is_file() {
input_path
.parent()
.and_then(Path::parent)
.and_then(Path::parent)
.map(Path::to_path_buf)
.context("infer knot2 scan_path from bare repo path")?
} else {
input_path.to_path_buf()
};
let sealed_key_file = args
.from_sealed_store
.clone()
.or(cfg_sealed_store)
.or_else(|| {
let direct = input_path.join("sealed-keys");
if direct.is_file() {
return Some(direct);
}
let parent = scan_path.parent()?.join("sealed-keys");
parent.is_file().then_some(parent)
});
let master_key_env = args
.from_master_key_env
.clone()
.or(cfg_master_env)
.or_else(|| {
if std::env::var_os("KNOT_MASTER_KEY").is_some() {
Some("KNOT_MASTER_KEY".to_string())
} else {
Some(default_master_env.to_string())
}
});
let knot_did = match args.from_knot_did.as_deref().or(cfg_knot_did.as_deref()) {
Some(raw) => Some(knot_types::KnotId::new(raw).context("parse source knot DID")?),
None => discover_knot2_web_did(&scan_path),
};
let layout = knot_git::Layout::new(&scan_path);
let repo_git_dir = if input_path.join("HEAD").is_file() {
input_path.to_path_buf()
} else {
layout
.guarded_path(repo_did)
.context("resolve repo path in knot2 scan_path")?
};
let meta_path = knot_did
.as_ref()
.and_then(|did| layout.meta_path(did).ok())
.filter(|p| p.join("HEAD").is_file());
let registry_manifest = meta_path
.as_ref()
.and_then(|mp| knot_git::Repo::open(mp).ok())
.and_then(|meta| crate::registry::load_state(&meta).ok().flatten())
.and_then(|reg| {
let rec = reg.record_of(repo_did)?;
Some(SourceManifest {
owner: rec.owner.clone(),
name: rec.name.clone(),
rkey: rec.rkey.clone(),
policy: rec.policy,
created_at: rec.created_at,
})
});
Ok(Knot2Source {
scan_path,
repo_git_dir,
meta_path,
knot_did,
sealed_key_file,
master_key_env,
registry_manifest,
})
}
pub(crate) fn chown_tree_to_dir_owner(target: &Path, reference_dir: &Path) {
#[cfg(unix)]
{
use std::os::unix::fs::{MetadataExt as _, lchown};
let Ok(meta) = std::fs::metadata(reference_dir) else {
return;
};
let (uid, gid) = (meta.uid(), meta.gid());
fn walk(path: &Path, uid: u32, gid: u32) {
let _ = lchown(path, Some(uid), Some(gid));
if let Ok(entries) = std::fs::read_dir(path) {
for entry in entries.flatten() {
let p = entry.path();
let _ = lchown(&p, Some(uid), Some(gid));
if entry.file_type().is_ok_and(|ft| ft.is_dir()) {
walk(&p, uid, gid);
}
}
}
}
walk(target, uid, gid);
}
}
fn deregister_from_knot2(source: &Knot2Source, repo_did: &RepoDid) -> anyhow::Result<()> {
if let Some(ref meta_path) = source.meta_path
&& let Ok(meta_repo) = knot_git::Repo::open(meta_path)
{
let _ = drop_pending_genesis(&meta_repo, repo_did);
if let Some(reg) = crate::registry::load_state(&meta_repo)?
&& let Some(rec) = reg.record_of(repo_did)
{
let knot_did = source
.knot_did
.as_ref()
.context("source knot DID required to deregister repo in knot2 meta.git")?;
let store_path = source
.sealed_key_file
.as_deref()
.context("source sealed_key_file required to deregister repo in knot2 meta.git")?;
let master_env = source
.master_key_env
.as_deref()
.unwrap_or("KNOT_MASTER_KEY");
let foreign_secrets = open_foreign_store(store_path, master_env)?;
let signer = foreign_secrets.signer(knot_did).with_context(|| {
format!(
"load source knot signer for {knot_did} from {}",
store_path.display()
)
})?;
let cob_store = knot_cob::CobStore::new(&meta_repo);
if let Some(&object) = cob_store.list::<knot_cobs::RepoRegistryCob>()?.first() {
let now = UnixSeconds::new(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs() as i64,
);
knot_cobs::deregister_repo(
&cob_store,
&knot_cob::CobHome::from(knot_did),
object,
knot_cobs::RepoRef {
owner: rec.owner.clone(),
rkey: rec.rkey.clone(),
},
repo_did.clone(),
&signer,
now,
)
.context("deregister migrated repo in knot2 meta.git")?;
}
}
chown_tree_to_dir_owner(meta_path, &source.scan_path);
}
let layout = knot_git::Layout::new(&source.scan_path);
layout
.remove(repo_did)
.context("remove migrated repo directory from knot2 scan_path")?;
Ok(())
}
fn read_source_manifest(git_dir: &Path) -> anyhow::Result<Option<SourceManifest>> {
let Ok(repo) = knot_git::Repo::open(git_dir) else {
return Ok(None);
};
let store = knot_cob::CobStore::new(&repo);
let Some(&object) = store.list::<knot_cobs::ManifestCob>()?.first() else {
return Ok(None);
};
let knot_cobs::SocialState::Live(live) =
store.get::<knot_cobs::ManifestCob>(object)?.into_state()
else {
return Ok(None);
};
let Some(version) = live.version() else {
return Ok(None);
};
let type_name = knot_types::TypeName::new(knot_record::manifest::MANIFEST_COLLECTION)?;
let body = store
.changes_of(&type_name, object, knot_cob::HistoryModel::Convergent, None)?
.changes
.into_iter()
.rev()
.filter_map(|change| change.record.into_body())
.find(|body| body.cid() == version)
.context("live manifest cob is missing its record bytes")?;
let stored: crate::cob_write::StoredManifest =
serde_ipld_dagcbor::from_slice(body.as_bytes()).context("decode manifest cob record")?;
let rkey = RepoRkey::new(stored.declaration.name.as_str().to_lowercase())
.context("manifest cob name does not form a valid rkey")?;
Ok(Some(SourceManifest {
owner: stored.declaration.owner,
name: stored.declaration.name,
rkey,
policy: stored.contribution_policy.unwrap_or_default(),
created_at: live.created_at(),
}))
}
pub async fn import(config: Arc<ValidatedConfig>, args: ImportArgs) -> anyhow::Result<()> {
let input_path = args.path().to_path_buf();
let knot2_source = match args.from {
Some(ImportFrom::Knot2) => {
let raw_did = args
.did
.as_deref()
.context("--did is required with --from knot2")?;
let repo_did = RepoDid::new(raw_did).context("invalid --did")?;
Some(resolve_knot2_source(
&input_path,
&repo_did,
&args,
&config.master_key_env,
)?)
}
None => None,
};
let source_repo_path = knot2_source
.as_ref()
.map(|s| s.repo_git_dir.clone())
.unwrap_or_else(|| input_path.clone());
let dest_existing = args
.did
.as_deref()
.and_then(|d| RepoDid::new(d).ok())
.and_then(|did| config.layout().ok()?.guarded_path(&did).ok())
.filter(|p| p.join("HEAD").is_file());
let effective_repo_path = if source_repo_path.exists() {
source_repo_path.clone()
} else if let Some(ref existing) = dest_existing {
existing.clone()
} else {
source_repo_path.clone()
};
let lfs_pointers = detect_lfs_pointers(&effective_repo_path)?;
if !lfs_pointers.is_empty() {
anyhow::ensure!(
args.allow_lfs_pointers,
"detected {} Git LFS pointer file(s) (e.g. {}). Static knots serve the pointers as plain content; standard `git lfs pull` will not work. Re-run with --allow-lfs-pointers to import anyway.",
lfs_pointers.len(),
lfs_pointers.first().map_or("?", String::as_str)
);
eprintln!(
"warning: importing {} Git LFS pointer file(s); the static knot serves pointers as plain content",
lfs_pointers.len()
);
}
let git_dir = resolve_git_dir(&effective_repo_path)
.with_context(|| format!("resolve git dir for {}", effective_repo_path.display()))?;
let dest_registry_manifest = args
.did
.as_deref()
.and_then(|d| RepoDid::new(d).ok())
.and_then(|did| {
let meta = config
.layout()
.ok()?
.bootstrap_meta(&config.knot_did)
.ok()?;
let reg = crate::registry::load_state(&meta).ok()??;
let rec = reg.record_of(&did)?;
Some(SourceManifest {
owner: rec.owner.clone(),
name: rec.name.clone(),
rkey: rec.rkey.clone(),
policy: rec.policy,
created_at: rec.created_at,
})
});
let manifest = match knot2_source
.as_ref()
.and_then(|s| s.registry_manifest.as_ref())
.or(dest_registry_manifest.as_ref())
{
Some(m) => Some(SourceManifest {
owner: m.owner.clone(),
name: m.name.clone(),
rkey: m.rkey.clone(),
policy: m.policy,
created_at: m.created_at,
}),
None => read_source_manifest(&git_dir)?,
};
let raw_owner = match (&args.owner, &manifest) {
(Some(raw), Some(m)) => {
let given = OwnerDid::new(raw).context("invalid owner DID")?;
anyhow::ensure!(
given == m.owner,
"source manifest declares owner {}, which conflicts with --owner {given}",
m.owner
);
raw.clone()
}
(Some(raw), None) => raw.clone(),
(None, Some(m)) => m.owner.as_str().to_string(),
(None, None) => config.owner.as_str().to_string(),
};
let raw_name = args
.name
.clone()
.or_else(|| manifest.as_ref().map(|m| m.name.as_str().to_string()));
let raw_rkey = match (&args.rkey, &raw_name, &manifest) {
(Some(rkey), _, _) => rkey.clone(),
(None, _, Some(m)) => m.rkey.as_str().to_string(),
(None, Some(name), None) => name.to_lowercase(),
(None, None, None) => {
anyhow::bail!("--rkey (or --name) is required when the source repo has no manifest cob")
}
};
let source_annotation = args.source.clone();
let policy = manifest.as_ref().map(|m| m.policy).unwrap_or_default();
let manifest_created_at = manifest.as_ref().map(|m| m.created_at.get());
let adopt_options = AdoptOptions {
did: args.did.clone(),
rotation_key_env: args.rotation_key_env.clone(),
from_sealed_store: knot2_source
.as_ref()
.and_then(|s| s.sealed_key_file.clone())
.or(args.from_sealed_store.clone()),
from_master_key_env: knot2_source
.as_ref()
.and_then(|s| s.master_key_env.clone())
.or(args.from_master_key_env.clone()),
from_knot_did: knot2_source
.as_ref()
.and_then(|s| s.knot_did.as_ref().map(|d| d.as_str().to_string()))
.or(args.from_knot_did.clone()),
};
let repo_did = provision_repo(
config.clone(),
raw_owner,
raw_rkey,
raw_name,
args.default_branch,
source_annotation,
policy,
manifest_created_at,
adopt_options,
ProvisionSource::Import(source_repo_path),
Genesis::Live,
)
.await?;
if let Ok(layout) = config.layout() {
if let Ok(repo_path) = layout.guarded_path(&repo_did) {
chown_tree_to_dir_owner(&repo_path, &config.scan_path);
}
if let Ok(meta_path) = layout.meta_path(&config.knot_did) {
chown_tree_to_dir_owner(&meta_path, &config.scan_path);
}
}
if let Some(ref k2) = knot2_source {
deregister_from_knot2(k2, &repo_did)?;
let pub_res = crate::writes::publish(&config, &repo_did)
.await
.context("generate and activate after migration");
let _ = tokio::join!(
crate::index_page::sync_handles(&config),
crate::keys::sync(&config, false)
);
chown_tree_to_dir_owner(&config.output_root, &config.scan_path);
pub_res?;
}
println!("{repo_did}");
Ok(())
}
const LFS_POINTER_PREFIX: &str = "version https://git-lfs";
const LFS_SCAN_MAX_BLOB_BYTES: u64 = 4096;
fn detect_lfs_pointers(path: &std::path::Path) -> anyhow::Result<Vec<String>> {
let git = |git_args: &[&str]| {
let mut command = std::process::Command::new("git");
crate::git_export::true_history(&mut command)
.arg("-C")
.arg(path)
.args(git_args)
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_CONFIG_SYSTEM", "/dev/null")
.env("GIT_CONFIG_COUNT", "1")
.env("GIT_CONFIG_KEY_0", "safe.directory")
.env("GIT_CONFIG_VALUE_0", "*")
.env("GIT_TERMINAL_PROMPT", "0");
command
};
let run_git = |git_args: &[&str]| -> anyhow::Result<String> {
let output = git(git_args).output().context("run git for LFS scan")?;
if !output.status.success() {
anyhow::bail!(
"git {} failed: {}",
git_args.first().unwrap_or(&""),
String::from_utf8_lossy(&output.stderr)
);
}
Ok(String::from_utf8_lossy(&output.stdout).to_string())
};
if path.join(".git").exists() || path.join("HEAD").exists() {
if let Ok(out) = run_git(&["grep", "-I", "-l", LFS_POINTER_PREFIX, "--", "."]) {
let hits: Vec<String> = out.lines().map(str::to_string).collect();
if !hits.is_empty() {
return Ok(hits);
}
}
}
use std::io::{BufRead as _, Read as _, Write as _};
use std::process::Stdio;
let mut listing = git(&[
"cat-file",
"--batch-all-objects",
"--batch-check=%(objectname) %(objecttype) %(objectsize)",
])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.context("run git cat-file --batch-all-objects for LFS scan")?;
let mut batch = git(&["cat-file", "--batch"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.context("run git cat-file --batch for LFS scan")?;
let objects = std::io::BufReader::new(listing.stdout.take().expect("piped stdout"));
let wanted = batch.stdin.take().expect("piped stdin");
let answers = batch.stdout.take().expect("piped stdout");
let mut pointers = Vec::new();
// git answers while it reads. the oids feed from another thread or both pipes fill
// `answers` drops before the scope joins the feeder. an early error can't block it
let read = std::thread::scope(|scope| -> anyhow::Result<()> {
let mut answers = std::io::BufReader::new(answers);
let feed = scope.spawn(move || -> std::io::Result<()> {
let mut wanted = std::io::BufWriter::new(wanted);
for line in objects.lines() {
let line = line?;
let mut parts = line.split_whitespace();
if let (Some(oid), Some("blob"), Some(size)) =
(parts.next(), parts.next(), parts.next())
&& size
.parse::<u64>()
.is_ok_and(|size| (1..=LFS_SCAN_MAX_BLOB_BYTES).contains(&size))
{
writeln!(wanted, "{oid}")?;
}
}
wanted.flush()
});
let (mut header, mut content) = (String::new(), Vec::new());
while answers.read_line(&mut header)? > 0 {
let mut parts = header.split_whitespace();
let (Some(oid), Some("blob"), Some(size)) = (parts.next(), parts.next(), parts.next())
else {
anyhow::bail!("git cat-file --batch said {:?}", header.trim_end());
};
content.resize(size.parse::<usize>()? + 1, 0);
answers.read_exact(&mut content)?;
if content.starts_with(LFS_POINTER_PREFIX.as_bytes()) {
pointers.push(oid.to_string());
}
header.clear();
}
feed.join().expect("feed thread")?;
Ok(())
});
for (what, child) in [("--batch-all-objects", listing), ("--batch", batch)] {
let output = child.wait_with_output().context("wait for git cat-file")?;
anyhow::ensure!(
output.status.success(),
"git cat-file {what} failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
read?;
Ok(pointers)
}
pub async fn create(config: Arc<ValidatedConfig>, args: CreateArgs) -> anyhow::Result<()> {
let repo_did = provision_repo(
config.clone(),
args.owner,
args.rkey,
args.name,
args.default_branch,
None,
knot_types::ContributionPolicy::default(),
None,
AdoptOptions::default(),
ProvisionSource::Create,
Genesis::Live,
)
.await?;
println!("{repo_did}");
Ok(())
}
pub(crate) async fn create_for_push(
config: Arc<ValidatedConfig>,
raw_rkey: String,
raw_name: Option<String>,
) -> anyhow::Result<RepoDid> {
let owner = config.owner.as_str().to_string();
provision_repo(
config,
owner,
raw_rkey,
raw_name,
None,
None,
knot_types::ContributionPolicy::default(),
None,
AdoptOptions::default(),
ProvisionSource::Create,
Genesis::Live,
)
.await
}
pub(crate) async fn create_for_xrpc(
config: Arc<ValidatedConfig>,
owner: &OwnerDid,
name: &RepoName,
rkey: &RepoRkey,
default_branch: &BranchName,
source: Option<String>,
) -> anyhow::Result<RepoDid> {
provision_repo(
config,
owner.as_str().to_string(),
rkey.as_str().to_string(),
Some(name.as_str().to_string()),
Some(default_branch.as_str().to_string()),
source,
knot_types::ContributionPolicy::default(),
None,
AdoptOptions::default(),
ProvisionSource::Create,
Genesis::Pending,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn provision_repo(
config: Arc<ValidatedConfig>,
raw_owner: String,
raw_rkey: String,
raw_name: Option<String>,
default_branch: Option<String>,
source_annotation: Option<String>,
policy: knot_types::ContributionPolicy,
manifest_created_at: Option<i64>,
adopt: AdoptOptions,
source: ProvisionSource,
genesis: Genesis,
) -> anyhow::Result<RepoDid> {
use knot_runtime::Signer as _;
let owner = OwnerDid::new(&raw_owner).context("invalid owner DID")?;
let rkey = RepoRkey::new(&raw_rkey).context("invalid repo rkey")?;
let name = match raw_name {
Some(name) => RepoName::new(name).context("invalid repo name")?,
None => RepoName::new(raw_rkey.clone()).context("invalid repo name from rkey")?,
};
let explicit_did = match adopt.did.as_deref() {
Some(raw) => {
let did = RepoDid::new(raw).context("invalid --did")?;
anyhow::ensure!(
did.as_str().starts_with("did:plc:"),
"--did must be a did:plc identifier, got {did}"
);
anyhow::ensure!(
did.as_str() != config.knot_did.as_str(),
"--did must not be the knot's own identity"
);
Some(did)
}
None => None,
};
let layout = config.layout()?;
let meta_path = layout.meta_path(&config.knot_did)?;
let meta_repo = layout
.bootstrap_meta(&config.knot_did)
.context("bootstrap meta.git")?;
let index = knot_index::Index::new(&meta_path, layout.clone());
let _ = index.rebuild();
let existing_resolved = match index.resolve_repo(&owner, &rkey) {
knot_index::Resolved::Ready(Some(did))
if matches!(
index.rkey_of(&did),
knot_index::Resolved::Ready(Some(ref canonical)) if canonical == &rkey
) =>
{
Some(did)
}
_ => None,
};
if let Some(ref did) = explicit_did {
if let knot_index::Resolved::Ready(Some(existing)) = index.ownership_of(did)
&& (existing.owner != owner || existing.rkey != rkey)
{
anyhow::bail!(
"{did} is already hosted here as ({}, {})",
existing.owner,
existing.rkey
);
}
if let Some(ref existing_did) = existing_resolved
&& existing_did != did
{
anyhow::bail!("repository ({owner}, {rkey}) already hosted as {existing_did}");
}
}
let pending_entries = load_pending_genesis(&meta_repo)?;
if let Some(ref did) = explicit_did
&& let Some(conflict) = pending_entries.iter().find(|entry| {
entry.did != *did
&& entry.owner.as_ref() == Some(&owner)
&& entry.rkey.as_ref() == Some(&rkey)
})
{
anyhow::bail!(
"repository ({owner}, {rkey}) already has a pending genesis as {}",
conflict.did
);
}
let pending_match = pending_entries.iter().find(|entry| match &explicit_did {
Some(did) => &entry.did == did,
None => {
(entry.owner.as_ref() == Some(&owner) && entry.rkey.as_ref() == Some(&rkey))
|| existing_resolved.as_ref() == Some(&entry.did)
}
});
if pending_match.is_none()
&& explicit_did.is_none()
&& let Some(existing_did) = existing_resolved
{
anyhow::bail!("repository ({owner}, {rkey}) already hosted as {existing_did}");
}
let secrets = crate::identity::open_store(&config)?;
secrets
.ensure(&config.knot_did)
.context("ensure knot signing key")?;
let knot_signer = secrets
.signer(&config.knot_did)
.context("get knot signer")?;
let service_url =
KnotServiceUrl::new(format!("https://{}", config.hostname)).context("knot service url")?;
let existing_created_at = explicit_did.as_ref().and_then(|did| {
crate::registry::load_state(&meta_repo)
.ok()
.flatten()
.and_then(|reg| reg.record_of(did).map(|r| r.created_at.get()))
});
let (seed, created_at, pending_op) = match pending_match {
Some(entry) => (
entry.seed,
existing_created_at.unwrap_or(entry.created_at),
Some((entry.did.clone(), entry.operation.clone())),
),
None => {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap();
(
now.as_nanos() as u64,
existing_created_at
.or(manifest_created_at)
.unwrap_or(now.as_secs() as i64),
None,
)
}
};
let transport =
knot_runtime::ReqwestHttp::new(config.http_limits()).context("init http transport")?;
let atproto = config.atproto(transport.clone())?;
let (repo_did, plc_operation) = if let Some(did) = explicit_did.clone() {
let target = knot_atproto::RepoTarget::new(&knot_signer.public_key(), &service_url);
let last_op = atproto
.last_plc_operation(&did)
.await
.context("fetch last plc operation for adopted repo")?
.with_context(|| {
format!("no plc operation found on directory for adopted repo {did}")
})?;
let op = if target
.satisfied_by(&last_op)
.context("check adopted repo plc operation")?
{
None
} else {
let from_master_env = adopt
.from_master_key_env
.as_deref()
.unwrap_or(&config.master_key_env);
let signer = find_rotation_signer(
&last_op,
&did,
&secrets,
&config.knot_did,
&RotationSource {
rotation_key_env: adopt.rotation_key_env.as_deref(),
from_sealed_store: adopt.from_sealed_store.as_deref(),
from_master_key_env: from_master_env,
from_knot_did: adopt.from_knot_did.as_deref(),
},
)?;
let update = knot_atproto::update_operation(&signer, &last_op, &target)
.context("sign plc update operation for adopted repo")?;
Some(update)
};
(did, op)
} else if let Some((did, op)) = pending_op {
(did, Some(op))
} else {
let entropy = knot_runtime::SeededEntropy::new(seed);
let nonce = knot_atproto::MintNonce::mint(&entropy, &owner, &rkey);
let prepared = knot_atproto::prepare_repo_did(&knot_signer, &service_url, &nonce)
.context("prepare repo did")?;
(prepared.did.clone(), Some(prepared.operation().clone()))
};
let guarded_path = layout.guarded_path(&repo_did)?;
if guarded_path.exists() {
knot_git::Repo::open(&guarded_path).context("open existing repo on resume")?;
} else {
match source {
ProvisionSource::Create => {
let repo = layout.create(&repo_did).context("create bare repo")?;
if let Some(branch_str) = &default_branch {
let branch = BranchName::new(branch_str.clone())
.context("invalid default branch name")?;
repo.set_head(&branch)
.context("set repo head to default branch")?;
}
}
ProvisionSource::Import(source_path) => {
let git_dir = resolve_git_dir(&source_path)
.with_context(|| format!("resolve git dir for {}", source_path.display()))?;
knot_git::Repo::open(&git_dir).with_context(|| {
format!("verify knot_git::Repo opens {}", git_dir.display())
})?;
refuse_other_history(&git_dir)?;
let temp_dir = tempfile::Builder::new()
.prefix(".import-staging-")
.tempdir_in(&config.scan_path)
.context("create staging tempdir in scan path")?;
copy_dir_recursive(&git_dir, temp_dir.path()).context("stage git repository")?;
make_bare(temp_dir.path()).context("make the staged repository bare")?;
let staged_repo = knot_git::Repo::open(temp_dir.path())
.context("verify knot_git::Repo opens staged repository")?;
if let Some(branch_str) = &default_branch {
let branch = BranchName::new(branch_str.clone())
.context("invalid default branch name")?;
staged_repo
.set_head(&branch)
.context("set repo head to default branch")?;
}
// the import is the one cold graph write, and later pushes only add a layer
crate::commit_graph::rewrite(temp_dir.path());
drop_derived_indexes(temp_dir.path()).context("drop the source's pack indexes")?;
if let Some(parent) = guarded_path.parent() {
std::fs::create_dir_all(parent)
.context("create parent directory for guarded path")?;
}
std::fs::rename(temp_dir.path(), &guarded_path)
.context("move staged repository to guarded path")?;
let _ = temp_dir.keep();
}
}
}
let cob_store = knot_cob::CobStore::new(&meta_repo);
let cob_home = knot_cob::CobHome::from(&config.knot_did);
let now = UnixSeconds::new(created_at);
let registration = Registration {
owner: owner.clone(),
rkey: rkey.clone(),
name: name.clone(),
repo: repo_did.clone(),
created_at: now,
policy,
fostered: false,
retired_at: None,
};
register_in_meta(&cob_store, &cob_home, registration, &knot_signer, now)?;
if let Some(operation) = &plc_operation {
let entry = PendingGenesis {
did: repo_did.clone(),
operation: operation.clone(),
owner: Some(owner.clone()),
rkey: Some(rkey.clone()),
name: Some(name.to_string()),
source: source_annotation,
seed,
created_at,
};
enqueue_pending_genesis(&meta_repo, entry)?;
}
let live = async {
if let Some(operation) = &plc_operation {
atproto
.submit_plc_operation(&repo_did, operation)
.await
.context("submit plc operation")?;
}
verify_did_document(
&transport,
&config.plc_directory,
&repo_did,
&config.hostname,
)
.await
.context("verify live repo DID document on PLC")
}
.await;
match (live, genesis) {
(Ok(()), _) => {
drop_pending_genesis(&meta_repo, &repo_did)?;
}
(Err(error), Genesis::Pending) => {
tracing::warn!(repo = %repo_did, "the genesis stays pending: {error:#}");
}
(Err(error), Genesis::Live) => return Err(error),
}
let fresh_index = knot_index::Index::new(&meta_path, layout.clone());
fresh_index
.rebuild()
.context("rebuild fresh index after provisioning")?;
match fresh_index.ownership_of(&repo_did) {
knot_index::Resolved::Ready(Some(ref_info))
if ref_info.owner == owner && ref_info.rkey == rkey => {}
knot_index::Resolved::Ready(Some(ref_info)) => {
anyhow::bail!(
"ownership mismatch for {repo_did}: expected ({owner}, {rkey}), found ({}, {})",
ref_info.owner,
ref_info.rkey
);
}
other => {
anyhow::bail!("failed to resolve ownership of {repo_did} in fresh index: {other:?}");
}
}
Ok(repo_did)
}
fn register_in_meta(
store: &knot_cob::CobStore,
home: &knot_cob::CobHome,
registration: Registration,
signer: &dyn knot_runtime::Signer,
now: UnixSeconds,
) -> anyhow::Result<()> {
match store.list::<knot_cobs::RepoRegistryCob>()?.as_slice() {
[] => store
.create(
home,
&knot_cobs::RegistryChange::Register(registration),
signer,
now,
)
.map(|_| ())
.map_err(|e| anyhow::anyhow!("create registry cob: {e}")),
[object] => knot_cobs::register_repo(store, home, *object, registration, signer, now)
.map(|_| ())
.map_err(|e| anyhow::anyhow!("register repo in cob: {e}")),
many => anyhow::bail!("{} repo registry objects share meta-repo", many.len()),
}
}
fn resolve_git_dir(path: &Path) -> anyhow::Result<PathBuf> {
let dot_git = path.join(".git");
if dot_git.is_dir() {
Ok(dot_git)
} else if dot_git.is_file() {
let content = std::fs::read_to_string(&dot_git)?;
let rest = content
.strip_prefix("gitdir:")
.ok_or_else(|| anyhow::anyhow!("invalid .git file at {}", dot_git.display()))?;
let p = PathBuf::from(rest.trim());
Ok(if p.is_relative() { path.join(p) } else { p })
} else if path.join("HEAD").is_file() {
Ok(path.to_path_buf())
} else {
anyhow::bail!("no git repository found at {}", path.display());
}
}
/// clients see a different history for a shallow clone, grafts or replace refs.
fn refuse_other_history(git_dir: &Path) -> anyhow::Result<()> {
anyhow::ensure!(
!git_dir.join("shallow").exists(),
"{} is a shallow clone: import needs its whole history (git fetch --unshallow)",
git_dir.display()
);
anyhow::ensure!(
!git_dir.join("info/grafts").exists(),
"{} has info/grafts, which rewrite history for git alone: remove them before importing",
git_dir.display()
);
let mut list = crate::git_export::git_command(git_dir);
list.args([
"for-each-ref",
"--count=1",
"--format=%(refname)",
crate::git_ssh::REPLACE_REFS,
]);
let listed = crate::git_export::run_cmd(list, "git for-each-ref refs/replace")?;
let replaced = String::from_utf8_lossy(&listed.stdout);
anyhow::ensure!(
replaced.trim().is_empty(),
"{} has replace refs ({}), which substitute objects for git alone: delete them \
(git replace -d) before importing",
git_dir.display(),
replaced.trim()
);
Ok(())
}
/// git trusts a source's bitmap. one claiming a hidden object under a public commit would publish it.
fn drop_derived_indexes(git_dir: &Path) -> anyhow::Result<()> {
let dir = git_dir.join("objects/pack");
let entries = match std::fs::read_dir(&dir) {
Ok(entries) => entries,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error).with_context(|| format!("list {}", dir.display())),
};
for entry in entries {
let path = entry?.path();
let name = path.file_name().unwrap_or_default().to_string_lossy();
if !name.starts_with("multi-pack-index")
&& !path
.extension()
.is_some_and(|ext| ext == "bitmap" || ext == "rev")
{
continue;
}
if path.is_dir() {
std::fs::remove_dir_all(&path)
} else {
std::fs::remove_file(&path)
}
.with_context(|| format!("remove {}", path.display()))?;
}
Ok(())
}
fn make_bare(git_dir: &Path) -> anyhow::Result<()> {
match std::fs::remove_dir_all(git_dir.join("hooks")) {
Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(error.into()),
_ => {}
}
let output = std::process::Command::new("git")
.args(["config", "--file"])
.arg(git_dir.join("config"))
.args(["core.bare", "true"])
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_CONFIG_SYSTEM", "/dev/null")
.output()
.context("run git config")?;
anyhow::ensure!(
output.status.success(),
"git config core.bare failed: {}",
String::from_utf8_lossy(&output.stderr)
);
Ok(())
}
fn copy_dir_recursive(src: &Path, dst: &Path) -> anyhow::Result<()> {
std::fs::create_dir_all(dst)?;
for entry in std::fs::read_dir(src).with_context(|| format!("read {}", src.display()))? {
let entry = entry?;
if entry
.file_name()
.to_string_lossy()
.starts_with(crate::maintenance::SCRATCH)
{
continue;
}
let ft = entry.file_type()?;
let target = dst.join(entry.file_name());
if ft.is_dir() {
copy_dir_recursive(&entry.path(), &target)?;
} else if ft.is_file() {
std::fs::copy(entry.path(), &target)
.with_context(|| format!("copy {}", entry.path().display()))?;
} else if ft.is_symlink() {
let link_target = std::fs::read_link(entry.path())?;
#[cfg(unix)]
std::os::unix::fs::symlink(&link_target, &target)?;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_copy_leaves_out_a_bitmap_maintenance_is_building() {
let dir = tempfile::tempdir().unwrap();
let (src, dst) = (dir.path().join("src"), dir.path().join("dst"));
let scratch = format!("{}x", crate::maintenance::SCRATCH);
std::fs::create_dir_all(src.join(&scratch).join("objects/pack")).unwrap();
std::fs::write(src.join(&scratch).join("objects/pack/multi-pack-index"), "").unwrap();
std::fs::write(src.join("HEAD"), "ref: refs/heads/main\n").unwrap();
copy_dir_recursive(&src, &dst).unwrap();
assert!(dst.join("HEAD").exists());
assert!(!dst.join(&scratch).exists());
}
#[test]
fn read_source_manifest_extracts_owner_name_and_created_at() {
use knot_cob::{Approved, Materialized, Record, WriteContext};
use knot_record::manifest::{
MANIFEST_RKEY, ManifestMeta, ManifestRecord, ManifestSource, manifest_collection,
};
use knot_types::{
AccountDid, Blocked, ContributionPermission, ContributionPolicy, RecordAddress,
RecordRkey, RepoRole, SubjectNumber,
};
let dir = tempfile::tempdir().unwrap();
let repo_did = RepoDid::new("did:plc:migratedrepo123456789012").unwrap();
let owner = OwnerDid::new("did:plc:owner1234567890123456789").unwrap();
let editor = AccountDid::new(owner.as_str()).unwrap();
let name = RepoName::new("My-Repo").unwrap();
let created_at = UnixSeconds::new(1_700_000_123);
let repo_path = dir.path().join("repo");
let repo = knot_git::Repo::create(&repo_path).unwrap();
let store = knot_cob::CobStore::new(&repo);
let home = knot_cob::CobHome::from(&repo_did);
let signer = knot_runtime::K256Signer::generate(&knot_runtime::SeededEntropy::new(19));
let record = ManifestRecord::new(
owner.clone(),
name.clone(),
ManifestMeta {
source: Some(ManifestSource::Tangled {
did: RepoDid::new("did:plc:upstream1234567890123456").unwrap(),
}),
..ManifestMeta::default()
},
created_at,
editor.clone(),
)
.unwrap()
.body()
.unwrap();
let change: knot_cobs::ManifestChange = knot_cobs::SocialChange::Open {
creation: knot_cobs::Creation {
author: editor.clone(),
address: RecordAddress::new(
manifest_collection(),
RecordRkey::new(MANIFEST_RKEY).unwrap(),
),
number: SubjectNumber::new(1).unwrap(),
parent: None,
produces: record.cid(),
created_at,
},
body: knot_cobs::Manifest,
};
let cx = WriteContext::new(
&editor,
&repo_did,
ContributionPermission::new(Blocked::No, RepoRole::Owner, ContributionPolicy::Anyone),
);
let approved = Approved::signed_by(
&cx,
Materialized {
change,
record: Record::version(record),
},
)
.unwrap();
store
.create_approved::<knot_cobs::ManifestCob>(&home, &approved, &signer, created_at)
.unwrap();
let read = read_source_manifest(&repo_path)
.unwrap()
.expect("manifest present");
assert_eq!(read.owner, owner);
assert_eq!(read.name, name);
assert_eq!(read.rkey.as_str(), "my-repo");
assert_eq!(read.created_at, created_at);
}
}