// Package gitclient is a small git protocol v2 client over smart http, just
// enough to read refs, trees and blobs from a knot without cloning.
package gitclient
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"maps"
"net/http"
"path"
"slices"
"strings"
"sync"
)
var (
ErrRefNotFound = errors.New("branch or tag not found")
ErrPathNotFound = errors.New("path not found at the commit")
ErrNotDir = fmt.Errorf("%w: not a directory", ErrPathNotFound)
ErrTooLarge = errors.New("pack exceeds the read limit")
ErrUnsupported = errors.New("server lacks a required v2 capability")
)
// hex object id, sha1 or sha256 depending on the repo's object-format
type Oid string
type Client struct {
http *http.Client
url string // repo base, e.g. https://knot.example/did:plc:abc
maxBytes int64 // cap on one fetch's pack
mu sync.Mutex
caps *Caps
}
func New(httpClient *http.Client, repoURL string, maxBytes int64) *Client {
if httpClient == nil {
httpClient = http.DefaultClient
}
return &Client{http: httpClient, url: strings.TrimSuffix(repoURL, "/"), maxBytes: maxBytes}
}
type Caps struct {
ObjectFormat string // "sha1" or "sha256"
LsRefs bool
Fetch []string // fetch= features: shallow, filter, ...
}
func (c *Client) Caps(ctx context.Context) (Caps, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.caps != nil {
return *c.caps, nil
}
u := c.url + "/info/refs?service=git-upload-pack"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return Caps{}, err
}
req.Header.Set("Git-Protocol", "version=2")
resp, err := c.http.Do(req)
if err != nil {
return Caps{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return Caps{}, httpError(resp)
}
caps := Caps{ObjectFormat: "sha1"}
p := newPktReader(io.LimitReader(resp.Body, 1<<20))
v2 := false
for {
kind, line, err := p.next()
if err != nil && !v2 {
return Caps{}, fmt.Errorf("%w: %s doesn't speak protocol v2", ErrUnsupported, u)
}
if err != nil {
return Caps{}, fmt.Errorf("reading %s: %w", u, err)
}
if kind != pktData {
// a v0-style "# service=" banner ends in a flush before the v2 advert
if v2 {
break
}
continue
}
key, val, _ := strings.Cut(strings.TrimSuffix(string(line), "\n"), "=")
switch key {
case "version 2":
v2 = true
case "ls-refs":
caps.LsRefs = true
case "fetch":
caps.Fetch = strings.Fields(val)
case "object-format":
caps.ObjectFormat = val
}
}
if caps.ObjectFormat != "sha1" && caps.ObjectFormat != "sha256" {
return Caps{}, fmt.Errorf("%w: object-format=%s", ErrUnsupported, caps.ObjectFormat)
}
c.caps = &caps
return caps, nil
}
type Ref struct {
Name string
Oid Oid
Peeled Oid // set for annotated tags when Peel was asked
Symref string // target of HEAD when Symrefs was asked
}
type LsRefsOpts struct {
Prefixes []string // ref-prefix lines
Peel bool
Symrefs bool
}
// command=ls-refs
func (c *Client) LsRefs(ctx context.Context, opts LsRefsOpts) ([]Ref, error) {
caps, err := c.Caps(ctx)
if err != nil {
return nil, err
}
if !caps.LsRefs {
return nil, fmt.Errorf("%w: ls-refs", ErrUnsupported)
}
var args []string
if opts.Peel {
args = append(args, "peel")
}
if opts.Symrefs {
args = append(args, "symrefs")
}
for _, prefix := range opts.Prefixes {
args = append(args, "ref-prefix "+prefix)
}
resp, err := c.command(ctx, caps, "ls-refs", args)
if err != nil {
return nil, err
}
defer resp.Body.Close()
p := newPktReader(&capReader{r: resp.Body, n: c.maxBytes})
var refs []Ref
for {
kind, line, err := p.next()
if err != nil {
return nil, fmt.Errorf("ls-refs: %w", err)
}
if kind != pktData {
return refs, nil
}
text := strings.TrimSuffix(string(line), "\n")
if msg, ok := strings.CutPrefix(text, "ERR "); ok {
return nil, fmt.Errorf("remote: %s", msg)
}
fields := strings.Split(text, " ")
if len(fields) < 2 || !isOid(fields[0]) {
return nil, fmt.Errorf("ls-refs: bad line %q", text)
}
ref := Ref{Oid: Oid(fields[0]), Name: fields[1]}
for _, attr := range fields[2:] {
if v, ok := strings.CutPrefix(attr, "peeled:"); ok {
ref.Peeled = Oid(v)
} else if v, ok := strings.CutPrefix(attr, "symref-target:"); ok {
ref.Symref = v
}
}
refs = append(refs, ref)
}
}
type FetchOpts struct {
Wants []Oid
Deepen int // 0 means no deepen line
Filter string // e.g. "tree:0", "blob:none"; empty means none
}
// command=fetch with done and no haves, since there's no local object store
func (c *Client) Fetch(ctx context.Context, opts FetchOpts) (Objects, error) {
caps, err := c.Caps(ctx)
if err != nil {
return nil, err
}
if opts.Filter != "" && !slices.Contains(caps.Fetch, "filter") {
return nil, fmt.Errorf("%w: fetch=filter", ErrUnsupported)
}
if opts.Deepen > 0 && !slices.Contains(caps.Fetch, "shallow") {
return nil, fmt.Errorf("%w: fetch=shallow", ErrUnsupported)
}
if len(opts.Wants) == 0 {
return Objects{}, nil
}
args := []string{"no-progress", "ofs-delta"}
for _, want := range opts.Wants {
if len(want) != oidLen(caps.ObjectFormat) || !isOid(string(want)) {
return nil, fmt.Errorf("bad object id %q", want)
}
args = append(args, "want "+string(want))
}
if opts.Deepen > 0 {
args = append(args, fmt.Sprintf("deepen %d", opts.Deepen))
}
if opts.Filter != "" {
args = append(args, "filter "+opts.Filter)
}
args = append(args, "done")
resp, err := c.command(ctx, caps, "fetch", args)
if err != nil {
return nil, err
}
defer resp.Body.Close()
pack, err := packfileSection(newPktReader(resp.Body))
if err != nil {
return nil, fmt.Errorf("fetch: %w", err)
}
objs, err := readPack(&capReader{r: pack, n: c.maxBytes}, caps.ObjectFormat)
if err != nil {
return nil, fmt.Errorf("fetch: %w", err)
}
return objs, nil
}
// branches before tags, like git; tags peeled
func (c *Client) Resolve(ctx context.Context, ref string) (Oid, error) {
if ref == "" || strings.ContainsAny(ref, "\x00\n ") {
return "", ErrRefNotFound
}
candidates := []string{ref}
if !strings.HasPrefix(ref, "refs/") {
candidates = []string{"refs/heads/" + ref, "refs/tags/" + ref}
}
refs, err := c.LsRefs(ctx, LsRefsOpts{Prefixes: candidates, Peel: true})
if err != nil {
return "", err
}
for _, name := range candidates {
for _, r := range refs {
if r.Name != name {
continue
}
if r.Peeled != "" {
return r.Peeled, nil
}
return r.Oid, nil
}
}
return "", ErrRefNotFound
}
type File struct {
Name string
Mode string // octal, as in the tree
Oid Oid
Data []byte // nil for subtrees and submodules
}
// one level of dir, blobs filled in
func (c *Client) ReadDir(ctx context.Context, commit Oid, dir string) ([]File, error) {
caps, objs, oid, err := c.walk(ctx, commit, dir)
if err != nil {
return nil, err
}
entries, err := c.tree(ctx, caps, objs, oid, "tree:1")
if err != nil {
return nil, err
}
files := make([]File, 0, len(entries))
for _, e := range entries {
files = append(files, File{Name: e.Name, Mode: e.Mode, Oid: e.Oid})
}
if err := c.fillBlobs(ctx, objs, files); err != nil {
return nil, err
}
return files, nil
}
// everything under dir, named relative to it; the dir tree is wanted unfiltered
// so one pack brings it all
func (c *Client) ReadTree(ctx context.Context, commit Oid, dir string) ([]File, error) {
caps, objs, oid, err := c.walk(ctx, commit, dir)
if err != nil {
return nil, err
}
var files []File
queue := []File{{Oid: oid}}
for len(queue) > 0 {
t := queue[0]
queue = queue[1:]
entries, err := c.tree(ctx, caps, objs, t.Oid, "")
if err != nil {
return nil, err
}
for _, e := range entries {
f := File{Name: path.Join(t.Name, e.Name), Mode: e.Mode, Oid: e.Oid}
if isTreeMode(e.Mode) {
queue = append(queue, f)
}
files = append(files, f)
}
}
if err := c.fillBlobs(ctx, objs, files); err != nil {
return nil, err
}
return files, nil
}
// middle trees come alone (tree:0) so sibling blobs stay on the server; servers
// differ on filter depth for a wanted tree, hence fillBlobs
func (c *Client) walk(ctx context.Context, commit Oid, dir string) (Caps, Objects, Oid, error) {
caps, err := c.Caps(ctx)
if err != nil {
return Caps{}, nil, "", err
}
objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{commit}, Deepen: 1, Filter: "tree:1"})
if err != nil {
return Caps{}, nil, "", err
}
obj, ok := objs[commit]
if !ok || obj.Kind != KindCommit {
return Caps{}, nil, "", fmt.Errorf("fetch of commit %s didn't return it", commit)
}
tree, err := commitTree(obj.Data)
if err != nil {
return Caps{}, nil, "", err
}
for _, name := range strings.Split(dir, "/") {
if name == "" {
continue
}
entries, err := c.tree(ctx, caps, objs, tree, "tree:0")
if err != nil {
return Caps{}, nil, "", err
}
i := slices.IndexFunc(entries, func(e treeEntry) bool { return e.Name == name })
if i < 0 {
return Caps{}, nil, "", fmt.Errorf("%s: %w", dir, ErrPathNotFound)
}
if !isTreeMode(entries[i].Mode) {
return Caps{}, nil, "", fmt.Errorf("%s: %w", dir, ErrNotDir)
}
tree = entries[i].Oid
}
return caps, objs, tree, nil
}
func (c *Client) fillBlobs(ctx context.Context, objs Objects, files []File) error {
var missing []Oid
for _, f := range files {
if _, ok := objs[f.Oid]; isBlobMode(f.Mode) && !ok && !slices.Contains(missing, f.Oid) {
missing = append(missing, f.Oid)
}
}
if len(missing) > 0 {
// wanted objects pass any filter; it keeps the server on its selective path
more, err := c.Fetch(ctx, FetchOpts{Wants: missing, Filter: "tree:0"})
if err != nil {
return err
}
maps.Copy(objs, more)
}
for i, f := range files {
if !isBlobMode(f.Mode) {
continue
}
obj, ok := objs[f.Oid]
if !ok || obj.Kind != KindBlob {
return fmt.Errorf("fetch of blob %s didn't return it", f.Oid)
}
files[i].Data = obj.Data
}
return nil
}
func (c *Client) tree(ctx context.Context, caps Caps, have Objects, oid Oid, filter string) ([]treeEntry, error) {
if _, ok := have[oid]; !ok {
objs, err := c.Fetch(ctx, FetchOpts{Wants: []Oid{oid}, Filter: filter})
if err != nil {
return nil, err
}
maps.Copy(have, objs)
}
obj, ok := have[oid]
if !ok || obj.Kind != KindTree {
return nil, fmt.Errorf("fetch of tree %s didn't return it", oid)
}
return parseTree(obj.Data, caps.ObjectFormat)
}
func isBlobMode(mode string) bool {
mode = strings.TrimLeft(mode, "0")
return mode != "40000" && mode != "160000"
}
func isTreeMode(mode string) bool {
return strings.TrimLeft(mode, "0") == "40000"
}
func (c *Client) command(ctx context.Context, caps Caps, command string, args []string) (*http.Response, error) {
var body bytes.Buffer
writePkt(&body, "command="+command+"\n")
// git refuses non-sha1 unless the format is echoed back
if caps.ObjectFormat != "sha1" {
writePkt(&body, "object-format="+caps.ObjectFormat+"\n")
}
writeDelim(&body)
for _, arg := range args {
if err := writePkt(&body, arg+"\n"); err != nil {
return nil, err
}
}
writeFlush(&body)
u := c.url + "/git-upload-pack"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, &body)
if err != nil {
return nil, err
}
req.Header.Set("Git-Protocol", "version=2")
req.Header.Set("Content-Type", "application/x-git-upload-pack-request")
req.Header.Set("Accept", "application/x-git-upload-pack-result")
resp, err := c.http.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
defer resp.Body.Close()
return nil, httpError(resp)
}
return resp, nil
}
// knot2 says why in a plain text body, like a want it won't serve
func httpError(resp *http.Response) error {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
err := fmt.Errorf("%s %s: %s", resp.Request.Method, resp.Request.URL, resp.Status)
if msg := strings.TrimSpace(string(body)); msg != "" && !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
err = fmt.Errorf("%w: %s", err, strings.ToValidUTF8(msg, "?"))
}
return err
}