commit a9f1d0c689cdc96355acf76df0b5fe8599a9665a Author: Kosmos Date: Tue Sep 22 22:47:05 2026 +0000 Add spec, schema and README for pkgatlas Specification for a cross-distro file -> package resolver, grounded in verified upstream formats for deb (Contents), rpm (filelists.xml) and pacman (*.files.tar.gz). No implementation: spec only. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e524961 --- /dev/null +++ b/README.md @@ -0,0 +1,91 @@ +# pkgatlas + +Cross-distro **file → package** resolver. Answers one question well: + +> Given a build-time dependency signal — a linker flag (`-lcurses`), a missing header +> (`ncurses.h`), a pkg-config name, or a bare file path — **which package do I install** +> on *(distro, release, arch)*? + +Built for build automation. Primary consumer: **saggar** (auto-magic dependency detection), +where a `-lcurses not found` error currently has no reliable answer. + +## Why + +That mapping does exist upstream, but only as large per-distro indexes that are slow to +query, differently shaped per package manager, and unavailable mid-build: + +| Manager | Source | Size | +|---|---|---| +| deb | `Contents-.gz` | 12.7 – 47.6 MB | +| rpm | `filelists.xml.zst` | 39.7 – 48.9 MB (→ 875 MB uncompressed) | +| pacman | `.files.tar.gz` | 1.5 – 51 MB | + +pkgatlas precomputes them into one small, uniform, queryable artifact. + +The same mapping is also **distro-dependent**: `-lcurses` is `libncurses-dev` on Debian but +`termcap-devel` on openSUSE. So answers cannot be inferred across distros — they must be +built per distro. + +## Shape + +``` +pkgatlas-build ──> snapshot .sqlite ──┬──> pkgatlas-serve (HTTP API) + (ETL) (the artifact) └──> published file (offline query) +``` + +The **artifact is the product**. Consumers can pin a snapshot for reproducible builds, or +query locally with no network dependency mid-build. + +## Status + +**Pre-implementation.** Spec and schema are written and grounded in verified upstream +formats; the Go implementation is in progress. + +- [`docs/SPEC.md`](docs/SPEC.md) — full specification +- [`schema.sql`](schema.sql) — database schema + +## API (planned) + +``` +GET /v1/resolve?lib=curses&distro=ubuntu&release=24.04&arch=riscv64 +GET /v1/resolve?header=ncurses.h +GET /v1/resolve?pc=ncurses +GET /v1/resolve?file=libcurses.so +``` + +```json +{ + "query": { "kind": "lib", "value": "curses" }, + "resolved": [{ + "manager": "deb", "distro": "ubuntu", "release": "24.04", "arch": "riscv64", + "path": "usr/lib/riscv64-linux-gnu/libcurses.so", + "package": "libncurses-dev", + "install": "apt-get install -y libncurses-dev" + }], + "ambiguous": false, + "truncated": false +} +``` + +`resolved` is always an array — an empty one means "genuinely unavailable on this distro", +which is a real answer. When a flag maps to several packages (`-lcurl` → +`libcurl4-openssl-dev` *and* `libcurl4-gnutls-dev`), `ambiguous` is `true` and the consumer +chooses; pkgatlas never picks silently. + +## Scope + +v1 covers **deb**, **rpm** and **pacman**. `apk` and `nix` are deferred. Language-registry +metadata (npm/cargo/pypi) is explicitly out of scope — that is [deps.dev](https://deps.dev)'s job. + +## Build + +```sh +CGO_ENABLED=0 go build ./... +go test ./... +``` + +Requires Go 1.25+. No CGO: SQLite is `modernc.org/sqlite`, zstd is `klauspost/compress`. + +## License + +TBD. diff --git a/docs/SPEC.md b/docs/SPEC.md new file mode 100644 index 0000000..5ce1e7f --- /dev/null +++ b/docs/SPEC.md @@ -0,0 +1,321 @@ +# pkgatlas — SPEC v1 + +**One question, answered well:** given a build-time dependency signal, which package do I +install on *(distro, release, arch)*? + +Signals: a linker flag (`-lcurses`), a missing header (`ncurses.h`), a pkg-config name +(`ncurses`), or a bare file path. + +**Primary consumer: saggar** (auto-magic dependency detection). Secondary: any build agent. + +--- + +## 1. Product shape + +Build once, serve forever. + +``` +pkgatlas-build ──> snapshot .sqlite ──┬──> pkgatlas-serve (HTTP) + (ETL) (the artifact) └──> published file (local query) +``` + +The **artifact is the product**. Each build emits a versioned SQLite database per +(manager, release, arch). It is served over HTTP *and* published as a downloadable file. + +Why this matters: a consumer can pin an exact snapshot for reproducible builds, and can +query locally with zero network dependency mid-build. A resolver that cannot break your +build is worth more than a fast one. + +**Non-goals (v1)** + +- No web UI. Human browsing can come later as a separate app over the same API. +- No `apk` / `nix` (deferred — see §7). +- No language-registry data (npm/cargo/pypi). That is deps.dev's job. +- **No fuzzy/heuristic matching.** All three v1 sources are exact file→package lookups. + If a query cannot be answered exactly, return empty; never guess. + +--- + +## 2. Verified upstream sources + +Every fact below was fetched and inspected. Sizes are real observed responses. + +| Manager | Source | Observed size | +|---|---|---| +| deb | `Contents-.gz` | 12.7 MB (trixie amd64), 47.6 MB (noble riscv64), 18.4 MB (sid amd64) | +| rpm | `filelists.xml.zst` | 39.7 MB (openSUSE TW), 48.9 MB (Fedora 43) | +| pacman | `.files.tar.gz` | 1.5 MB (core), 51.0 MB (extra), 0.1 MB (multilib) | + +### 2.1 deb — Debian / Ubuntu + +``` +Debian https://deb.debian.org/debian/dists/{release}/{component}/Contents-{arch}.gz +Ubuntu amd64 https://archive.ubuntu.com/ubuntu/dists/{release}/Contents-{arch}.gz +Ubuntu other https://ports.ubuntu.com/ubuntu-ports/dists/{release}/Contents-{arch}.gz +``` + +> **Trap:** Ubuntu non-amd64 is **not** on `archive.ubuntu.com`. +> `dists/noble/Contents-riscv64.gz` returns 404 there; the same path on +> `ports.ubuntu.com/ubuntu-ports/` returns 200. Ubuntu supports +> `amd64 arm64 armhf i386 ppc64el riscv64 s390x` — so this affects most arches. + +Format: gzip-compressed text, one line per path: + +``` +/[,/...] +``` + +``` +usr/include/ncurses.h libdevel/libncurses-dev +usr/lib/x86_64-linux-gnu/libcurses.so libdevel/libncurses-dev +usr/lib/x86_64-linux-gnu/libcurl.so libdevel/libcurl4-gnutls-dev,libdevel/libcurl4-openssl-dev +``` + +- Paths have **no leading slash**. +- `prefix` is a repo/component label (`main`, `universe`, `libdevel`). It does **not** + indicate binary-vs-source. +- The right-hand side is **binary package names** — directly installable via `apt-get`. + +### 2.2 rpm — openSUSE / Fedora + +``` +openSUSE https://download.opensuse.org/tumbleweed/repo/oss/repodata/repomd.xml +Fedora https://dl.fedoraproject.org/pub/fedora/linux/releases/{rel}/Everything/{arch}/os/repodata/repomd.xml +``` + +`repomd.xml` lists content sections. Read `` → its `` +→ that is `filelists.xml.zst` (zstd-compressed). + +Verified structure: + +```xml + + + + + /usr/bin/0ad + /usr/lib64/0ad + /usr/lib64/0ad/libAtlasUI.so + +``` + +- **The package name is inline** (`name="0ad"`) — no join against `primary.xml` needed. +- `type="dir"` marks directories; skip them. +- Paths **do** have a leading slash (unlike deb and pacman). +- Names here are binary package names (`ncurses-devel`), verified against the real file. + +> **Note:** Fedora also publishes `filelists_db` (a prebuilt SQLite, 64.6 MB → 420 MB) and +> `filelists_zck` (zchunk). The `_db` is a genuine SQLite but awkward: `packages(pkgKey, +> pkgId)` where **pkgId is a SHA256 checksum, not a name** (needs a `primary_db` join), and +> `filelist(pkgKey, dirname, filenames, filetypes)` stores filenames **slash-separated in a +> single field**. v1 parses the XML instead. See §6 for zchunk. + +### 2.3 pacman — Arch + +``` +https://geo.mirror.pkgbuild.com/{repo}/os/{arch}/{repo}.files.tar.gz +``` + +Repos: `core`, `extra`, `multilib` (plus `*-testing`). Format: tar.gz, one directory per +package named `--/`, containing `files` and `desc`. + +The `files` member: + +``` +%FILES% +usr/ +usr/bin/ +usr/bin/clear +usr/include/ncurses.h +usr/lib/libcurses.so +usr/lib/libncurses.so.6 +``` + +- Paths have **no leading slash**; a trailing `/` means directory. +- Verified to contain the unversioned devel symlink `usr/lib/libncurses.so`, which is + exactly what `-lcurses` needs. + +> **Trap:** Arch's *web API* `?file=` parameter is a **no-op** — it returns unfiltered +> results (`0ad` first for any query). Do not use it. The data above is the correct source. +> (`?q=` does filter, but answers a different question.) + +--- + +## 3. Normalization + +Applied to every source so downstream storage is uniform: + +1. Strip any leading `/`. +2. Drop entries whose last character is `/` (directories). +3. `basename = path.Base(path)` — the hot lookup key. +4. Classify `kind`: + + | kind | rule | + |---|---| + | `lib` | `*.so` or `*.so.N`, `*.so.N.N.N` | + | `static` | `*.a` | + | `header` | `*.h`, `*.hpp`, `*.hh` | + | `pc` | `*.pc` | + | `cmake` | `*.cmake` | + | `m4` | `*.m4` | + | `bin` | path begins `usr/bin/` or `bin/` | + | `other` | everything else | + +5. **Keep only dev-relevant kinds** (`lib`, `static`, `header`, `pc`, `cmake`, `m4`, `bin`). + + Debian trixie `main` has 1,853,320 total entries; the dev-relevant subset is ~313,749, + and just `.so`+`.pc` is ~68,509. The gzipped dev subset is ~2.3 MB. Do not store the + full index — it is 6x the size for no benefit. + +--- + +## 4. Identifier model + +A query is exactly one of: + +| kind | input | lookup | +|---|---|---| +| `lib` | `curses` (from `-lcurses`) | file with basename `libcurses.so` | +| `header` | `ncurses.h` | file with basename `ncurses.h` | +| `pc` | `ncurses` | file with basename `ncurses.pc` | +| `file` | a path or basename | exact match on `path` or `basename` | + +The linker error names the exact flag (`-lpcre2-8`, never `-lpcre2`), so `lib` queries are +reliable as given: strip the leading `-l`, prefix `lib`, suffix `.so`. + +### Ambiguity must be surfaced, never resolved silently + +The same flag legitimately maps to different packages: + +| query | distro | result | +|---|---|---| +| `-lcurses` | debian trixie | `libncurses-dev` | +| `-lncurses` | openSUSE | `ncurses-devel` (`/usr/lib64/libncurses.so`) | +| `-lcurses` | openSUSE | `termcap-devel` (`/usr/lib64/curses/libcurses.so`) | +| `-lcurl` | debian trixie | `libcurl4-gnutls-dev`, `libcurl4-openssl-dev` | + +Consequences: + +- Responses always contain an **array**, plus `ambiguous: bool`. +- The same flag giving **different answers on different distros** is precisely why + per-distro ingestion is mandatory. Derivatives cannot be assumed to inherit their + parent's answers for file→package resolution. + +--- + +## 5. Schema and API + +See [`schema.sql`](./schema.sql) for the full DDL. + +### API + +``` +GET /v1/resolve?lib=curses[&distro=&release=&arch=] +GET /v1/resolve?header=ncurses.h[&...] +GET /v1/resolve?pc=ncurses[&...] +GET /v1/resolve?file=libcurses.so[&...] +GET /v1/snapshots +GET /v1/healthz +``` + +Exactly one of `lib|header|pc|file` is required. Filters are optional and repeatable; +**omitting them returns all known matches** (that is the point of a precomputed catalog). + +```json +{ + "query": {"kind": "lib", "value": "curses"}, + "resolved": [ + { + "manager": "deb", + "distro": "ubuntu", + "release": "24.04", + "arch": "riscv64", + "path": "usr/lib/riscv64-linux-gnu/libcurses.so", + "package": "libncurses-dev", + "install": "apt-get install -y libncurses-dev" + } + ], + "ambiguous": false, + "truncated": false +} +``` + +Contract: + +- `resolved` is always an array. Empty means **genuinely unavailable on that distro** — + a real answer the consumer can act on, not an error. +- `install` is always populated (per-manager command). +- `ambiguous: true` signals the consumer must choose; it must not pick silently. +- `truncated: true` when a result limit clipped the response. + +### CLI + +``` +pkgatlas-build --source deb --release trixie --arch amd64 --out out/trixie-amd64.sqlite +pkgatlas-build --source rpm --distro opensuse --release tumbleweed --arch x86_64 --out ... +pkgatlas-build --source pacman --repo core --repo extra --arch x86_64 --out ... +pkgatlas-serve --db out/ --addr :8080 +``` + +--- + +## 6. Refresh strategy + +- **Conditional GET** on every upstream fetch (`If-Modified-Since` / `ETag`). Debian's + `Release`/`InRelease` files give a cheap timestamp to compare before pulling a 47 MB + `Contents` file. Verified reachable: `dists/trixie/Release` (138,612 B). +- **zchunk** (RPM): Fedora publishes `*_zck` variants supporting ranged/delta downloads. + If usable, a weekly refresh pulls only changed chunks instead of ~40-90 MB per repo. + **Evaluate after v1 works**; do not block on it. +- **Mirror etiquette:** descriptive User-Agent, prefer mirrors over primary hosts, back off + on 429/503. This runs weekly forever — do not get rate-limited off Debian's CDN. + +--- + +## 7. Deferred + +- **apk (Alpine):** `https://dl-cdn.alpinelinux.org/alpine/v{ver}/{repo}/{arch}/APKINDEX.tar.gz` + (473 KB). Reachable and cheap. **Not yet verified to carry file-level paths** — confirm + before implementing. +- **nix:** needs the nix-index backend; `search.nixos.org` serves only a JS shell (1.7 KB). + Unverified. +- **zchunk delta refresh** (§6). + +--- + +## 8. Implementation constraints + +- **Go 1.25.x.** Module path set to the final repo URL. +- **stdlib `net/http` only** — no web framework. +- **`modernc.org/sqlite`** (pure Go; `rusqlite`-style CGO deps are not acceptable). +- **`github.com/klauspost/compress/zstd`** (pure Go) for `.zst` inputs. +- **`CGO_ENABLED=0 go build ./...` must succeed.** +- Unit tests must not touch the network; parse from small checked-in fixtures + (`testdata/`). Real-source tests belong behind an opt-in build tag. +- Streaming, not buffering: RPM `filelists` is 875 MB uncompressed. Use + `xml.Decoder` token streaming and a batched SQLite writer. Never `io.ReadAll` a source. + +### Interfaces + +```go +type Snapshot struct { + Manager, Distro, Release, Arch, Repo string + UpstreamDate, FetchedAt, SourceURL string +} + +type FileEntry struct { + Path, Basename, Kind, Repo, Package, Version string +} + +type Source interface { + Name() string + // Versions lists ingestable (release, arch, repo) combinations. + Versions(ctx context.Context) ([]Snapshot, error) + // Fetch streams one snapshot's index. Caller closes. + Fetch(ctx context.Context, s Snapshot) (io.ReadCloser, error) + // Parse streams entries to emit. Must not buffer the whole document. + Parse(ctx context.Context, r io.Reader, s Snapshot, emit func(FileEntry) error) error +} +``` + +Adding a manager = one file implementing `Source` + a fixture + a case in the builder. diff --git a/schema.sql b/schema.sql new file mode 100644 index 0000000..2eb972d --- /dev/null +++ b/schema.sql @@ -0,0 +1,86 @@ +-- pkgatlas schema v1 +-- +-- One SQLite database per (manager, release, arch) snapshot, or a combined catalog +-- containing many snapshots. The same schema serves both. +-- +-- Design notes: +-- * `file.basename` is the hot lookup key (a -lX query becomes basename `libX.so`). +-- * `path` is normalized: no leading slash, directories excluded (see SPEC §3). +-- * Dev-relevant kinds only are stored; the full upstream index is ~6x larger for +-- no benefit (SPEC §3.5). + +PRAGMA journal_mode = WAL; +PRAGMA foreign_keys = ON; + +-- Key/value build metadata: schema_version, generator, built_at, ... +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +-- One row per ingested upstream index. +CREATE TABLE IF NOT EXISTS snapshot ( + id INTEGER PRIMARY KEY, + manager TEXT NOT NULL, -- deb | rpm | pacman + distro TEXT NOT NULL, -- debian | ubuntu | opensuse | fedora | arch + release TEXT NOT NULL, -- trixie | noble | tumbleweed | 43 + arch TEXT NOT NULL, -- amd64 | riscv64 | x86_64 | arm64 | noarch + repo TEXT, -- core | extra | oss | main | universe | NULL + upstream_date TEXT, -- from Release / repomd, when available + fetched_at TEXT NOT NULL, -- RFC3339 + source_url TEXT NOT NULL, + UNIQUE (manager, distro, release, arch, repo) +); + +CREATE TABLE IF NOT EXISTS package ( + id INTEGER PRIMARY KEY, + snapshot_id INTEGER NOT NULL REFERENCES snapshot(id) ON DELETE CASCADE, + name TEXT NOT NULL, -- binary package name (installable) + version TEXT, + src_package TEXT, + repo TEXT +); + +CREATE INDEX IF NOT EXISTS idx_package_name ON package(name); +CREATE INDEX IF NOT EXISTS idx_package_snapshot ON package(snapshot_id); + +-- A single file provided by a package. +CREATE TABLE IF NOT EXISTS file ( + id INTEGER PRIMARY KEY, + snapshot_id INTEGER NOT NULL REFERENCES snapshot(id) ON DELETE CASCADE, + package_id INTEGER NOT NULL REFERENCES package(id) ON DELETE CASCADE, + path TEXT NOT NULL, -- normalized: 'usr/lib/x86_64-linux-gnu/libcurses.so' + basename TEXT NOT NULL, -- 'libcurses.so' + kind TEXT NOT NULL -- lib|static|header|pc|cmake|m4|bin +); + +CREATE INDEX IF NOT EXISTS idx_file_basename ON file(basename); +CREATE INDEX IF NOT EXISTS idx_file_path ON file(path); +CREATE INDEX IF NOT EXISTS idx_file_package ON file(package_id); +CREATE INDEX IF NOT EXISTS idx_file_snapshot ON file(snapshot_id); + +-- Flattened view matching the /v1/resolve response shape. +CREATE VIEW IF NOT EXISTS v_resolve AS +SELECT + f.basename AS basename, + f.path AS path, + f.kind AS kind, + p.name AS package, + p.version AS pkg_version, + s.manager AS manager, + s.distro AS distro, + s.release AS release, + s.arch AS arch, + s.repo AS repo +FROM file f +JOIN package p ON p.id = f.package_id +JOIN snapshot s ON s.id = f.snapshot_id; + +-- Core lookup: resolve a library flag to installable packages. +-- +-- SELECT DISTINCT package, distro, release, arch, path +-- FROM v_resolve WHERE basename = 'libcurses.so'; +-- +-- Ambiguity is expected (SPEC §4): '-lcurl' legitimately returns two packages, and the +-- same flag can resolve differently per distro. Callers must treat >1 distinct package +-- as ambiguous rather than picking one.