From dd1c70c91a743ef1e1c016cb3bac48a4de8955ff Mon Sep 17 00:00:00 2001 From: Valentin Haudiquet Date: Fri, 18 Sep 2026 13:52:24 +0200 Subject: [PATCH] new: drive the license menu, parsing and sniffing from data/licenses.yml --- data/licenses.yml | 139 +++++++++++++++++++++++++++ src/new/detect.rs | 78 +++++---------- src/new/licenses.rs | 224 +++++++++++++++++++++++++++++++++++++++++++ src/new/mod.rs | 2 + src/new/options.rs | 105 ++++++++++++++++---- src/new/questions.rs | 48 ++++------ 6 files changed, 496 insertions(+), 100 deletions(-) create mode 100644 data/licenses.yml create mode 100644 src/new/licenses.rs diff --git a/data/licenses.yml b/data/licenses.yml new file mode 100644 index 0000000..e0faf06 --- /dev/null +++ b/data/licenses.yml @@ -0,0 +1,139 @@ +## License knowledge of `pkh new`, in one place: the wizard menu labels +## (src/new/questions.rs), the spellings accepted by License::parse and the +## SPDX URL template (src/new/options.rs), and the license-file sniffing +## inputs (src/new/detect.rs) all read this table, so the three lists — +## previously kept in sync by comments only — cannot drift apart anymore. +## Adding or changing a curated license is one entry below. +## +## Keys: +## id: SPDX identifier: written to debian/copyright, returned +## by the license sniff and substituted into +## license_url_template (minus a trailing '+' of the +## "or later" spellings) +## menu: label offered by the wizard license question (the +## free-text "Other (enter a SPDX identifier)" entry +## stays in Rust — it is UX, not data) +## spellings: inputs accepted by License::parse, matched +## case-insensitively; each entry must accept its own id +## (a consistency test in options.rs locks ids, spellings +## and the License enum together) +## detect_markers: marker sets driving the LICENSE/COPYING text sniff of +## detect.rs. A set matches when every marker of `all` +## occurs in the lowercased license text and none of +## `unless` does; an entry matches when any of its sets +## does. The sets are written to be mutually exclusive: +## the `unless` markers keep multi-license texts on the +## entry carrying the stronger reference (a MIT-named +## file also quoting the GPL or the Apache license is a +## GPL/Apache file) and keep GPL sets off LGPL texts, +## whose name contains theirs. The entry order below +## (menu order) therefore only breaks ties. +## +## detect_files (top level): the candidate license file names the sniff +## reads, in preference order, shared by every license (the case-variant +## directory scan around them stays in Rust). +## +## license_url_template: the SPDX license page URL, with {id} substituted +## for the debian/copyright reference paragraph. +## +## The behavioral lock for the markers is the LICENSE_TEXTS test table in +## src/new/detect.rs: a bad marker edit fails those tests, not packages. + +license_url_template: https://spdx.org/licenses/{id}.html + +detect_files: + - LICENSE + - LICENSE.md + - LICENSE.txt + - COPYING + - COPYING.txt + +# Entries in wizard-menu order. +licenses: + - id: MIT + menu: MIT + spellings: [MIT] + detect_markers: + - all: + - mit license + unless: + - apache license + - general public license + - all: + - permission is hereby granted, free of charge + unless: + - apache license + - general public license + - id: Apache-2.0 + menu: Apache-2.0 + spellings: [Apache-2.0] + detect_markers: + - all: + - apache license + - version 2 + - id: GPL-2.0+ + menu: GPL-2.0+ + spellings: [GPL-2.0+] + detect_markers: + - all: + - general public license + unless: + - version 3 + - lesser general public license + - id: GPL-3.0+ + menu: GPL-3.0+ + spellings: [GPL-3.0+] + detect_markers: + - all: + - general public license + - version 3 + unless: + - lesser general public license + - id: LGPL-2.1+ + menu: LGPL-2.1+ + spellings: [LGPL-2.1+] + detect_markers: + - all: + - lesser general public license + unless: + - version 3 + - all: + - lesser general public license + - version 2.1 + - id: LGPL-3.0+ + menu: LGPL-3.0+ + spellings: [LGPL-3.0+] + detect_markers: + - all: + - lesser general public license + - version 3 + unless: + - version 2.1 + - id: BSD-2-Clause + menu: BSD-2-Clause + spellings: [BSD-2-Clause] + detect_markers: + - all: + - redistribution and use in source and binary forms + unless: + - endorse or promote + - isc license + - permission to use, copy, modify, and/or distribute this software + - id: BSD-3-Clause + menu: BSD-3-Clause + spellings: [BSD-3-Clause] + detect_markers: + - all: + - redistribution and use in source and binary forms + - endorse or promote + unless: + - isc license + - permission to use, copy, modify, and/or distribute this software + - id: ISC + menu: ISC + spellings: [ISC] + detect_markers: + - all: + - isc license + - all: + - permission to use, copy, modify, and/or distribute this software diff --git a/src/new/detect.rs b/src/new/detect.rs index d57e81c..a2e2431 100644 --- a/src/new/detect.rs +++ b/src/new/detect.rs @@ -23,6 +23,7 @@ use std::path::Path; use regex::Regex; +use super::licenses; use super::options::TemplateId; /// Outcome of the detection. @@ -109,23 +110,14 @@ fn has_shebang(path: &Path) -> bool { content.starts_with(b"#!") } -/// License files looked at by [`sniff_license`], in preference order. -const LICENSE_FILES: [&str; 5] = [ - "LICENSE", - "LICENSE.md", - "LICENSE.txt", - "COPYING", - "COPYING.txt", -]; - /// Sniff the license of the project in `dir` from its `LICENSE`/`COPYING` /// file: an `SPDX-License-Identifier:` line wins, otherwise the text is -/// matched against a short list of recognizable licenses (MIT, BSD-2/3, -/// Apache-2.0, GPL-2/3, LGPL-2.1/3, ISC). `None` when no license file -/// exists or nothing recognizable is found. +/// matched against the marker sets of the bundled license table +/// (`data/licenses.yml`: MIT, BSD-2/3, Apache-2.0, GPL-2/3, LGPL-2.1/3, +/// ISC). `None` when no license file exists or nothing recognizable is +/// found. pub fn sniff_license(dir: &Path) -> Option { - let content = LICENSE_FILES - .iter() + let content = licenses::detect_files() .find_map(|name| std::fs::read_to_string(dir.join(name)).ok()) // Case variants and suffixes (LICENSE-MIT, LICENCE, cpYING…): the // first top-level file whose name looks like a license notice. @@ -166,43 +158,10 @@ pub fn sniff_license(dir: &Path) -> Option { return Some(id); } + // The recognizable-license markers live in the bundled table; the + // LICENSE_TEXTS test below is their behavioral lock. let text = content.to_ascii_lowercase(); - if text.contains("apache license") && text.contains("version 2") { - return Some("Apache-2.0".to_string()); - } - if text.contains("lesser general public license") { - return if text.contains("version 3") && !text.contains("version 2.1") { - Some("LGPL-3.0+".to_string()) - } else { - Some("LGPL-2.1+".to_string()) - }; - } - if text.contains("general public license") { - return if text.contains("version 3") { - Some("GPL-3.0+".to_string()) - } else { - Some("GPL-2.0+".to_string()) - }; - } - if text.contains("mit license") || text.contains("permission is hereby granted, free of charge") - { - return Some("MIT".to_string()); - } - if text.contains("isc license") - || text.contains("permission to use, copy, modify, and/or distribute this software") - { - return Some("ISC".to_string()); - } - if text.contains("redistribution and use in source and binary forms") { - // The third clause (name endorsement) is what sets BSD-3 apart - // from BSD-2. - return if text.contains("endorse or promote") { - Some("BSD-3-Clause".to_string()) - } else { - Some("BSD-2-Clause".to_string()) - }; - } - None + licenses::detect_from_text(&text).map(str::to_string) } #[cfg(test)] @@ -302,8 +261,10 @@ mod tests { assert_eq!(detect(dir.path()), Detection::Empty); } - /// Distinctive (shortened) excerpts of the recognizable license texts. - const LICENSE_TEXTS: [(&str, &str); 9] = [ + /// Distinctive (shortened) excerpts of the recognizable license texts: + /// the behavioral lock of the marker sets in `data/licenses.yml` — a + /// bad marker edit fails here, not on real packages. + const LICENSE_TEXTS: [(&str, &str); 11] = [ ( "MIT", "MIT License\n\nPermission is hereby granted, free of charge, to any person", @@ -337,6 +298,19 @@ mod tests { "ISC", "ISC License\nPermission to use, copy, modify, and/or distribute this software", ), + // Dual-licensed preamble: the GPL reference outranks the MIT + // boilerplate, like the old hardcoded cascade decided. + ( + "GPL-2.0+", + "MIT License\n\nAlternatively, under the terms of the GNU General Public License,\ + \nversion 2 of the License.", + ), + // LGPL text naming both versions: the 2.1 wording wins. + ( + "LGPL-2.1+", + "GNU LESSER GENERAL PUBLIC LICENSE\nVersion 2.1, February 1999\n\ + This is version 2.1; version 3 is available separately.", + ), ]; #[test] diff --git a/src/new/licenses.rs b/src/new/licenses.rs new file mode 100644 index 0000000..2217d7d --- /dev/null +++ b/src/new/licenses.rs @@ -0,0 +1,224 @@ +//! License reference data for `pkh new`, bundled as `data/licenses.yml`. +//! +//! The license knowledge of the scaffolder used to live in three places — +//! the wizard menu (`questions.rs`), the parse/SPDX mapping (`options.rs`) +//! and the license-file sniffing inputs (`detect.rs`) — kept in sync by +//! comments only. It is one table now: [`entries`] carries each curated +//! license's menu label, accepted spellings and detection markers, +//! [`detect_files`] the candidate license file names and [`url_template`] +//! the SPDX page URL template, so the lists cannot drift apart and adding +//! a license is a YAML entry. +//! +//! The `License` enum stays in `options.rs` (`NewOptions` and the template +//! rendering match on its variants); the enum's variants and the table's +//! ids are locked together by a consistency test there. The free-text +//! "Other (enter a SPDX identifier)" wizard entry is UX, not data, and +//! stays in Rust — like `License::Custom`'s code-driven parse path. + +use serde::Deserialize; + +use crate::data::embed_data; + +/// One marker set of the license-text sniff: the set matches when every +/// marker of `all` occurs in the lowercased license text and none of +/// `unless` does (all-markers-within-a-list, any-marker-list semantics). +#[derive(Debug, Deserialize)] +pub(crate) struct DetectMarkerSet { + /// Substrings that must all occur in the license text + pub(crate) all: Vec, + /// Substrings that must all be absent for the set to match + #[serde(default)] + pub(crate) unless: Vec, +} + +/// One curated license of the bundled table: everything `pkh new` knows +/// about it — identifier, menu label, accepted spellings and sniff markers +#[derive(Debug, Deserialize)] +pub(crate) struct LicenseEntry { + /// SPDX identifier: written to `debian/copyright`, returned by the + /// license sniff, substituted into [`url_template`] + pub(crate) id: String, + /// Label offered by the wizard license menu + pub(crate) menu: String, + /// Inputs accepted by `License::parse` (case-insensitive); must + /// include the id itself + pub(crate) spellings: Vec, + /// Marker sets of the license-text sniff (see [`detect_from_text`]) + pub(crate) detect_markers: Vec, +} + +/// The bundled license table (`data/licenses.yml`): the curated licenses +/// plus the shared sniffing inputs and the SPDX URL template +#[derive(Debug, Deserialize)] +struct LicensesData { + /// SPDX license page URL template (`{id}` placeholder) + license_url_template: String, + /// Candidate license file names of the sniff, in preference order + detect_files: Vec, + /// The curated licenses, in wizard-menu order + licenses: Vec, +} + +embed_data! { + static ref LICENSES_DATA: LicensesData = "../../data/licenses.yml" +} + +/// The curated license entries, in the order offered by the wizard menu +pub(crate) fn entries() -> &'static [LicenseEntry] { + &LICENSES_DATA.licenses +} + +/// The entry whose `spellings` contain `input` (case-insensitive): the +/// table lookup behind `License::parse`'s curated arm +pub(crate) fn entry_for_spelling(input: &str) -> Option<&'static LicenseEntry> { + entries().iter().find(|entry| { + entry + .spellings + .iter() + .any(|s| s.eq_ignore_ascii_case(input)) + }) +} + +/// The candidate license file names looked at by the license sniff, in +/// preference order, shared by every license +pub(crate) fn detect_files() -> impl Iterator { + LICENSES_DATA.detect_files.iter().map(String::as_str) +} + +/// The SPDX identifier of the (lowercased) license `text`, `None` when +/// nothing is recognizable: the first entry (menu order) with a matching +/// marker set wins. The caller lowercases the text; the markers are +/// lowercase substrings. +pub(crate) fn detect_from_text(text: &str) -> Option<&'static str> { + entries() + .iter() + .find(|entry| markers_match(entry, text)) + .map(|entry| entry.id.as_str()) +} + +/// Whether any marker set of `entry` matches `text` (see +/// [`DetectMarkerSet`] for the semantics) +fn markers_match(entry: &LicenseEntry, text: &str) -> bool { + entry.detect_markers.iter().any(|set| { + set.all.iter().all(|marker| text.contains(marker)) + && set.unless.iter().all(|marker| !text.contains(marker)) + }) +} + +/// The SPDX license page URL template of the table, carrying the license +/// identifier as an `{id}` placeholder +pub(crate) fn url_template() -> &'static str { + LICENSES_DATA.license_url_template.as_str() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every entry is menu-ready: non-empty id/menu, at least one spelling + /// and one non-empty marker set, and no duplicate ids or menus (the + /// wizard select would silently shadow a duplicate menu label). + #[test] + fn entries_are_well_formed() { + assert!(!entries().is_empty()); + let mut ids: Vec<&str> = Vec::new(); + let mut menus: Vec<&str> = Vec::new(); + for entry in entries() { + assert!(!entry.id.is_empty()); + assert!(!entry.menu.is_empty()); + assert!(!entry.spellings.is_empty()); + assert!( + !entry.detect_markers.is_empty(), + "entry '{}' has no marker set", + entry.id + ); + for set in &entry.detect_markers { + assert!( + !set.all.is_empty(), + "entry '{}' has an empty marker set", + entry.id + ); + } + ids.push(entry.id.as_str()); + menus.push(entry.menu.as_str()); + } + ids.sort_unstable(); + ids.dedup(); + menus.sort_unstable(); + menus.dedup(); + assert_eq!(ids.len(), entries().len(), "duplicate ids"); + assert_eq!(menus.len(), entries().len(), "duplicate menus"); + } + + /// The shared sniffing inputs and the URL template: the canonical file + /// names in preference order, and a template the spdx_url + /// substitution can render. + #[test] + fn sniff_inputs_and_url_template() { + let files: Vec<&str> = detect_files().collect(); + assert!(!files.is_empty()); + assert_eq!(files[0], "LICENSE"); + assert!(files.contains(&"COPYING")); + assert!(url_template().starts_with("https://spdx.org/licenses/")); + assert!(url_template().contains("{id}")); + } + + /// The marker semantics on hand-built texts: every `all` marker must + /// occur, every `unless` marker must not, any set of an entry + /// suffices, and the sets are mutually exclusive across entries — + /// the LGPL/GPL substring relation and the multi-license texts land + /// on the same entry the old hardcoded cascade picked. + #[test] + fn marker_sets_keep_the_old_cascade_results() { + // Nothing recognizable. + assert_eq!(detect_from_text("do whatever you want"), None); + // Version discrimination of the GPL family. + assert_eq!( + detect_from_text("gnu general public license\nversion 3"), + Some("GPL-3.0+") + ); + assert_eq!( + detect_from_text("gnu general public license\nversion 2, june 1991"), + Some("GPL-2.0+") + ); + // "lesser general public license" contains "general public + // license": LGPL texts must stay on their entries. + assert_eq!( + detect_from_text("gnu lesser general public license\nversion 3"), + Some("LGPL-3.0+") + ); + assert_eq!( + detect_from_text("gnu lesser general public license\nversion 2.1"), + Some("LGPL-2.1+") + ); + // A text naming both versions is the 2.1 wording (version 2.1 + // wins), like the old unless-less branch pair did. + assert_eq!( + detect_from_text("gnu lesser general public license\nversion 3, like version 2.1"), + Some("LGPL-2.1+") + ); + // Multi-license texts: the GPL/Apache reference is the stronger + // one, so MIT does not steal them. + assert_eq!( + detect_from_text("mit license\nunder the gnu general public license, version 2"), + Some("GPL-2.0+") + ); + assert_eq!( + detect_from_text( + "permission is hereby granted, free of charge\ndual-licensed under the apache license version 2" + ), + Some("Apache-2.0") + ); + // BSD-2 vs BSD-3: the endorsement clause is the discriminator. + assert_eq!( + detect_from_text("redistribution and use in source and binary forms"), + Some("BSD-2-Clause") + ); + assert_eq!( + detect_from_text( + "redistribution and use in source and binary forms\nmay be used to endorse or promote" + ), + Some("BSD-3-Clause") + ); + } +} diff --git a/src/new/mod.rs b/src/new/mod.rs index b14c0ca..a0c343b 100644 --- a/src/new/mod.rs +++ b/src/new/mod.rs @@ -16,6 +16,8 @@ pub mod debian; pub mod detect; pub mod git; +/// License reference data for the scaffolder (bundled `data/licenses.yml`) +pub(crate) mod licenses; pub mod options; pub mod orig; pub mod origin; diff --git a/src/new/options.rs b/src/new/options.rs index 0c9504a..a83812f 100644 --- a/src/new/options.rs +++ b/src/new/options.rs @@ -15,6 +15,7 @@ use crate::debian::DebianVersion; use crate::debian::deps::{Deps, ParseOpts}; use crate::distro_info; use crate::new::detect::{self, Detection}; +use crate::new::licenses; use crate::new::origin::{Forge, GitOrigin}; use crate::new::templates; @@ -213,6 +214,10 @@ impl OrigOrigin { /// Upstream license of the package: a curated SPDX list plus a free-text /// fallback for anything else (including "unknown" until the user picks one). +/// The curated identifiers, their accepted spellings and the menu labels +/// come from the bundled license table (`data/licenses.yml`); this enum +/// stays code because [`NewOptions`] and the template rendering match on +/// its variants — a consistency test keeps the two in lockstep. #[derive(Debug, Clone, PartialEq, Eq)] pub enum License { /// MIT @@ -239,21 +244,34 @@ pub enum License { } impl License { - /// Map a license string to a [`License`]: curated SPDX identifiers are - /// matched case-insensitively, anything else becomes - /// [`License::Custom`] verbatim. + /// Map a license string to a [`License`]: the curated SPDX identifiers + /// of the bundled license table (`data/licenses.yml`, matched through + /// their spellings, case-insensitively) become their variant, + /// anything else becomes [`License::Custom`] verbatim (lowercased). pub fn parse(s: &str) -> License { - match s.to_ascii_lowercase().as_str() { - "mit" => License::Mit, - "apache-2.0" => License::Apache2, - "gpl-2.0+" => License::Gpl2Plus, - "gpl-3.0+" => License::Gpl3Plus, - "lgpl-2.1+" => License::Lgpl21Plus, - "lgpl-3.0+" => License::Lgpl3Plus, - "bsd-2-clause" => License::Bsd2Clause, - "bsd-3-clause" => License::Bsd3Clause, - "isc" => License::Isc, - other => License::Custom(other.to_string()), + match licenses::entry_for_spelling(s) { + Some(entry) => Self::from_spdx(&entry.id) + .expect("licenses.yml ids and License variants are kept in sync by a test"), + None => License::Custom(s.to_ascii_lowercase()), + } + } + + /// The variant of a curated license's SPDX identifier — the inverse of + /// [`License::spdx`] for the non-`Custom` variants, `None` for any + /// other string. The ids accepted here are locked to the bundled + /// license table by a consistency test. + fn from_spdx(id: &str) -> Option { + match id { + "MIT" => Some(License::Mit), + "Apache-2.0" => Some(License::Apache2), + "GPL-2.0+" => Some(License::Gpl2Plus), + "GPL-3.0+" => Some(License::Gpl3Plus), + "LGPL-2.1+" => Some(License::Lgpl21Plus), + "LGPL-3.0+" => Some(License::Lgpl3Plus), + "BSD-2-Clause" => Some(License::Bsd2Clause), + "BSD-3-Clause" => Some(License::Bsd3Clause), + "ISC" => Some(License::Isc), + _ => None, } } @@ -274,12 +292,11 @@ impl License { } /// SPDX license data page URL (without the trailing `+` of the - /// "or later" spellings), for the copyright reference paragraph. + /// "or later" spellings), for the copyright reference paragraph: the + /// identifier substituted into the URL template of the bundled + /// license table. pub fn spdx_url(&self) -> String { - format!( - "https://spdx.org/licenses/{}.html", - self.spdx().trim_end_matches('+') - ) + licenses::url_template().replace("{id}", self.spdx().trim_end_matches('+')) } } @@ -1177,6 +1194,56 @@ mod tests { ); } + /// The bundled license table and the `License` enum are two halves of + /// one list and must never drift apart: every entry id must map to a + /// variant (and the entry must accept its own id as a spelling, so + /// the menu answers parse back), every variant must have an entry, + /// and the counts must agree. The hardcoded variant list here is the + /// lock: adding a variant or a YAML entry without the other half + /// fails this test. + #[test] + fn licenses_table_and_enum_are_in_sync() { + let variants = [ + License::Mit, + License::Apache2, + License::Gpl2Plus, + License::Gpl3Plus, + License::Lgpl21Plus, + License::Lgpl3Plus, + License::Bsd2Clause, + License::Bsd3Clause, + License::Isc, + ]; + assert_eq!( + licenses::entries().len(), + variants.len(), + "licenses.yml and the License enum carry a different number of licenses" + ); + for variant in variants { + let entry = licenses::entries() + .iter() + .find(|entry| entry.id == variant.spdx()) + .unwrap_or_else(|| panic!("no licenses.yml entry for '{}'", variant.spdx())); + assert_eq!(entry.id, variant.spdx()); + assert!( + entry + .spellings + .iter() + .any(|s| s.eq_ignore_ascii_case(&entry.id)), + "entry '{}' must accept its own id as a spelling", + entry.id + ); + // The menu label parses back into the same variant. + assert_eq!(License::parse(&entry.menu), variant); + } + for entry in licenses::entries() { + let variant = License::from_spdx(&entry.id).unwrap_or_else(|| { + panic!("licenses.yml entry '{}' has no License variant", entry.id) + }); + assert_eq!(variant.spdx(), entry.id); + } + } + #[test] fn collision_detection() { assert!(check_file_collisions(&["a".into(), "b".into()]).is_ok()); diff --git a/src/new/questions.rs b/src/new/questions.rs index fa82212..13a1302 100644 --- a/src/new/questions.rs +++ b/src/new/questions.rs @@ -24,6 +24,7 @@ use indicatif::MultiProgress; use crate::new::detect::{self, Detection}; use crate::new::git; +use crate::new::licenses; use crate::new::options::{self, NewCli, NewOptions, SourceDir, SourceFormat, TemplateId}; use crate::new::origin::GitOrigin; use crate::new::templates::{self, ProbeResult, ScaffoldOutcome}; @@ -39,20 +40,6 @@ const SOURCE_PATH: &str = "Package the sources in another directory…"; /// The "everything else" entry of the license menu. const LICENSE_OTHER: &str = "Other (enter a SPDX identifier)"; -/// The curated SPDX identifiers of the license menu (without the free-text -/// entry), matching [`options::License::parse`]'s known spellings. -pub const KNOWN_LICENSES: [&str; 9] = [ - "MIT", - "Apache-2.0", - "GPL-2.0+", - "GPL-3.0+", - "LGPL-2.1+", - "LGPL-3.0+", - "BSD-2-Clause", - "BSD-3-Clause", - "ISC", -]; - /// Labels of the interactive `select` questions. `prompt::select` renders /// `>