From e4dd5c9816844c7a4b9429ee951b913ac80fcc89 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 17:27:29 +0000 Subject: [PATCH 01/44] Initial commit with task details Adding .gitkeep for PR creation (default mode). This file will be removed when the task is complete. Issue: https://github.com/link-foundation/link-cli/issues/105 --- .gitkeep | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitkeep diff --git a/.gitkeep b/.gitkeep new file mode 100644 index 0000000..747a20d --- /dev/null +++ b/.gitkeep @@ -0,0 +1 @@ +# .gitkeep file auto-generated at 2026-10-04T17:27:29.775Z for PR creation at branch issue-105-5190ec6fb31a for issue https://github.com/link-foundation/link-cli/issues/105 \ No newline at end of file From 67b8a67260cbf4a198aeae91c992d10cc491f10e Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 17:54:34 +0000 Subject: [PATCH 02/44] feat(rust): add text and binary LiNo protocol codecs Add the protocol module for issue #105: a dot-stuffed UTF-8 text protocol and a binary links packet whose header carries the link count and whose reference width grows with it (8/16/32/64 bits). External (Hybrid) references, the sequence section and progressive widths are optional and independently switchable. Includes golden vectors and hostile-input limits. --- .gitkeep | 1 - rust/examples/lino_binary_dump.rs | 39 +++ rust/src/lib.rs | 4 + rust/src/protocol/client.rs | 49 +++ rust/src/protocol/error.rs | 42 +++ rust/src/protocol/format.rs | 102 ++++++ rust/src/protocol/mapping.rs | 516 +++++++++++++++++++++++++++ rust/src/protocol/mod.rs | 56 +++ rust/src/protocol/packet.rs | 496 ++++++++++++++++++++++++++ rust/src/protocol/protocols.rs | 297 ++++++++++++++++ rust/src/protocol/server.rs | 295 ++++++++++++++++ rust/tests/protocol_packet_tests.rs | 525 ++++++++++++++++++++++++++++ 12 files changed, 2421 insertions(+), 1 deletion(-) delete mode 100644 .gitkeep create mode 100644 rust/examples/lino_binary_dump.rs create mode 100644 rust/src/protocol/client.rs create mode 100644 rust/src/protocol/error.rs create mode 100644 rust/src/protocol/format.rs create mode 100644 rust/src/protocol/mapping.rs create mode 100644 rust/src/protocol/mod.rs create mode 100644 rust/src/protocol/packet.rs create mode 100644 rust/src/protocol/protocols.rs create mode 100644 rust/src/protocol/server.rs create mode 100644 rust/tests/protocol_packet_tests.rs diff --git a/.gitkeep b/.gitkeep deleted file mode 100644 index 747a20d..0000000 --- a/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# .gitkeep file auto-generated at 2026-10-04T17:27:29.775Z for PR creation at branch issue-105-5190ec6fb31a for issue https://github.com/link-foundation/link-cli/issues/105 \ No newline at end of file diff --git a/rust/examples/lino_binary_dump.rs b/rust/examples/lino_binary_dump.rs new file mode 100644 index 0000000..ee31f2a --- /dev/null +++ b/rust/examples/lino_binary_dump.rs @@ -0,0 +1,39 @@ +//! Prints the binary LiNo encoding of a document under every option set. +//! +//! ```text +//! cargo run --example lino_binary_dump -- "() ((1 1))" +//! ``` + +use link_cli::protocol::{encode_document, parse_document, BinaryLinoOptions}; + +fn main() { + let text = std::env::args() + .nth(1) + .unwrap_or_else(|| "() ((1 1))".into()); + let document = parse_document(&text).expect("valid LiNo"); + println!("document: {text}"); + for external_references in [false, true] { + for sequences in [false, true] { + let options = BinaryLinoOptions { + external_references, + sequences, + progressive_widths: false, + }; + let packet = encode_document(&document, options).expect("encodable"); + let bytes = packet.to_bytes().expect("serializable"); + let hex = bytes + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::>() + .join(" "); + println!( + "external_references={external_references:<5} sequences={sequences:<5} {:>4} bytes: {hex}", + bytes.len() + ); + if std::env::var_os("VERBOSE").is_some() { + println!(" doublets: {:?}", packet.doublets); + println!(" sequences: {:?}", packet.sequences); + } + } + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 2b892f7..60ba2bf 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -50,6 +50,7 @@ pub mod named_types; pub mod parser; pub mod persistent_transformations; pub mod pinned_types; +pub mod protocol; pub mod query_options; pub mod query_processor; pub mod query_processor_substitution; @@ -85,6 +86,9 @@ pub use persistent_transformations::{ INTERNAL_NAME_PREFIX, }; pub use pinned_types::{PinnedTypes, PinnedTypesAccess, PinnedTypesDecorator}; +pub use protocol::{ + BinaryLinoOptions, BinaryLinoProtocol, LinksClient, LinksServer, LinoProtocol, TextLinoProtocol, +}; pub use query_options::QueryOptions; pub use query_processor::QueryProcessor; pub use storage::{ diff --git a/rust/src/protocol/client.rs b/rust/src/protocol/client.rs new file mode 100644 index 0000000..755a032 --- /dev/null +++ b/rust/src/protocol/client.rs @@ -0,0 +1,49 @@ +//! A TCP client for [`LinksServer`](super::LinksServer). + +use super::error::{ProtocolError, ProtocolResult}; +use super::format::{format_document, parse_document}; +use super::mapping::LinoDocument; +use super::protocols::{LinoConnection, LinoProtocol}; +use super::server::error_message; +use std::net::{TcpStream, ToSocketAddrs}; + +/// Sends substitution queries to a LiNo server over any [`LinoProtocol`]. +#[derive(Debug)] +pub struct LinksClient { + connection: LinoConnection>, +} + +impl LinksClient { + /// Connects to `address` using `protocol` for every message. + pub fn connect( + address: impl ToSocketAddrs, + protocol: impl LinoProtocol + 'static, + ) -> ProtocolResult { + let stream = TcpStream::connect(address)?; + let _ = stream.set_nodelay(true); + Ok(Self { + connection: LinoConnection::new(stream, Box::new(protocol)), + }) + } + + /// Sends a parsed document and returns the reply document. + /// + /// An `(error: 'message')` reply becomes [`ProtocolError::Remote`]. + pub fn request(&mut self, document: &LinoDocument) -> ProtocolResult { + let reply = self.connection.request(document)?; + if let Some(message) = error_message(&reply) { + return Err(ProtocolError::Remote(message.to_string())); + } + Ok(reply) + } + + /// Runs a LiNo substitution query; the empty query reads every link. + pub fn query(&mut self, query: &str) -> ProtocolResult { + self.request(&parse_document(query)?) + } + + /// Like [`query`](Self::query), returning the reply as canonical text. + pub fn query_text(&mut self, query: &str) -> ProtocolResult { + Ok(format_document(&self.query(query)?)) + } +} diff --git a/rust/src/protocol/error.rs b/rust/src/protocol/error.rs new file mode 100644 index 0000000..2b061fc --- /dev/null +++ b/rust/src/protocol/error.rs @@ -0,0 +1,42 @@ +//! Errors raised by the LiNo network protocols. + +use std::io; +use thiserror::Error; + +/// Everything that can go wrong while encoding, decoding or exchanging a +/// LiNo message. +#[derive(Debug, Error)] +pub enum ProtocolError { + /// The underlying transport failed. + #[error("I/O error: {0}")] + Io(#[from] io::Error), + + /// The peer sent bytes that are not a valid message. + #[error("malformed message: {0}")] + Malformed(String), + + /// The message text is not valid LiNo. + #[error("invalid LiNo: {0}")] + InvalidLino(String), + + /// The message exceeds one of the configured [`DecodeLimits`](super::DecodeLimits). + #[error("limit exceeded: {0}")] + LimitExceeded(String), + + /// The document cannot be represented with the chosen options. + #[error("cannot encode: {0}")] + Unencodable(String), + + /// The server answered with an `(error: …)` document. + #[error("server error: {0}")] + Remote(String), +} + +impl ProtocolError { + pub(crate) fn malformed(message: impl Into) -> Self { + Self::Malformed(message.into()) + } +} + +/// Result alias used throughout [`crate::protocol`]. +pub type ProtocolResult = Result; diff --git a/rust/src/protocol/format.rs b/rust/src/protocol/format.rs new file mode 100644 index 0000000..f6e7157 --- /dev/null +++ b/rust/src/protocol/format.rs @@ -0,0 +1,102 @@ +//! Canonical LiNo text for documents, shared by every protocol. +//! +//! `links_notation::format_links` does not quote references that contain +//! spaces or other delimiters, so it cannot be used to put a decoded binary +//! message back on the wire. This formatter guarantees that +//! `parse(format(document)) == document` for every parsed document, which is +//! what makes the text and binary protocols interchangeable. + +use super::error::{ProtocolError, ProtocolResult}; +use super::mapping::LinoDocument; +use links_notation::{parse_lino_to_links, LiNo}; + +/// Parses LiNo text into a document. Blank input is the empty document. +pub fn parse_document(text: &str) -> ProtocolResult { + if text.trim().is_empty() { + return Ok(Vec::new()); + } + parse_lino_to_links(text).map_err(|error| ProtocolError::InvalidLino(error.to_string())) +} + +/// Formats a document as canonical LiNo text, one top-level link per line. +/// +/// A top-level link without an id and with at least two values is written +/// without its outer parentheses, the way queries are usually typed: +/// `() ((1 1))`. +pub fn format_document(document: &[LiNo]) -> String { + document + .iter() + .map(format_top_level) + .collect::>() + .join("\n") +} + +fn format_top_level(link: &LiNo) -> String { + match link { + LiNo::Link { id: None, values } if values.len() >= 2 => join_values(values), + link => format_link(link), + } +} + +/// Formats one link as it appears nested inside another link. +pub fn format_link(link: &LiNo) -> String { + match link { + LiNo::Ref(reference) => format_reference(reference), + LiNo::Link { id: None, values } => match values.as_slice() { + // `(a)` parses back as the reference `a`, so a one-reference + // link needs a second pair of parentheses. + [LiNo::Ref(reference)] => format!("(({}))", format_reference(reference)), + values => format!("({})", join_values(values)), + }, + LiNo::Link { + id: Some(id), + values, + } => { + if values.is_empty() { + format!("({}:)", format_reference(id)) + } else { + format!("({}: {})", format_reference(id), join_values(values)) + } + } + } +} + +fn join_values(values: &[LiNo]) -> String { + values.iter().map(format_link).collect::>().join(" ") +} + +/// Quotes a reference when it would not survive parsing as a bare word. +/// +/// links-notation opens a quoted reference with a run of `N` equal quote +/// characters, closes it with the next run of exactly `N`, and reads `2N` +/// quotes inside as `N` literal ones. The opening run is counted greedily, so +/// the chosen quote must differ from the first character; `N` is one more +/// than the longest run of that quote inside, and odd, because an even +/// delimiter run may be read as an empty reference. +pub fn format_reference(reference: &str) -> String { + let needs_quotes = reference.is_empty() + || reference.chars().any(|character| { + character.is_whitespace() || matches!(character, '(' | ')' | ':' | '\'' | '"' | '`') + }); + if !needs_quotes { + return reference.to_string(); + } + let first = reference.chars().next(); + let (quote, count) = ['\'', '"', '`'] + .into_iter() + .filter(|"e| first != Some(quote)) + .map(|quote| (quote, (longest_run(reference, quote) + 1) | 1)) + .min_by_key(|&(_, count)| count) + .expect("a reference starts with at most one of three quote characters"); + let delimiter = quote.to_string().repeat(count); + format!("{delimiter}{reference}{delimiter}") +} + +fn longest_run(text: &str, quote: char) -> usize { + let (mut longest, mut current) = (0, 0); + for character in text.chars() { + current = if character == quote { current + 1 } else { 0 }; + longest = longest.max(current); + } + longest +} diff --git a/rust/src/protocol/mapping.rs b/rust/src/protocol/mapping.rs new file mode 100644 index 0000000..bb89eec --- /dev/null +++ b/rust/src/protocol/mapping.rs @@ -0,0 +1,516 @@ +//! Lossless mapping between LiNo documents and [`LinksPacket`]s. +//! +//! The mapping only uses links, the way linksplatform represents data: +//! +//! - `()` is the null link `0`. +//! - A numeric reference `n` is `(Number unary(n))`, where `unary(0)` is null, +//! `2^0` is the marker `One`, `2^k` is `(2^(k-1) 2^(k-1))` and other numbers +//! are right-nested sums of powers of two from the highest bit down. With +//! external references enabled, `n` is sent as an external reference instead. +//! - Any other reference is `(String code points…)`; every code point is a +//! unary number, or an external reference when those are enabled. +//! - A link without an id and with exactly two values is a plain doublet. +//! - A link without an id and any other number of values is a list. +//! - A link with an id is `(Identified id values…)`. +//! - The document is a list of its top-level links, stored last (the root). +//! +//! Typed values and lists are encoded as a variable-length sequence +//! `[marker, elements…]` when the sequence section is enabled, and as the +//! doublet `(marker chain)` otherwise, where `chain` is the nil-terminated cons +//! list `(e1 (e2 (… (en 0))))`. A plain list in sequence mode has no marker. +//! Identical sub-links are emitted once and shared, because links are +//! content-addressed. + +use super::error::{ProtocolError, ProtocolResult}; +use super::packet::{ + external_capacity, DecodeLimits, LinksPacket, Reference, FIRST_LINK_ADDRESS, IDENTIFIED, LIST, + NULL, NUMBER, ONE, STRING, +}; +use links_notation::LiNo; +use std::collections::HashMap; + +/// A LiNo document: the list of top-level links of a message. +pub type LinoDocument = Vec>; + +/// Optional features of the binary LiNo protocol. +/// +/// Every feature is off by default; each one can be switched on +/// independently, like stacking a decorator. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct BinaryLinoOptions { + /// Send numbers and code points as Hybrid external references instead of + /// in-band unary links. Halves the internal address range of each width. + pub external_references: bool, + /// Use the variable-length sequence section for lists, strings and links + /// with ids instead of cons chains of doublets. + pub sequences: bool, + /// Let the reference width grow with the address instead of using the + /// width of the largest address for every reference. + pub progressive_widths: bool, +} + +impl BinaryLinoOptions { + /// Enables or disables external references. + pub fn with_external_references(mut self, enabled: bool) -> Self { + self.external_references = enabled; + self + } + + /// Enables or disables the sequence section. + pub fn with_sequences(mut self, enabled: bool) -> Self { + self.sequences = enabled; + self + } + + /// Enables or disables progressive reference widths. + pub fn with_progressive_widths(mut self, enabled: bool) -> Self { + self.progressive_widths = enabled; + self + } +} + +/// Converts a document into a packet. +pub fn encode_document( + document: &[LiNo], + options: BinaryLinoOptions, +) -> ProtocolResult { + let mut encoder = Encoder::new(options); + if !document.is_empty() { + let items = document + .iter() + .map(|link| encoder.encode(link)) + .collect::>(); + if options.sequences { + encoder.sequences.push(items); + } else { + let chain = encoder.chain(&items); + encoder.doublets.push((Node::Internal(LIST), chain)); + } + } + encoder.finish() +} + +/// Converts a packet back into a document. +pub fn decode_document( + packet: &LinksPacket, + limits: &DecodeLimits, +) -> ProtocolResult { + let Some(root) = packet.last_address() else { + return Ok(Vec::new()); + }; + let decoder = Decoder::new(packet, limits); + let root = Reference::Internal(root); + let items = match decoder.view(root)? { + View::Sequence(items) if !starts_with_marker(items) => items.to_vec(), + View::Doublet(Reference::Internal(LIST), chain) => decoder.chain(chain)?, + _ => return Err(ProtocolError::malformed("the root link is not a list")), + }; + let mut budget = limits.max_nodes; + items + .into_iter() + .map(|item| decoder.decode(item, 0, &mut budget)) + .collect() +} + +/// Parses a canonical unsigned decimal number (no sign, no leading zeros). +pub(crate) fn canonical_number(text: &str) -> Option { + let canonical = !text.is_empty() + && text.bytes().all(|byte| byte.is_ascii_digit()) + && (text == "0" || !text.starts_with('0')); + canonical.then(|| text.parse().ok()).flatten() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum Node { + Internal(u64), + External(u64), + Doublet(usize), + Sequence(usize), +} + +struct Encoder { + options: BinaryLinoOptions, + doublets: Vec<(Node, Node)>, + sequences: Vec>, + doublet_index: HashMap<(Node, Node), usize>, + sequence_index: HashMap, usize>, + powers: Vec, +} + +impl Encoder { + fn new(options: BinaryLinoOptions) -> Self { + Self { + options, + doublets: Vec::new(), + sequences: Vec::new(), + doublet_index: HashMap::new(), + sequence_index: HashMap::new(), + powers: vec![Node::Internal(ONE)], + } + } + + fn doublet(&mut self, source: Node, target: Node) -> Node { + if let Some(&index) = self.doublet_index.get(&(source, target)) { + return Node::Doublet(index); + } + let index = self.doublets.len(); + self.doublets.push((source, target)); + self.doublet_index.insert((source, target), index); + Node::Doublet(index) + } + + fn sequence(&mut self, items: Vec) -> Node { + if let Some(&index) = self.sequence_index.get(&items) { + return Node::Sequence(index); + } + let index = self.sequences.len(); + self.sequences.push(items.clone()); + self.sequence_index.insert(items, index); + Node::Sequence(index) + } + + /// A two-element link. Fixed doublets may only refer to fixed doublets, + /// so a pair holding a sequence becomes a two-element sequence. + fn pair(&mut self, first: Node, second: Node) -> Node { + if matches!(first, Node::Sequence(_)) || matches!(second, Node::Sequence(_)) { + self.sequence(vec![first, second]) + } else { + self.doublet(first, second) + } + } + + fn chain(&mut self, items: &[Node]) -> Node { + items + .iter() + .rev() + .fold(Node::Internal(NULL), |tail, &head| self.doublet(head, tail)) + } + + fn typed(&mut self, marker: u64, elements: Vec) -> Node { + if self.options.sequences { + let mut items = Vec::with_capacity(elements.len() + 1); + items.push(Node::Internal(marker)); + items.extend(elements); + self.sequence(items) + } else { + let chain = self.chain(&elements); + self.doublet(Node::Internal(marker), chain) + } + } + + fn power(&mut self, exponent: usize) -> Node { + while self.powers.len() <= exponent { + let previous = *self.powers.last().expect("powers start with One"); + let next = self.doublet(previous, previous); + self.powers.push(next); + } + self.powers[exponent] + } + + fn unary(&mut self, value: u64) -> Node { + let bits = (0..64).rev().filter(|bit| value & (1u64 << bit) != 0); + let powers = bits.map(|bit| self.power(bit)).collect::>(); + let Some((&last, rest)) = powers.split_last() else { + return Node::Internal(NULL); + }; + rest.iter() + .rev() + .fold(last, |sum, &power| self.doublet(power, sum)) + } + + fn scalar(&mut self, value: u64) -> Node { + if self.options.external_references && value <= external_capacity(8) { + Node::External(value) + } else { + self.unary(value) + } + } + + fn reference(&mut self, text: &str) -> Node { + if let Some(value) = canonical_number(text) { + if self.options.external_references && value <= external_capacity(8) { + return Node::External(value); + } + let unary = self.unary(value); + return self.doublet(Node::Internal(NUMBER), unary); + } + let code_points = text + .chars() + .map(|character| self.scalar(u64::from(u32::from(character)))) + .collect(); + self.typed(STRING, code_points) + } + + fn encode(&mut self, link: &LiNo) -> Node { + match link { + LiNo::Ref(text) => self.reference(text), + LiNo::Link { id: None, values } => match values.as_slice() { + [] => Node::Internal(NULL), + [first, second] => { + let first = self.encode(first); + let second = self.encode(second); + self.pair(first, second) + } + values => { + let elements = values.iter().map(|value| self.encode(value)).collect(); + if self.options.sequences { + self.sequence(elements) + } else { + self.typed(LIST, elements) + } + } + }, + LiNo::Link { + id: Some(id), + values, + } => { + let mut elements = Vec::with_capacity(values.len() + 1); + elements.push(self.reference(id)); + elements.extend(values.iter().map(|value| self.encode(value))); + self.typed(IDENTIFIED, elements) + } + } + } + + fn finish(self) -> ProtocolResult { + let doublet_count = self.doublets.len() as u64; + let resolve = |node: Node| match node { + Node::Internal(address) => Reference::Internal(address), + Node::External(value) => Reference::External(value), + Node::Doublet(index) => Reference::Internal(FIRST_LINK_ADDRESS + index as u64), + Node::Sequence(index) => { + Reference::Internal(FIRST_LINK_ADDRESS + doublet_count + index as u64) + } + }; + let mut packet = LinksPacket { + external_references: self.options.external_references, + sequences_section: self.options.sequences, + min_width: 1, + doublets: self + .doublets + .iter() + .map(|&(source, target)| (resolve(source), resolve(target))) + .collect(), + sequences: self + .sequences + .iter() + .map(|items| items.iter().map(|&item| resolve(item)).collect()) + .collect(), + }; + packet.min_width = packet.required_min_width(!self.options.progressive_widths)?; + Ok(packet) + } +} + +enum View<'a> { + Null, + Marker(u64), + External(u64), + Doublet(Reference, Reference), + Sequence(&'a [Reference]), +} + +fn starts_with_marker(items: &[Reference]) -> bool { + matches!(items.first(), Some(Reference::Internal(address)) if (ONE..FIRST_LINK_ADDRESS).contains(address)) +} + +struct Decoder<'a> { + packet: &'a LinksPacket, + limits: &'a DecodeLimits, + /// `unary[i]` is the number doublet `i` denotes, if it is a unary number. + unary: Vec>, +} + +impl<'a> Decoder<'a> { + fn new(packet: &'a LinksPacket, limits: &'a DecodeLimits) -> Self { + // Links only refer backwards, so one forward pass evaluates every + // unary number without recursion. + let mut unary: Vec> = Vec::with_capacity(packet.doublets.len()); + for &(source, target) in &packet.doublets { + let value_of = |reference: Reference| match reference { + Reference::Internal(NULL) => Some(0), + Reference::Internal(ONE) => Some(1), + Reference::Internal(address) if address >= FIRST_LINK_ADDRESS => unary + .get((address - FIRST_LINK_ADDRESS) as usize) + .copied() + .flatten(), + _ => None, + }; + let value = value_of(source) + .zip(value_of(target)) + .and_then(|(source, target)| source.checked_add(target)); + unary.push(value); + } + Self { + packet, + limits, + unary, + } + } + + fn view(&self, reference: Reference) -> ProtocolResult> { + let address = match reference { + Reference::External(value) => return Ok(View::External(value)), + Reference::Internal(NULL) => return Ok(View::Null), + Reference::Internal(address) if address < FIRST_LINK_ADDRESS => { + return Ok(View::Marker(address)) + } + Reference::Internal(address) => address - FIRST_LINK_ADDRESS, + }; + let doublets = self.packet.doublets.len() as u64; + if address < doublets { + let (source, target) = self.packet.doublets[address as usize]; + Ok(View::Doublet(source, target)) + } else { + self.packet + .sequences + .get((address - doublets) as usize) + .map(|items| View::Sequence(items.as_slice())) + .ok_or_else(|| ProtocolError::malformed(format!("dangling reference {address}"))) + } + } + + fn number(&self, reference: Reference) -> ProtocolResult { + match reference { + Reference::External(value) => Ok(value), + Reference::Internal(NULL) => Ok(0), + Reference::Internal(ONE) => Ok(1), + Reference::Internal(address) if address >= FIRST_LINK_ADDRESS => self + .unary + .get((address - FIRST_LINK_ADDRESS) as usize) + .copied() + .flatten() + .ok_or_else(|| ProtocolError::malformed("expected a unary number")), + _ => Err(ProtocolError::malformed("expected a unary number")), + } + } + + /// The elements of a typed value given in doublet form: a cons chain, + /// optionally ending in a sequence holding the remaining elements. + fn chain(&self, mut tail: Reference) -> ProtocolResult> { + let mut elements = Vec::new(); + loop { + match self.view(tail)? { + View::Null => return Ok(elements), + View::Doublet(head, next) => { + if elements.len() >= self.limits.max_nodes { + return Err(ProtocolError::LimitExceeded("chain too long".into())); + } + elements.push(head); + tail = next; + } + View::Sequence(items) => { + elements.extend_from_slice(items); + return Ok(elements); + } + _ => return Err(ProtocolError::malformed("broken element chain")), + } + } + } + + fn typed( + &self, + marker: u64, + elements: &[Reference], + depth: usize, + budget: &mut usize, + ) -> ProtocolResult> { + match marker { + NUMBER => match elements { + [value] => Ok(LiNo::Ref(self.number(*value)?.to_string())), + _ => Err(ProtocolError::malformed("a number needs exactly one value")), + }, + STRING => { + let mut text = String::with_capacity(elements.len()); + for &element in elements { + let code_point = self.number(element)?; + let character = u32::try_from(code_point) + .ok() + .and_then(char::from_u32) + .ok_or_else(|| { + ProtocolError::malformed(format!("invalid code point {code_point}")) + })?; + text.push(character); + } + Ok(LiNo::Ref(text)) + } + LIST => self.list(elements, depth, budget), + IDENTIFIED => { + let (&id, values) = elements + .split_first() + .ok_or_else(|| ProtocolError::malformed("an identified link needs an id"))?; + let LiNo::Ref(id) = self.decode(id, depth + 1, budget)? else { + return Err(ProtocolError::malformed("a link id must be a reference")); + }; + let LiNo::Link { values, .. } = self.list(values, depth, budget)? else { + unreachable!("list always returns a link"); + }; + Ok(LiNo::Link { + id: Some(id), + values, + }) + } + _ => Err(ProtocolError::malformed(format!( + "marker {marker} cannot start a typed value" + ))), + } + } + + fn list( + &self, + elements: &[Reference], + depth: usize, + budget: &mut usize, + ) -> ProtocolResult> { + let values = elements + .iter() + .map(|&element| self.decode(element, depth + 1, budget)) + .collect::>>()?; + Ok(LiNo::Link { id: None, values }) + } + + fn decode( + &self, + reference: Reference, + depth: usize, + budget: &mut usize, + ) -> ProtocolResult> { + if depth >= self.limits.max_depth { + return Err(ProtocolError::LimitExceeded(format!( + "nesting deeper than {}", + self.limits.max_depth + ))); + } + *budget = budget + .checked_sub(1) + .ok_or_else(|| ProtocolError::LimitExceeded("too many LiNo nodes".into()))?; + match self.view(reference)? { + View::Null => Ok(LiNo::Link { + id: None, + values: Vec::new(), + }), + View::External(value) => Ok(LiNo::Ref(value.to_string())), + View::Marker(marker) => Err(ProtocolError::malformed(format!( + "marker {marker} used as a value" + ))), + View::Doublet(Reference::Internal(NUMBER), value) => { + self.typed(NUMBER, &[value], depth, budget) + } + View::Doublet(Reference::Internal(marker), chain) + if (ONE..FIRST_LINK_ADDRESS).contains(&marker) => + { + let elements = self.chain(chain)?; + self.typed(marker, &elements, depth, budget) + } + View::Doublet(source, target) => self.list(&[source, target], depth, budget), + View::Sequence(items) => match items.split_first() { + Some((&Reference::Internal(marker), elements)) + if (ONE..FIRST_LINK_ADDRESS).contains(&marker) => + { + self.typed(marker, elements, depth, budget) + } + _ => self.list(items, depth, budget), + }, + } + } +} diff --git a/rust/src/protocol/mod.rs b/rust/src/protocol/mod.rs new file mode 100644 index 0000000..27e1b9d --- /dev/null +++ b/rust/src/protocol/mod.rs @@ -0,0 +1,56 @@ +//! LiNo substitution operations over TCP/IP (issue #105). +//! +//! Two interchangeable protocols carry LiNo documents between a client and a +//! server: +//! +//! - [`TextLinoProtocol`]: UTF-8 LiNo text, each message ended by a line +//! holding only `.` (SMTP-style dot-stuffing keeps the framing unambiguous). +//! - [`BinaryLinoProtocol`]: a self-delimiting [`LinksPacket`] whose header +//! carries the number of links and whose reference width grows with the +//! number of links (8, 16, 32 or 64 bits). Its optional features — +//! [external references](BinaryLinoOptions::external_references), +//! [the sequence section](BinaryLinoOptions::sequences) and +//! [progressive widths](BinaryLinoOptions::progressive_widths) — are all +//! off by default and can be switched on one by one. +//! +//! Both implement [`LinoProtocol`], so a [`LinoConnection`] or a +//! [`LinksClient`] switches protocol by swapping one value. A +//! [`LinksServer`] detects the protocol of every message and answers in kind. +//! +//! A request is a substitution query, so it covers create, read, update and +//! delete; the reply lists the `(before) (after)` changes, exactly like +//! `clink --changes`. The empty request reads the whole store. +//! +//! ``` +//! use link_cli::protocol::{format_document, parse_document, BinaryLinoOptions, BinaryLinoProtocol}; +//! +//! let document = parse_document("() ((1 1))").unwrap(); +//! let binary = BinaryLinoProtocol::with_options( +//! BinaryLinoOptions::default().with_external_references(true), +//! ); +//! let bytes = binary.encode(&document).unwrap(); +//! assert_eq!(binary.decode(&bytes).unwrap(), document); +//! assert_eq!(format_document(&document), "() ((1 1))"); +//! ``` + +mod client; +mod error; +mod format; +mod mapping; +pub mod packet; +mod protocols; +mod server; + +pub use client::LinksClient; +pub use error::{ProtocolError, ProtocolResult}; +pub use format::{format_document, format_link, format_reference, parse_document}; +pub use mapping::{decode_document, encode_document, BinaryLinoOptions, LinoDocument}; +pub use packet::{DecodeLimits, LinksPacket, Reference}; +pub use protocols::{ + is_binary_start, read_any_document, BinaryLinoProtocol, LinoConnection, LinoProtocol, + MessageFormat, TextLinoProtocol, +}; +pub use server::{ + error_document, error_message, execute_request, AcceptedProtocols, LinksServer, ServerOptions, + ShutdownHandle, +}; diff --git a/rust/src/protocol/packet.rs b/rust/src/protocol/packet.rs new file mode 100644 index 0000000..7d311e7 --- /dev/null +++ b/rust/src/protocol/packet.rs @@ -0,0 +1,496 @@ +//! The binary links packet: the wire format of the binary LiNo protocol. +//! +//! ```text +//! byte 0 0x10 | flags high nibble 1 = format version 1 +//! bit 0 external references (Hybrid encoding) +//! bit 1 sequence section present +//! bits 2-3 log2 of the minimum width in bytes +//! LEB128 N number of fixed doublets +//! LEB128 M number of sequences (only when bit 1 is set) +//! N times source target one fixed doublet, refs `width(a)` bytes each +//! M times size ref_1 … ref_n one sequence, size and refs `width(a)` bytes each +//! ``` +//! +//! Addresses are implicit: `0` is null, `1..=5` are the reserved marker points +//! (never transmitted), the fixed doublets occupy `6..6+N` and the sequences +//! follow them, so the first sequence is "last fixed link + 1". +//! +//! Every reference stored in the link at address `a` uses +//! `width(a) = max(min_width, tier(a))` bytes, little-endian, where `tier(a)` +//! is the smallest of 1, 2, 4 and 8 bytes able to hold `a`. Because a link may +//! only refer to links *before* it, `tier(a)` always fits every internal +//! reference it can contain. With `min_width` equal to the tier of the largest +//! address the whole packet uses one uniform width (`0..256` addresses use +//! 8-bit references, `256..65536` 16-bit, and so on); with a smaller +//! `min_width` the width grows progressively with the address, the way LZW +//! code widths grow with the dictionary. +//! +//! With external references enabled the top bit of a reference marks it as +//! external, exactly like `Platform.Data.Hybrid`: value `v ≥ 1` is stored as +//! the two's-complement negation `2^bits - v` and value `0` as `2^(bits-1)`. +//! That halves the internal range of every width (`0..128` for 8-bit, …). + +use super::error::{ProtocolError, ProtocolResult}; +use std::io::{self, Read, Write}; + +/// The high nibble of the header byte. Text messages never start with a byte +/// in `0x10..=0x1F`, so the header doubles as a protocol detector. +pub const BINARY_VERSION_1: u8 = 0x10; + +const FLAG_EXTERNAL_REFERENCES: u8 = 0b0001; +const FLAG_SEQUENCES: u8 = 0b0010; +const WIDTH_SHIFT: u8 = 2; + +/// Null link address. +pub const NULL: u64 = 0; +/// Marker point `1`: the unary *one*; powers of two are `2^k = (2^(k-1) 2^(k-1))`. +pub const ONE: u64 = 1; +/// Marker point `2`: `(Number unary)` is a non-negative integer. +pub const NUMBER: u64 = 2; +/// Marker point `3`: `(String code points…)` is a Unicode string. +pub const STRING: u64 = 3; +/// Marker point `4`: `(List elements…)` is a list of links. +pub const LIST: u64 = 4; +/// Marker point `5`: `(Identified id values…)` is a link with an id. +pub const IDENTIFIED: u64 = 5; +/// Address of the first transmitted link. +pub const FIRST_LINK_ADDRESS: u64 = 6; + +/// The reference widths, in bytes, that a packet may use. +pub const WIDTHS: [u8; 4] = [1, 2, 4, 8]; + +/// One reference inside a packet. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Reference { + /// An address inside the packet (`0` null, `1..=5` markers, then links). + Internal(u64), + /// An external value, e.g. a number or a Unicode code point. + External(u64), +} + +impl Reference { + /// The null reference. + pub const NULL: Reference = Reference::Internal(NULL); +} + +/// Safety limits applied while decoding untrusted input. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DecodeLimits { + /// Maximum `N + M`. + pub max_links: u64, + /// Maximum total number of references inside all sequences. + pub max_sequence_items: u64, + /// Maximum number of LiNo nodes a packet may expand to. + pub max_nodes: usize, + /// Maximum LiNo nesting depth. + pub max_depth: usize, + /// Maximum size of a text message in bytes. + pub max_text_bytes: usize, +} + +impl Default for DecodeLimits { + fn default() -> Self { + Self { + max_links: 1 << 22, + max_sequence_items: 1 << 24, + max_nodes: 1 << 22, + max_depth: 1024, + max_text_bytes: 64 << 20, + } + } +} + +/// A decoded binary links packet. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LinksPacket { + /// Header bit 0: references may be external (Hybrid encoding). + pub external_references: bool, + /// Header bit 1: the packet carries a sequence section (possibly empty). + pub sequences_section: bool, + /// Minimum reference width in bytes: 1, 2, 4 or 8. + pub min_width: u8, + /// Fixed doublets at addresses `6..6+N`. + pub doublets: Vec<(Reference, Reference)>, + /// Variable-length sequences at addresses `6+N..6+N+M`. + pub sequences: Vec>, +} + +/// Largest internal address that fits in `width` bytes. +pub fn internal_capacity(width: u8, external_references: bool) -> u64 { + let bits = u32::from(width) * 8 - u32::from(external_references); + if bits >= 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + } +} + +/// Largest external value that fits in `width` bytes. +pub fn external_capacity(width: u8) -> u64 { + (1u64 << (u32::from(width) * 8 - 1)) - 1 +} + +/// Largest unsigned value (a sequence size) that fits in `width` bytes. +pub fn unsigned_capacity(width: u8) -> u64 { + internal_capacity(width, false) +} + +/// The narrowest width able to hold the internal address `address`. +pub fn address_tier(address: u64, external_references: bool) -> u8 { + WIDTHS + .into_iter() + .find(|&width| internal_capacity(width, external_references) >= address) + .unwrap_or(8) +} + +fn width_mask(width: u8) -> u64 { + internal_capacity(width, false) +} + +/// Encodes an external value at `width` the way `Platform.Data.Hybrid` does. +pub fn encode_external(value: u64, width: u8) -> u64 { + if value == 0 { + 1u64 << (u32::from(width) * 8 - 1) + } else { + value.wrapping_neg() & width_mask(width) + } +} + +/// Decodes a raw `width`-byte value, returning `Some(value)` for externals. +pub fn decode_external(raw: u64, width: u8) -> Option { + let external_zero = 1u64 << (u32::from(width) * 8 - 1); + if raw == external_zero { + Some(0) + } else if raw > external_zero { + Some(raw.wrapping_neg() & width_mask(width)) + } else { + None + } +} + +fn width_code(width: u8) -> ProtocolResult { + WIDTHS + .iter() + .position(|&candidate| candidate == width) + .map(|code| code as u8) + .ok_or_else(|| ProtocolError::Unencodable(format!("invalid width {width}"))) +} + +impl LinksPacket { + /// Address of the fixed doublet with zero-based index `index`. + pub fn doublet_address(&self, index: usize) -> u64 { + FIRST_LINK_ADDRESS + index as u64 + } + + /// Address of the sequence with zero-based index `index`. + pub fn sequence_address(&self, index: usize) -> u64 { + FIRST_LINK_ADDRESS + self.doublets.len() as u64 + index as u64 + } + + /// The highest address in the packet, or `None` for an empty packet. + pub fn last_address(&self) -> Option { + let count = self.doublets.len() + self.sequences.len(); + (count > 0).then(|| FIRST_LINK_ADDRESS + count as u64 - 1) + } + + /// Width used by every reference of the link at `address`. + pub fn width_at(&self, address: u64) -> u8 { + self.min_width + .max(address_tier(address, self.external_references)) + } + + /// The smallest `min_width` able to encode the packet. + /// + /// With `uniform` set, the result is at least the tier of the last + /// address, so every reference in the packet has the same width. + pub fn required_min_width(&self, uniform: bool) -> ProtocolResult { + let mut needed = 1u8; + let mut note = |address: u64, need: u8| { + if need > address_tier(address, self.external_references) { + needed = needed.max(need); + } + }; + for (index, &(source, target)) in self.doublets.iter().enumerate() { + let address = self.doublet_address(index); + note( + address, + self.reference_need(source)? + .max(self.reference_need(target)?), + ); + } + for (index, items) in self.sequences.iter().enumerate() { + let address = self.sequence_address(index); + let size_need = WIDTHS + .into_iter() + .find(|&width| unsigned_capacity(width) >= items.len() as u64) + .unwrap_or(8); + let mut need = size_need; + for &item in items { + need = need.max(self.reference_need(item)?); + } + note(address, need); + } + if uniform { + if let Some(last) = self.last_address() { + needed = needed.max(address_tier(last, self.external_references)); + } + } + Ok(needed) + } + + fn reference_need(&self, reference: Reference) -> ProtocolResult { + match reference { + Reference::Internal(_) => Ok(1), + Reference::External(value) => { + if !self.external_references { + return Err(ProtocolError::Unencodable( + "external reference in a packet without external references".into(), + )); + } + WIDTHS + .into_iter() + .find(|&width| external_capacity(width) >= value) + .ok_or_else(|| { + ProtocolError::Unencodable(format!( + "external value {value} exceeds 63 bits" + )) + }) + } + } + } + + fn header_byte(&self) -> ProtocolResult { + let mut header = BINARY_VERSION_1 | (width_code(self.min_width)? << WIDTH_SHIFT); + if self.external_references { + header |= FLAG_EXTERNAL_REFERENCES; + } + if self.sequences_section { + header |= FLAG_SEQUENCES; + } + Ok(header) + } + + fn raw_reference(&self, reference: Reference, address: u64, width: u8) -> ProtocolResult { + match reference { + Reference::Internal(target) => { + if target >= address { + return Err(ProtocolError::Unencodable(format!( + "link {address} refers forward to {target}" + ))); + } + Ok(target) + } + Reference::External(value) => { + if !self.external_references || value > external_capacity(width) { + return Err(ProtocolError::Unencodable(format!( + "external value {value} does not fit {width} byte(s) at link {address}" + ))); + } + Ok(encode_external(value, width)) + } + } + } + + /// Serializes the packet. + pub fn to_bytes(&self) -> ProtocolResult> { + let mut bytes = Vec::new(); + self.write_to(&mut bytes)?; + Ok(bytes) + } + + /// Writes the packet to `writer`. + pub fn write_to(&self, writer: &mut dyn Write) -> ProtocolResult<()> { + if !self.sequences_section && !self.sequences.is_empty() { + return Err(ProtocolError::Unencodable( + "sequences in a packet without a sequence section".into(), + )); + } + let mut out = vec![self.header_byte()?]; + write_leb128(&mut out, self.doublets.len() as u64); + if self.sequences_section { + write_leb128(&mut out, self.sequences.len() as u64); + } + for (index, &(source, target)) in self.doublets.iter().enumerate() { + let address = self.doublet_address(index); + let width = self.width_at(address); + write_raw(&mut out, self.raw_reference(source, address, width)?, width); + write_raw(&mut out, self.raw_reference(target, address, width)?, width); + } + for (index, items) in self.sequences.iter().enumerate() { + let address = self.sequence_address(index); + let width = self.width_at(address); + let size = items.len() as u64; + if size > unsigned_capacity(width) { + return Err(ProtocolError::Unencodable(format!( + "sequence size {size} does not fit {width} byte(s) at link {address}" + ))); + } + write_raw(&mut out, size, width); + for &item in items { + write_raw(&mut out, self.raw_reference(item, address, width)?, width); + } + } + writer.write_all(&out)?; + Ok(()) + } + + /// Parses a complete packet; trailing bytes are an error. + pub fn from_bytes(bytes: &[u8], limits: &DecodeLimits) -> ProtocolResult { + let mut cursor = bytes; + let packet = Self::read_from(&mut cursor, limits)? + .ok_or_else(|| ProtocolError::malformed("empty input"))?; + if !cursor.is_empty() { + return Err(ProtocolError::malformed(format!( + "{} trailing byte(s) after the packet", + cursor.len() + ))); + } + Ok(packet) + } + + /// Reads one packet from `reader`; `Ok(None)` on a clean end of stream. + pub fn read_from(reader: &mut dyn Read, limits: &DecodeLimits) -> ProtocolResult> { + let mut header = [0u8; 1]; + if read_exact_or_eof(reader, &mut header)? { + return Ok(None); + } + let header = header[0]; + if header & 0xF0 != BINARY_VERSION_1 { + return Err(ProtocolError::malformed(format!( + "unsupported binary header byte 0x{header:02X}" + ))); + } + let mut packet = LinksPacket { + external_references: header & FLAG_EXTERNAL_REFERENCES != 0, + sequences_section: header & FLAG_SEQUENCES != 0, + min_width: WIDTHS[usize::from((header >> WIDTH_SHIFT) & 0b11)], + ..LinksPacket::default() + }; + let doublet_count = read_leb128(reader)?; + let sequence_count = if packet.sequences_section { + read_leb128(reader)? + } else { + 0 + }; + let total = doublet_count + .checked_add(sequence_count) + .filter(|&total| total <= limits.max_links) + .ok_or_else(|| { + ProtocolError::LimitExceeded(format!( + "packet declares {doublet_count} + {sequence_count} links, limit is {}", + limits.max_links + )) + })?; + packet.doublets.reserve(total.min(4096) as usize); + for index in 0..doublet_count as usize { + let address = packet.doublet_address(index); + let width = packet.width_at(address); + let source = packet.read_reference(reader, address, width)?; + let target = packet.read_reference(reader, address, width)?; + packet.doublets.push((source, target)); + } + let mut items_left = limits.max_sequence_items; + for index in 0..sequence_count as usize { + let address = packet.sequence_address(index); + let width = packet.width_at(address); + let size = read_raw(reader, width)?; + items_left = items_left.checked_sub(size).ok_or_else(|| { + ProtocolError::LimitExceeded(format!( + "sequence items exceed the limit of {}", + limits.max_sequence_items + )) + })?; + let mut items = Vec::with_capacity(size.min(4096) as usize); + for _ in 0..size { + items.push(packet.read_reference(reader, address, width)?); + } + packet.sequences.push(items); + } + Ok(Some(packet)) + } + + fn read_reference( + &self, + reader: &mut dyn Read, + address: u64, + width: u8, + ) -> ProtocolResult { + let raw = read_raw(reader, width)?; + if self.external_references { + if let Some(value) = decode_external(raw, width) { + return Ok(Reference::External(value)); + } + } + if raw >= address { + return Err(ProtocolError::malformed(format!( + "link {address} refers to {raw}, which is not an earlier link" + ))); + } + Ok(Reference::Internal(raw)) + } +} + +fn write_raw(out: &mut Vec, value: u64, width: u8) { + out.extend_from_slice(&value.to_le_bytes()[..usize::from(width)]); +} + +fn read_raw(reader: &mut dyn Read, width: u8) -> ProtocolResult { + let mut bytes = [0u8; 8]; + read_exact(reader, &mut bytes[..usize::from(width)])?; + Ok(u64::from_le_bytes(bytes)) +} + +/// Appends `value` as unsigned LEB128. +pub fn write_leb128(out: &mut Vec, mut value: u64) { + loop { + let byte = (value & 0x7F) as u8; + value >>= 7; + if value == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +/// Reads an unsigned LEB128 value of at most 64 bits. +pub fn read_leb128(reader: &mut dyn Read) -> ProtocolResult { + let mut value = 0u64; + for shift in (0..64).step_by(7) { + let mut byte = [0u8; 1]; + read_exact(reader, &mut byte)?; + let payload = u64::from(byte[0] & 0x7F); + if shift == 63 && payload > 1 { + return Err(ProtocolError::malformed("LEB128 value overflows 64 bits")); + } + value |= payload << shift; + if byte[0] & 0x80 == 0 { + return Ok(value); + } + } + Err(ProtocolError::malformed("LEB128 value overflows 64 bits")) +} + +fn read_exact(reader: &mut dyn Read, buffer: &mut [u8]) -> ProtocolResult<()> { + reader.read_exact(buffer).map_err(|error| { + if error.kind() == io::ErrorKind::UnexpectedEof { + ProtocolError::malformed("unexpected end of packet") + } else { + ProtocolError::Io(error) + } + }) +} + +/// Fills `buffer`, returning `true` if the stream ended before the first byte. +fn read_exact_or_eof(reader: &mut dyn Read, buffer: &mut [u8]) -> ProtocolResult { + loop { + match reader.read(&mut buffer[..1]) { + Ok(0) => return Ok(true), + Ok(_) => break, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error.into()), + } + } + read_exact(reader, &mut buffer[1..])?; + Ok(false) +} diff --git a/rust/src/protocol/protocols.rs b/rust/src/protocol/protocols.rs new file mode 100644 index 0000000..5e0a319 --- /dev/null +++ b/rust/src/protocol/protocols.rs @@ -0,0 +1,297 @@ +//! Interchangeable message protocols for LiNo documents. +//! +//! Both [`TextLinoProtocol`] and [`BinaryLinoProtocol`] implement +//! [`LinoProtocol`], so code written against the trait switches between them +//! by swapping one value. [`LinoConnection`] decorates any byte stream (a +//! `TcpStream`, a pipe, an in-memory buffer) with a protocol. + +use super::error::{ProtocolError, ProtocolResult}; +use super::format::{format_document, parse_document}; +use super::mapping::{decode_document, encode_document, BinaryLinoOptions, LinoDocument}; +use super::packet::{address_tier, DecodeLimits, LinksPacket, BINARY_VERSION_1}; +use links_notation::LiNo; +use std::fmt::Debug; +use std::io::{BufRead, BufReader, Read, Write}; + +/// Reads and writes whole LiNo documents on a byte stream. +pub trait LinoProtocol: Debug + Send + Sync { + /// Writes one message carrying `document`. + fn write_document( + &self, + writer: &mut dyn Write, + document: &[LiNo], + ) -> ProtocolResult<()>; + + /// Reads one message; `Ok(None)` when the stream ended cleanly. + fn read_document(&self, reader: &mut dyn BufRead) -> ProtocolResult>; +} + +impl LinoProtocol for Box

{ + fn write_document( + &self, + writer: &mut dyn Write, + document: &[LiNo], + ) -> ProtocolResult<()> { + (**self).write_document(writer, document) + } + + fn read_document(&self, reader: &mut dyn BufRead) -> ProtocolResult> { + (**self).read_document(reader) + } +} + +/// UTF-8 LiNo text, one message per block of lines ended by a line holding +/// only `.`. Lines starting with `.` get one extra `.` (SMTP dot-stuffing). +/// +/// Lines may end with `\n` or `\r\n`; a carriage return right before a line +/// feed is dropped, so a quoted reference holding `\r\n` arrives as `\n` +/// (use [`BinaryLinoProtocol`] to carry such references exactly). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct TextLinoProtocol { + /// Limits applied when reading. + pub limits: DecodeLimits, +} + +impl TextLinoProtocol { + /// A text protocol with default limits. + pub fn new() -> Self { + Self::default() + } + + /// Frames already formatted LiNo text as one message. + pub fn write_text(writer: &mut dyn Write, text: &str) -> ProtocolResult<()> { + let mut out = String::with_capacity(text.len() + 4); + if !text.is_empty() { + for line in text.split('\n') { + if line.starts_with('.') { + out.push('.'); + } + out.push_str(line); + out.push('\n'); + } + } + out.push_str(".\n"); + writer.write_all(out.as_bytes())?; + writer.flush()?; + Ok(()) + } + + /// Reads one framed message as raw text, without parsing it. + pub fn read_text(&self, reader: &mut dyn BufRead) -> ProtocolResult> { + let mut text = Vec::new(); + let mut line = Vec::new(); + let mut first = true; + loop { + line.clear(); + let budget = self.limits.max_text_bytes.saturating_sub(text.len()) as u64 + 2; + let read = reader.take(budget).read_until(b'\n', &mut line)?; + if read == 0 { + if first { + return Ok(None); + } + return Err(ProtocolError::malformed( + "stream ended before the '.' terminator line", + )); + } + if line.last() != Some(&b'\n') { + if read as u64 >= budget { + return Err(ProtocolError::LimitExceeded(format!( + "text message longer than {} bytes", + self.limits.max_text_bytes + ))); + } + return Err(ProtocolError::malformed( + "stream ended before the '.' terminator line", + )); + } + line.pop(); + // CRLF line endings (telnet, netcat on Windows) are accepted. + if line.last() == Some(&b'\r') { + line.pop(); + } + if line == b"." { + break; + } + if !first { + text.push(b'\n'); + } + first = false; + let content = line.strip_prefix(b".").unwrap_or(&line); + text.extend_from_slice(content); + } + String::from_utf8(text) + .map(Some) + .map_err(|_| ProtocolError::malformed("text message is not valid UTF-8")) + } +} + +impl LinoProtocol for TextLinoProtocol { + fn write_document( + &self, + writer: &mut dyn Write, + document: &[LiNo], + ) -> ProtocolResult<()> { + Self::write_text(writer, &format_document(document)) + } + + fn read_document(&self, reader: &mut dyn BufRead) -> ProtocolResult> { + self.read_text(reader)? + .map(|text| parse_document(&text)) + .transpose() + } +} + +/// Binary links packets (see [`crate::protocol::packet`]); every message is +/// self-delimiting, so no extra framing is needed. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct BinaryLinoProtocol { + /// Optional features used when writing. + pub options: BinaryLinoOptions, + /// Limits applied when reading. + pub limits: DecodeLimits, +} + +impl BinaryLinoProtocol { + /// A binary protocol with every optional feature off. + pub fn new() -> Self { + Self::default() + } + + /// A binary protocol with the given optional features. + pub fn with_options(options: BinaryLinoOptions) -> Self { + Self { + options, + ..Self::default() + } + } + + /// Encodes a document into packet bytes. + pub fn encode(&self, document: &[LiNo]) -> ProtocolResult> { + encode_document(document, self.options)?.to_bytes() + } + + /// Decodes packet bytes into a document. + pub fn decode(&self, bytes: &[u8]) -> ProtocolResult { + decode_document(&LinksPacket::from_bytes(bytes, &self.limits)?, &self.limits) + } +} + +impl LinoProtocol for BinaryLinoProtocol { + fn write_document( + &self, + writer: &mut dyn Write, + document: &[LiNo], + ) -> ProtocolResult<()> { + writer.write_all(&self.encode(document)?)?; + writer.flush()?; + Ok(()) + } + + fn read_document(&self, reader: &mut dyn BufRead) -> ProtocolResult> { + LinksPacket::read_from(reader, &self.limits)? + .map(|packet| decode_document(&packet, &self.limits)) + .transpose() + } +} + +/// The wire format a message arrived in, so a reply can use the same one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MessageFormat { + /// [`TextLinoProtocol`]. + Text, + /// [`BinaryLinoProtocol`] with the options read from the packet header. + Binary(BinaryLinoOptions), +} + +impl MessageFormat { + /// A protocol that writes messages in this format. + pub fn protocol(self, limits: DecodeLimits) -> Box { + match self { + MessageFormat::Text => Box::new(TextLinoProtocol { limits }), + MessageFormat::Binary(options) => Box::new(BinaryLinoProtocol { options, limits }), + } + } +} + +/// True when `byte` starts a binary message rather than a text one. +pub fn is_binary_start(byte: u8) -> bool { + byte & 0xF0 == BINARY_VERSION_1 +} + +/// Reads one message in whichever protocol the peer used. +pub fn read_any_document( + reader: &mut dyn BufRead, + limits: &DecodeLimits, +) -> ProtocolResult> { + let Some(&first) = reader.fill_buf()?.first() else { + return Ok(None); + }; + if !is_binary_start(first) { + let text = TextLinoProtocol { limits: *limits }; + return Ok(text + .read_document(reader)? + .map(|document| (document, MessageFormat::Text))); + } + let Some(packet) = LinksPacket::read_from(reader, limits)? else { + return Ok(None); + }; + let progressive = packet + .last_address() + .is_some_and(|last| packet.min_width < address_tier(last, packet.external_references)); + let options = BinaryLinoOptions { + external_references: packet.external_references, + sequences: packet.sequences_section, + progressive_widths: progressive, + }; + let document = decode_document(&packet, limits)?; + Ok(Some((document, MessageFormat::Binary(options)))) +} + +/// A byte stream decorated with a [`LinoProtocol`]. +#[derive(Debug)] +pub struct LinoConnection { + stream: BufReader, + protocol: P, +} + +impl LinoConnection { + /// Wraps `stream`. + pub fn new(stream: S, protocol: P) -> Self { + Self { + stream: BufReader::new(stream), + protocol, + } + } + + /// The protocol in use. + pub fn protocol(&self) -> &P { + &self.protocol + } + + /// Sends one document. + pub fn send(&mut self, document: &[LiNo]) -> ProtocolResult<()> { + let mut buffer = Vec::new(); + self.protocol.write_document(&mut buffer, document)?; + let stream = self.stream.get_mut(); + stream.write_all(&buffer)?; + stream.flush()?; + Ok(()) + } + + /// Receives one document; `Ok(None)` when the peer closed the stream. + pub fn receive(&mut self) -> ProtocolResult> { + self.protocol.read_document(&mut self.stream) + } + + /// Sends `document` and waits for the reply. + pub fn request(&mut self, document: &[LiNo]) -> ProtocolResult { + self.send(document)?; + self.receive()? + .ok_or_else(|| ProtocolError::malformed("connection closed before the reply")) + } + + /// Unwraps the underlying stream. + pub fn into_inner(self) -> S { + self.stream.into_inner() + } +} diff --git a/rust/src/protocol/server.rs b/rust/src/protocol/server.rs new file mode 100644 index 0000000..3f406c7 --- /dev/null +++ b/rust/src/protocol/server.rs @@ -0,0 +1,295 @@ +//! A TCP server exposing a links store through the LiNo protocols. +//! +//! Each connection gets its own thread that only parses and formats +//! messages; every request is executed by the thread that called +//! [`LinksServer::serve`], one at a time, so the store needs neither `Send` +//! nor locking. The protocol is detected per message and the reply uses the +//! same protocol, so text and binary clients can share one server. + +use super::error::{ProtocolError, ProtocolResult}; +use super::format::{format_document, parse_document}; +use super::mapping::LinoDocument; +use super::packet::DecodeLimits; +use super::protocols::{read_any_document, MessageFormat}; +use crate::named_type_links::NamedTypeLinks; +use crate::query_processor::QueryProcessor; +use links_notation::LiNo; +use std::io::{BufReader, BufWriter, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream, ToSocketAddrs}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::sync::Arc; +use std::thread; + +/// Which protocols a server accepts. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AcceptedProtocols { + /// Detect the protocol of every message (the default). + #[default] + Any, + /// Only [`TextLinoProtocol`](super::TextLinoProtocol) messages. + Text, + /// Only [`BinaryLinoProtocol`](super::BinaryLinoProtocol) messages. + Binary, +} + +impl AcceptedProtocols { + fn accepts(self, format: MessageFormat) -> bool { + matches!( + (self, format), + (AcceptedProtocols::Any, _) + | (AcceptedProtocols::Text, MessageFormat::Text) + | (AcceptedProtocols::Binary, MessageFormat::Binary(_)) + ) + } +} + +/// Server configuration. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ServerOptions { + /// Print every request and reply to stderr. + pub trace: bool, + /// Passed to [`QueryProcessor::with_auto_create_missing_references`]. + pub auto_create_missing_references: bool, + /// Protocols the server answers. + pub accept: AcceptedProtocols, + /// Limits applied to incoming messages. + pub limits: DecodeLimits, +} + +enum Job { + Request { + document: LinoDocument, + reply: Sender, + }, + Shutdown, +} + +/// Stops a running [`LinksServer::serve`] from another thread. +#[derive(Clone, Debug)] +pub struct ShutdownHandle { + jobs: Sender, + stopping: Arc, + address: SocketAddr, +} + +impl std::fmt::Debug for Job { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Job::Request { document, .. } => write!(formatter, "Request({document:?})"), + Job::Shutdown => write!(formatter, "Shutdown"), + } + } +} + +impl ShutdownHandle { + /// Asks the server to stop; `serve` returns after the current request. + pub fn shutdown(&self) { + self.stopping.store(true, Ordering::SeqCst); + let _ = self.jobs.send(Job::Shutdown); + // Wake the accept loop so it notices the flag. + let _ = TcpStream::connect(self.address); + } +} + +/// A bound, not yet serving, LiNo server. +#[derive(Debug)] +pub struct LinksServer { + listener: TcpListener, + options: ServerOptions, + jobs: Sender, + queue: Receiver, + stopping: Arc, +} + +impl LinksServer { + /// Binds to `address` (use port `0` for an ephemeral port). + pub fn bind(address: impl ToSocketAddrs, options: ServerOptions) -> ProtocolResult { + let listener = TcpListener::bind(address)?; + let (jobs, queue) = mpsc::channel(); + Ok(Self { + listener, + options, + jobs, + queue, + stopping: Arc::new(AtomicBool::new(false)), + }) + } + + /// The address the server listens on. + pub fn local_addr(&self) -> ProtocolResult { + Ok(self.listener.local_addr()?) + } + + /// A handle that stops [`serve`](Self::serve). + pub fn shutdown_handle(&self) -> ProtocolResult { + Ok(ShutdownHandle { + jobs: self.jobs.clone(), + stopping: Arc::clone(&self.stopping), + address: self.local_addr()?, + }) + } + + /// Serves `storage` until a [`ShutdownHandle`] stops the server. + pub fn serve(self, storage: &mut S) -> ProtocolResult<()> { + let LinksServer { + listener, + options, + jobs, + queue, + stopping, + } = self; + let acceptor = { + let stopping = Arc::clone(&stopping); + thread::spawn(move || accept_loop(listener, jobs, stopping, options)) + }; + let processor = QueryProcessor::new(false) + .with_auto_create_missing_references(options.auto_create_missing_references); + while let Ok(job) = queue.recv() { + match job { + Job::Shutdown => break, + Job::Request { document, reply } => { + if options.trace { + eprintln!("[server] request: {}", format_document(&document)); + } + let response = execute_request(storage, &processor, &document); + if options.trace { + eprintln!("[server] reply: {}", format_document(&response)); + } + let _ = reply.send(response); + } + } + } + stopping.store(true, Ordering::SeqCst); + drop(queue); + let _ = acceptor.join(); + Ok(()) + } +} + +fn accept_loop( + listener: TcpListener, + jobs: Sender, + stopping: Arc, + options: ServerOptions, +) { + for stream in listener.incoming() { + if stopping.load(Ordering::SeqCst) { + return; + } + let Ok(stream) = stream else { continue }; + let jobs = jobs.clone(); + thread::spawn(move || { + if let Err(error) = handle_connection(stream, jobs, options) { + if options.trace { + eprintln!("[server] connection closed: {error}"); + } + } + }); + } +} + +fn handle_connection( + stream: TcpStream, + jobs: Sender, + options: ServerOptions, +) -> ProtocolResult<()> { + let _ = stream.set_nodelay(true); + let mut writer = BufWriter::new(stream.try_clone()?); + let mut reader = BufReader::new(stream); + loop { + let (document, format) = match read_any_document(&mut reader, &options.limits) { + Ok(Some(message)) => message, + Ok(None) => return Ok(()), + Err(error) => { + // The stream may be out of sync; answer in text and hang up. + let reply = error_document(&error.to_string()); + let _ = MessageFormat::Text + .protocol(options.limits) + .write_document(&mut writer, &reply); + return Err(error); + } + }; + let response = if options.accept.accepts(format) { + let (reply, response) = mpsc::channel(); + jobs.send(Job::Request { document, reply }) + .map_err(|_| ProtocolError::malformed("server is shutting down"))?; + response + .recv() + .map_err(|_| ProtocolError::malformed("server is shutting down"))? + } else { + error_document("this server does not accept this protocol") + }; + format + .protocol(options.limits) + .write_document(&mut writer, &response)?; + writer.flush()?; + } +} + +/// The reply document for a failed request: `(error: 'message')`. +pub fn error_document(message: &str) -> LinoDocument { + vec![LiNo::Link { + id: Some("error".to_string()), + values: vec![LiNo::Ref(message.to_string())], + }] +} + +/// Returns the message of an `(error: 'message')` reply. +pub fn error_message(document: &[LiNo]) -> Option<&str> { + match document { + [LiNo::Link { + id: Some(id), + values, + }] if id == "error" => match values.as_slice() { + [LiNo::Ref(message)] => Some(message), + _ => None, + }, + _ => None, + } +} + +/// Executes one request against `storage`. +/// +/// A non-empty document is a substitution query; the reply holds one +/// `(before) (after)` line per change, exactly like `clink --changes`. The +/// empty document asks for every link, one `(index: source target)` per line. +/// Failures produce an [`error_document`]. +pub fn execute_request( + storage: &mut S, + processor: &QueryProcessor, + document: &[LiNo], +) -> LinoDocument { + match try_execute_request(storage, processor, document) { + Ok(reply) => reply, + Err(error) => error_document(&format!("{error:#}")), + } +} + +fn try_execute_request( + storage: &mut S, + processor: &QueryProcessor, + document: &[LiNo], +) -> anyhow::Result { + let lines = if document.is_empty() { + storage.lino_lines()? + } else { + let changes = processor.process_query(storage, &format_document(document))?; + if !changes.is_empty() { + storage.save()?; + } + changes + .iter() + .map(|(before, after)| { + let before = before.map(|link| storage.format_lino(&link)).transpose()?; + let after = after.map(|link| storage.format_lino(&link)).transpose()?; + Ok(format!( + "({}) ({})", + before.unwrap_or_default(), + after.unwrap_or_default() + )) + }) + .collect::>>()? + }; + Ok(parse_document(&lines.join("\n"))?) +} diff --git a/rust/tests/protocol_packet_tests.rs b/rust/tests/protocol_packet_tests.rs new file mode 100644 index 0000000..373f7a7 --- /dev/null +++ b/rust/tests/protocol_packet_tests.rs @@ -0,0 +1,525 @@ +//! Binary and text LiNo protocol codecs (issue #105). + +use link_cli::external_reference; +use link_cli::protocol::packet::{ + address_tier, decode_external, encode_external, external_capacity, internal_capacity, +}; +use link_cli::protocol::{ + decode_document, encode_document, format_document, format_reference, parse_document, + read_any_document, BinaryLinoOptions, BinaryLinoProtocol, DecodeLimits, LinksPacket, + LinoProtocol, MessageFormat, ProtocolError, Reference, TextLinoProtocol, +}; +use std::io::Cursor; + +const CORPUS: &[&str] = &[ + "() ((1 1))", + "((1: 1 1)) ((1: 1 2))", + "((1 1)) ()", + "((1: 1 1)) ()", + "(($i: $s $t)) (($i: $s $t))", + "((($index: $source $target)) (($index: $target $source)))", + "(a b c d)", + "(name: 'with space' \"it's\")", + "((a))", + "(((a)))", + "(a (b c) ((d)))", + "1\n2\n3", + "hello", + "😀 привет 世界", + "0", + "007", + "9223372036854775807", + "9223372036854775808", + "18446744073709551615", + "18446744073709551616", + "'multi\nline'", + ".dot", + "'a''b\"c`d'", + "(1: (2: 3 4) 5)", + "() ()", + "(* *)", + "(type: type type)", +]; + +fn all_options() -> Vec { + let mut options = Vec::new(); + for external_references in [false, true] { + for sequences in [false, true] { + for progressive_widths in [false, true] { + options.push(BinaryLinoOptions { + external_references, + sequences, + progressive_widths, + }); + } + } + } + options +} + +fn hex(bytes: &[u8]) -> String { + bytes + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::>() + .join(" ") +} + +fn unhex(text: &str) -> Vec { + text.split_whitespace() + .map(|byte| u8::from_str_radix(byte, 16).unwrap()) + .collect() +} + +fn encode_hex(text: &str, external_references: bool, sequences: bool) -> String { + let binary = BinaryLinoProtocol::with_options(BinaryLinoOptions { + external_references, + sequences, + progressive_widths: false, + }); + hex(&binary.encode(&parse_document(text).unwrap()).unwrap()) +} + +/// Golden vectors shared with the C# test suite: both implementations must +/// produce exactly these bytes. +const GOLDEN: &[(&str, bool, bool, &str)] = &[ + ( + "() ((1 1))", + false, + false, + "10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b", + ), + ( + "() ((1 1))", + false, + true, + "12 02 03 02 01 06 06 01 07 02 00 08 01 09", + ), + ( + "() ((1 1))", + true, + false, + "11 06 ff ff 06 00 04 07 00 08 09 00 04 0a", + ), + ( + "() ((1 1))", + true, + true, + "13 01 03 ff ff 01 06 02 00 07 01 08", + ), + ("hi", true, false, "11 05 97 00 98 06 03 07 08 00 04 09"), + ("hi", true, true, "13 00 02 03 03 98 97 01 06"), + ("", false, false, "10 00"), + ("", true, true, "13 00 00"), +]; + +#[test] +fn golden_vectors_are_stable() { + for &(text, external_references, sequences, expected) in GOLDEN { + assert_eq!( + encode_hex(text, external_references, sequences), + expected, + "{text:?} external={external_references} sequences={sequences}" + ); + let binary = BinaryLinoProtocol::new(); + assert_eq!( + binary.decode(&unhex(expected)).unwrap(), + parse_document(text).unwrap() + ); + } +} + +#[test] +fn every_option_set_round_trips_the_corpus() { + for text in CORPUS { + let document = parse_document(text).unwrap(); + for options in all_options() { + let binary = BinaryLinoProtocol::with_options(options); + let bytes = binary.encode(&document).unwrap(); + assert!((0x10..=0x1F).contains(&bytes[0])); + assert_eq!( + binary.decode(&bytes).unwrap(), + document, + "{text:?} with {options:?}" + ); + } + } +} + +#[test] +fn canonical_text_round_trips_the_corpus() { + for text in CORPUS { + let document = parse_document(text).unwrap(); + let canonical = format_document(&document); + assert_eq!( + parse_document(&canonical).unwrap(), + document, + "{text:?} → {canonical:?}" + ); + } + assert_eq!( + format_document(&parse_document("(()((1 1)))").unwrap()), + "() ((1 1))" + ); + assert_eq!( + format_document(&parse_document("((1: 1 1)) ((1: 1 2))").unwrap()), + "((1: 1 1)) ((1: 1 2))" + ); +} + +#[test] +fn references_are_quoted_only_when_needed() { + assert_eq!(format_reference("plain"), "plain"); + assert_eq!(format_reference("$x"), "$x"); + assert_eq!(format_reference(""), "''"); + assert_eq!(format_reference("a b"), "'a b'"); + assert_eq!(format_reference("it's"), "\"it's\""); + assert_eq!(format_reference("'\""), "`'\"`"); + assert_eq!(format_reference("'\"`"), "\"\"\"'\"`\"\"\""); + for reference in [ + "", + "a b", + "a:b", + "(x)", + "it's", + "'\"", + "'\"`", + "tab\there", + "a\nb", + "'x", + "x'", + "'", + "''", + "'''", + "\"'`x`'\"", + "a''''b\"\"`", + "`'\"\"\"'''``", + "(a) ''", + ] { + let document = parse_document(&format_reference(reference)).unwrap(); + assert_eq!(format_document(&document), format_reference(reference)); + assert_eq!( + document, + vec![links_notation::LiNo::Ref(reference.to_string())] + ); + } +} + +#[test] +fn width_tiers_follow_the_number_of_links() { + assert_eq!(address_tier(0, false), 1); + assert_eq!(address_tier(255, false), 1); + assert_eq!(address_tier(256, false), 2); + assert_eq!(address_tier(65_535, false), 2); + assert_eq!(address_tier(65_536, false), 4); + assert_eq!(address_tier(u64::from(u32::MAX), false), 4); + assert_eq!(address_tier(u64::from(u32::MAX) + 1, false), 8); + // External references take the top bit, halving every range. + assert_eq!(address_tier(127, true), 1); + assert_eq!(address_tier(128, true), 2); + assert_eq!(address_tier(32_767, true), 2); + assert_eq!(address_tier(32_768, true), 4); + assert_eq!(internal_capacity(8, false), u64::MAX); + assert_eq!(internal_capacity(8, true), i64::MAX as u64); + assert_eq!(external_capacity(1), 127); +} + +#[test] +fn external_references_match_platform_data_hybrid() { + for width in [1u8, 2, 4, 8] { + for value in [0, 1, 2, 100, external_capacity(width)] { + let raw = encode_external(value, width); + assert_eq!( + decode_external(raw, width), + Some(value), + "{value} @ {width}" + ); + } + assert_eq!(decode_external(internal_capacity(width, true), width), None); + } + assert_eq!(encode_external(1, 1), 0xFF); + assert_eq!(encode_external(0, 1), 0x80); + for value in [0u32, 1, 5, 1000, i32::MAX as u32] { + assert_eq!( + encode_external(u64::from(value), 4), + u64::from(external_reference(value)) + ); + } +} + +fn many_links(count: usize) -> String { + // Distinct pairs of distinct names keep every link unique. + (0..count) + .map(|index| format!("(n{index} m{index})")) + .collect::>() + .join("\n") +} + +#[test] +fn uniform_width_grows_past_256_addresses() { + let document = parse_document(&many_links(3)).unwrap(); + let small = encode_document(&document, BinaryLinoOptions::default()).unwrap(); + assert!(small.last_address().unwrap() <= 255); + assert_eq!(small.min_width, 1); + + let document = parse_document(&many_links(400)).unwrap(); + let uniform = encode_document(&document, BinaryLinoOptions::default()).unwrap(); + assert!(uniform.last_address().unwrap() > 255); + assert_eq!(uniform.min_width, 2); + let uniform_bytes = uniform.to_bytes().unwrap(); + let header = 1 + 2; // header byte + two-byte LEB128 count + assert_eq!(uniform_bytes.len(), header + uniform.doublets.len() * 4); + + let progressive = encode_document( + &document, + BinaryLinoOptions::default().with_progressive_widths(true), + ) + .unwrap(); + assert_eq!(progressive.min_width, 1); + let progressive_bytes = progressive.to_bytes().unwrap(); + assert!(progressive_bytes.len() < uniform_bytes.len()); + let limits = DecodeLimits::default(); + for bytes in [uniform_bytes, progressive_bytes] { + let packet = LinksPacket::from_bytes(&bytes, &limits).unwrap(); + assert_eq!(decode_document(&packet, &limits).unwrap(), document); + } +} + +#[test] +fn large_external_values_raise_the_minimum_width() { + let options = BinaryLinoOptions::default() + .with_external_references(true) + .with_progressive_widths(true); + let packet = encode_document(&parse_document("(1000 1)").unwrap(), options).unwrap(); + assert_eq!(packet.min_width, 2); + let packet = encode_document(&parse_document("(100 1)").unwrap(), options).unwrap(); + assert_eq!(packet.min_width, 1); + // Beyond 63 bits the number falls back to in-band unary links. + let packet = + encode_document(&parse_document("18446744073709551615").unwrap(), options).unwrap(); + assert!(packet.doublets.len() > 60); +} + +#[test] +fn hand_built_packets_decode() { + // One doublet (One One) is 2^1 in unary, so `(Number 7)` is the number 2. + let packet = LinksPacket { + min_width: 1, + doublets: vec![ + (Reference::Internal(1), Reference::Internal(1)), + (Reference::Internal(2), Reference::Internal(6)), + (Reference::Internal(7), Reference::NULL), + (Reference::Internal(4), Reference::Internal(8)), + ], + ..LinksPacket::default() + }; + let bytes = packet.to_bytes().unwrap(); + assert_eq!(hex(&bytes), "10 04 01 01 02 06 07 00 04 08"); + let protocol = BinaryLinoProtocol::new(); + assert_eq!(format_document(&protocol.decode(&bytes).unwrap()), "2"); +} + +fn expect_error(bytes: &[u8], limits: DecodeLimits) -> ProtocolError { + let protocol = BinaryLinoProtocol { + limits, + ..BinaryLinoProtocol::default() + }; + protocol.decode(bytes).expect_err("must be rejected") +} + +#[test] +fn malformed_packets_are_rejected() { + let limits = DecodeLimits::default(); + let malformed = |bytes: &[u8]| { + assert!( + matches!(expect_error(bytes, limits), ProtocolError::Malformed(_)), + "{}", + hex(bytes) + ); + }; + malformed(&[]); + malformed(&[0x20, 0x00]); // unknown version nibble + malformed(&[0x10, 0x01, 0x06, 0x00]); // refers to itself + malformed(&[0x10, 0x01, 0x07, 0x00]); // refers forward + malformed(&[0x10, 0x02, 0x00]); // truncated + malformed(&[0x10, 0x00, 0x00]); // trailing byte + malformed(&[ + 0x10, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + ]); // LEB128 overflow + malformed(&[0x10, 0x01, 0x01, 0x01]); // root is not a list + malformed(&[0x10, 0x02, 0x00, 0x02, 0x04, 0x06]); // marker 2 used as a value + malformed(&[0x10, 0x01, 0x04, 0x02]); // a bare marker in a list chain + // (String (0x110000)) is not a valid code point. + let invalid = LinksPacket { + external_references: true, + min_width: 4, + doublets: vec![ + (Reference::External(0x11_0000), Reference::NULL), + (Reference::Internal(3), Reference::Internal(6)), + (Reference::Internal(7), Reference::NULL), + (Reference::Internal(4), Reference::Internal(8)), + ], + ..LinksPacket::default() + }; + malformed(&invalid.to_bytes().unwrap()); +} + +#[test] +fn hostile_packets_hit_limits() { + let limits = DecodeLimits { + max_links: 8, + ..DecodeLimits::default() + }; + assert!(matches!( + expect_error(&[0x10, 0x09], limits), + ProtocolError::LimitExceeded(_) + )); + + // A doubling chain `d(k) = (d(k-1) d(k-1))` expands to 2^k nodes. + let mut doublets = vec![(Reference::NULL, Reference::NULL)]; + for address in 6..60u64 { + doublets.push((Reference::Internal(address), Reference::Internal(address))); + } + let last = 5 + doublets.len() as u64; + doublets.push((Reference::Internal(last), Reference::NULL)); + doublets.push((Reference::Internal(4), Reference::Internal(last + 1))); + let bomb = LinksPacket { + min_width: 1, + doublets, + ..LinksPacket::default() + }; + let small_budget = DecodeLimits { + max_nodes: 1 << 12, + ..DecodeLimits::default() + }; + assert!(matches!( + expect_error(&bomb.to_bytes().unwrap(), small_budget), + ProtocolError::LimitExceeded(_) + )); + + let deep = format!("{}x{}", "(y ".repeat(40), ")".repeat(40)); + let bytes = BinaryLinoProtocol::new() + .encode(&parse_document(&deep).unwrap()) + .unwrap(); + let shallow = DecodeLimits { + max_depth: 10, + ..DecodeLimits::default() + }; + assert!(matches!( + expect_error(&bytes, shallow), + ProtocolError::LimitExceeded(_) + )); +} + +#[test] +fn text_messages_are_dot_stuffed() { + let document = parse_document(".dot\n'first\n.second'\n(a b)").unwrap(); + let mut wire = Vec::new(); + TextLinoProtocol::new() + .write_document(&mut wire, &document) + .unwrap(); + let wire_text = String::from_utf8(wire.clone()).unwrap(); + assert_eq!(wire_text, "..dot\n'first\n..second'\na b\n.\n"); + let mut reader = Cursor::new(wire); + let received = TextLinoProtocol::new() + .read_document(&mut reader) + .unwrap() + .unwrap(); + assert_eq!(received, document); + assert!(TextLinoProtocol::new() + .read_document(&mut reader) + .unwrap() + .is_none()); +} + +#[test] +fn text_messages_accept_crlf_and_reject_truncation() { + let mut reader = Cursor::new(b"() ((1 1))\r\n.\r\n".to_vec()); + let document = TextLinoProtocol::new() + .read_document(&mut reader) + .unwrap() + .unwrap(); + assert_eq!(format_document(&document), "() ((1 1))"); + + let mut empty = Cursor::new(b".\n".to_vec()); + assert_eq!( + TextLinoProtocol::new().read_document(&mut empty).unwrap(), + Some(Vec::new()) + ); + + let mut truncated = Cursor::new(b"() ((1 1))\n".to_vec()); + assert!(matches!( + TextLinoProtocol::new().read_document(&mut truncated), + Err(ProtocolError::Malformed(_)) + )); + + let tiny = TextLinoProtocol { + limits: DecodeLimits { + max_text_bytes: 8, + ..DecodeLimits::default() + }, + }; + let mut long = Cursor::new(format!("{}\n.\n", "a".repeat(100)).into_bytes()); + assert!(matches!( + tiny.read_document(&mut long), + Err(ProtocolError::LimitExceeded(_)) + )); +} + +#[test] +fn protocols_are_detected_per_message() { + let document = parse_document("() ((1 1))").unwrap(); + let options = BinaryLinoOptions::default() + .with_external_references(true) + .with_sequences(true); + let mut wire = Vec::new(); + TextLinoProtocol::new() + .write_document(&mut wire, &document) + .unwrap(); + BinaryLinoProtocol::with_options(options) + .write_document(&mut wire, &document) + .unwrap(); + TextLinoProtocol::new() + .write_document(&mut wire, &document) + .unwrap(); + + let limits = DecodeLimits::default(); + let mut reader = Cursor::new(wire); + let formats = std::iter::from_fn(|| read_any_document(&mut reader, &limits).unwrap()) + .map(|(received, format)| { + assert_eq!(received, document); + format + }) + .collect::>(); + assert_eq!( + formats, + vec![ + MessageFormat::Text, + MessageFormat::Binary(options), + MessageFormat::Text + ] + ); +} + +#[test] +fn every_short_reference_over_delimiters_round_trips() { + let alphabet = ['\'', '"', '`', 'a', ' ', '(', ')', ':']; + let mut stack = vec![String::new()]; + while let Some(reference) = stack.pop() { + if !reference.is_empty() { + let text = format_reference(&reference); + let document = parse_document(&text) + .unwrap_or_else(|error| panic!("{reference:?} as {text:?}: {error}")); + assert_eq!( + document, + vec![links_notation::LiNo::Ref(reference.clone())], + "{reference:?} as {text:?}" + ); + } + if reference.chars().count() < 5 { + for character in alphabet { + stack.push(format!("{reference}{character}")); + } + } + } +} From df5625859ba4c2197d1135f30a4933d252e29db9 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 17:56:50 +0000 Subject: [PATCH 03/44] feat(rust): serve and query links over TCP with either protocol LinksServer detects the protocol of every message and replies in kind, executing requests serially against the store; LinksClient runs substitution queries over any LinoProtocol. Replies mirror clink --changes and the empty request reads the whole store. --- rust/src/protocol/server.rs | 68 ++++++--- rust/tests/protocol_tcp_tests.rs | 243 +++++++++++++++++++++++++++++++ 2 files changed, 291 insertions(+), 20 deletions(-) create mode 100644 rust/tests/protocol_tcp_tests.rs diff --git a/rust/src/protocol/server.rs b/rust/src/protocol/server.rs index 3f406c7..25df0bd 100644 --- a/rust/src/protocol/server.rs +++ b/rust/src/protocol/server.rs @@ -7,10 +7,11 @@ //! same protocol, so text and binary clients can share one server. use super::error::{ProtocolError, ProtocolResult}; -use super::format::{format_document, parse_document}; +use super::format::format_document; use super::mapping::LinoDocument; use super::packet::DecodeLimits; use super::protocols::{read_any_document, MessageFormat}; +use crate::link::Link; use crate::named_type_links::NamedTypeLinks; use crate::query_processor::QueryProcessor; use links_notation::LiNo; @@ -271,25 +272,52 @@ fn try_execute_request( processor: &QueryProcessor, document: &[LiNo], ) -> anyhow::Result { - let lines = if document.is_empty() { - storage.lino_lines()? - } else { - let changes = processor.process_query(storage, &format_document(document))?; - if !changes.is_empty() { - storage.save()?; - } - changes - .iter() - .map(|(before, after)| { - let before = before.map(|link| storage.format_lino(&link)).transpose()?; - let after = after.map(|link| storage.format_lino(&link)).transpose()?; - Ok(format!( - "({}) ({})", - before.unwrap_or_default(), - after.unwrap_or_default() - )) + if document.is_empty() { + let mut links = storage.all_links(); + links.sort_by_key(|link| link.index); + return links.iter().map(|link| link_lino(storage, link)).collect(); + } + let changes = processor.process_query(storage, &format_document(document))?; + if !changes.is_empty() { + storage.save()?; + } + changes + .iter() + .map(|(before, after)| { + Ok(LiNo::Link { + id: None, + values: vec![ + change_side(storage, before.as_ref())?, + change_side(storage, after.as_ref())?, + ], }) - .collect::>>()? + }) + .collect() +} + +/// `(index: source target)`, naming every reference that has a name. +fn link_lino(storage: &mut S, link: &Link) -> anyhow::Result> { + Ok(LiNo::Link { + id: Some(reference_name(storage, link.index)?), + values: vec![ + LiNo::Ref(reference_name(storage, link.source)?), + LiNo::Ref(reference_name(storage, link.target)?), + ], + }) +} + +/// `()` for a missing side of a change, `((index: source target))` otherwise. +fn change_side( + storage: &mut S, + link: Option<&Link>, +) -> anyhow::Result> { + let values = match link { + Some(link) => vec![link_lino(storage, link)?], + None => Vec::new(), }; - Ok(parse_document(&lines.join("\n"))?) + Ok(LiNo::Link { id: None, values }) +} + +fn reference_name(storage: &mut S, id: u32) -> anyhow::Result { + Ok(storage.get_name(id)?.unwrap_or_else(|| id.to_string())) } diff --git a/rust/tests/protocol_tcp_tests.rs b/rust/tests/protocol_tcp_tests.rs new file mode 100644 index 0000000..048c66b --- /dev/null +++ b/rust/tests/protocol_tcp_tests.rs @@ -0,0 +1,243 @@ +//! LiNo substitution operations over TCP (issue #105). + +use link_cli::protocol::{ + AcceptedProtocols, BinaryLinoOptions, BinaryLinoProtocol, LinksClient, LinksServer, + LinoProtocol, ProtocolError, ServerOptions, TextLinoProtocol, +}; +use link_cli::{LinkStorage, NamedTypesDecorator}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{SocketAddr, TcpStream}; +use std::thread::{self, JoinHandle}; +use std::time::Duration; +use tempfile::NamedTempFile; + +struct RunningServer { + address: SocketAddr, + shutdown: link_cli::protocol::ShutdownHandle, + thread: Option>, + _database: NamedTempFile, +} + +impl RunningServer { + fn start(options: ServerOptions, named: bool) -> Self { + let database = NamedTempFile::new().unwrap(); + let path = database.path().to_path_buf(); + let server = LinksServer::bind("127.0.0.1:0", options).unwrap(); + let address = server.local_addr().unwrap(); + let shutdown = server.shutdown_handle().unwrap(); + let thread = thread::spawn(move || { + if named { + let mut storage = NamedTypesDecorator::new(&path, false).unwrap(); + server.serve(&mut storage).unwrap(); + } else { + let mut storage = LinkStorage::new(&path, false).unwrap(); + server.serve(&mut storage).unwrap(); + } + }); + Self { + address, + shutdown, + thread: Some(thread), + _database: database, + } + } + + fn client(&self, protocol: impl LinoProtocol + 'static) -> LinksClient { + LinksClient::connect(self.address, protocol).unwrap() + } +} + +impl Drop for RunningServer { + fn drop(&mut self) { + self.shutdown.shutdown(); + if let Some(thread) = self.thread.take() { + thread.join().unwrap(); + } + } +} + +fn protocols() -> Vec> { + let mut protocols: Vec> = vec![Box::new(TextLinoProtocol::new())]; + for external_references in [false, true] { + for sequences in [false, true] { + for progressive_widths in [false, true] { + protocols.push(Box::new(BinaryLinoProtocol::with_options( + BinaryLinoOptions { + external_references, + sequences, + progressive_widths, + }, + ))); + } + } + } + protocols +} + +#[test] +fn crud_works_over_every_protocol() { + for protocol in protocols() { + let name = format!("{protocol:?}"); + let server = RunningServer::start(ServerOptions::default(), false); + let mut client = server.client(protocol); + + // Create. + assert_eq!(client.query_text("() ((1 1))").unwrap(), "() ((1: 1 1))"); + assert_eq!(client.query_text("() ((2 2))").unwrap(), "() ((2: 2 2))"); + // Read. + assert_eq!( + client.query_text("").unwrap(), + "(1: 1 1)\n(2: 2 2)", + "{name}" + ); + assert_eq!( + client.query_text("((1: 1 1)) ((1: 1 1))").unwrap(), + "((1: 1 1)) ((1: 1 1))", + "{name}" + ); + // Update. + assert_eq!( + client.query_text("((1: 1 1)) ((1: 1 2))").unwrap(), + "((1: 1 1)) ((1: 1 2))", + "{name}" + ); + // Delete. + assert_eq!( + client.query_text("((1: 1 2)) ()").unwrap(), + "((1: 1 2)) ()", + "{name}" + ); + assert_eq!(client.query_text("").unwrap(), "(2: 2 2)", "{name}"); + } +} + +#[test] +fn text_and_binary_replies_are_identical_documents() { + let options = ServerOptions { + auto_create_missing_references: true, + ..ServerOptions::default() + }; + let server = RunningServer::start(options, true); + let mut text = server.client(TextLinoProtocol::new()); + let mut binary = server.client(BinaryLinoProtocol::with_options( + BinaryLinoOptions::default() + .with_external_references(true) + .with_sequences(true), + )); + text.query("() ((child: father mother))").unwrap(); + binary.query("() (('two words': 'it''s' \"x\"))").unwrap(); + let from_text = text.query("").unwrap(); + let from_binary = binary.query("").unwrap(); + assert_eq!(from_text, from_binary); + let listing = text.query_text("").unwrap(); + assert!(listing.contains("(child: father mother)"), "{listing}"); + assert!(listing.contains("('two words': \"it's\" x)"), "{listing}"); +} + +#[test] +fn query_errors_come_back_as_remote_errors() { + let server = RunningServer::start(ServerOptions::default(), false); + for protocol in protocols() { + let mut client = server.client(protocol); + match client.query("((99: 1 1)) ()") { + Err(ProtocolError::Remote(message)) => assert!(!message.is_empty()), + other => panic!("expected a remote error, got {other:?}"), + } + // The connection stays usable after an error reply. + assert_eq!(client.query_text("").unwrap(), ""); + } +} + +#[test] +fn a_raw_text_session_works_with_crlf() { + let server = RunningServer::start(ServerOptions::default(), false); + let mut stream = TcpStream::connect(server.address).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + stream.write_all(b"() ((1 1))\r\n.\r\n").unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut reply = String::new(); + reader.read_line(&mut reply).unwrap(); + assert_eq!(reply, "() ((1: 1 1))\n"); + reply.clear(); + reader.read_line(&mut reply).unwrap(); + assert_eq!(reply, ".\n"); + + stream.write_all(b".\n").unwrap(); + reply.clear(); + reader.read_line(&mut reply).unwrap(); + assert_eq!(reply, "(1: 1 1)\n"); +} + +#[test] +fn malformed_messages_get_an_error_and_the_connection_closes() { + let server = RunningServer::start(ServerOptions::default(), false); + let mut stream = TcpStream::connect(server.address).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + // A binary header announcing more links than the default limit allows. + stream + .write_all(&[0x10, 0xff, 0xff, 0xff, 0xff, 0x0f]) + .unwrap(); + let mut reply = String::new(); + stream.read_to_string(&mut reply).unwrap(); + assert!(reply.starts_with("(error: "), "{reply}"); + assert!(reply.ends_with("\n.\n"), "{reply}"); +} + +#[test] +fn servers_can_restrict_the_accepted_protocol() { + let options = ServerOptions { + accept: AcceptedProtocols::Binary, + ..ServerOptions::default() + }; + let server = RunningServer::start(options, false); + let mut text = server.client(TextLinoProtocol::new()); + assert!(matches!( + text.query("() ((1 1))"), + Err(ProtocolError::Remote(_)) + )); + let mut binary = server.client(BinaryLinoProtocol::new()); + assert_eq!(binary.query_text("() ((1 1))").unwrap(), "() ((1: 1 1))"); +} + +#[test] +fn concurrent_clients_share_one_store() { + let server = RunningServer::start(ServerOptions::default(), true); + let address = server.address; + let workers: Vec<_> = (0..4) + .map(|worker| { + thread::spawn(move || { + let mut client = if worker % 2 == 0 { + LinksClient::connect(address, TextLinoProtocol::new()).unwrap() + } else { + LinksClient::connect(address, BinaryLinoProtocol::new()).unwrap() + }; + for item in 0..5 { + let name = format!("w{worker}i{item}"); + let reply = client.query_text(&format!("() (({name}: {name} {name}))")); + let link = format!("({name}: {name} {name})"); + assert_eq!(reply.unwrap(), format!("({link}) ({link})")); + } + }) + }) + .collect(); + for worker in workers { + worker.join().unwrap(); + } + let mut client = server.client(TextLinoProtocol::new()); + let listing = client.query_text("").unwrap(); + assert_eq!(listing.lines().count(), 20, "{listing}"); +} + +#[test] +fn shutdown_stops_the_server() { + let mut server = RunningServer::start(ServerOptions::default(), false); + let mut client = server.client(TextLinoProtocol::new()); + client.query("() ((1 1))").unwrap(); + server.shutdown.shutdown(); + server.thread.take().unwrap().join().unwrap(); + assert!(client.query("").is_err()); +} From 84b081e9260383c756d78845262842c5fa4fe7b8 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 17:59:23 +0000 Subject: [PATCH 04/44] feat(rust): add clink --serve and --connect clink --serve exposes a database over TCP, accepting text and binary messages; clink --connect sends a query with the chosen protocol and binary options and prints the reply. --- ...20261004_120000_issue_105_tcp_protocols.md | 5 + rust/src/cli.rs | 69 ++++++++ rust/src/main.rs | 85 +++++++++ rust/tests/cli_tcp_tests.rs | 164 ++++++++++++++++++ 4 files changed, 323 insertions(+) create mode 100644 rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md create mode 100644 rust/tests/cli_tcp_tests.rs diff --git a/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md b/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md new file mode 100644 index 0000000..8ce1d3b --- /dev/null +++ b/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md @@ -0,0 +1,5 @@ +--- +bump: minor +--- + +Added LiNo substitution operations over TCP (issue #105): a dot-stuffed UTF-8 `TextLinoProtocol` and an interchangeable `BinaryLinoProtocol` whose packets carry the number of links and grow reference widths from 8 to 16, 32 and 64 bits, with optional external (Hybrid) references, a variable-length sequence section and progressive widths. `LinksServer`/`LinksClient` and the new `clink --serve`/`--connect` options (`--protocol`, `--external-references`, `--sequences`, `--progressive-widths`) run create, read, update and delete queries over either protocol. diff --git a/rust/src/cli.rs b/rust/src/cli.rs index ffa028f..8fd168b 100644 --- a/rust/src/cli.rs +++ b/rust/src/cli.rs @@ -38,6 +38,12 @@ pub struct Cli { pub list_branches: bool, pub list_tags: bool, pub show_log: bool, + pub serve: Option, + pub connect: Option, + pub protocol: Option, + pub external_references: bool, + pub sequences: bool, + pub progressive_widths: bool, } impl Default for Cli { @@ -73,6 +79,12 @@ impl Default for Cli { list_branches: false, list_tags: false, show_log: false, + serve: None, + connect: None, + protocol: None, + external_references: false, + sequences: false, + progressive_widths: false, } } } @@ -263,6 +275,30 @@ impl Cli { cli.show_log = parse_bool("--log", value)?; continue; } + if let Some(value) = inline_value(&arg, &["--serve"]) { + cli.serve = Some(value.to_string()); + continue; + } + if let Some(value) = inline_value(&arg, &["--connect"]) { + cli.connect = Some(value.to_string()); + continue; + } + if let Some(value) = inline_value(&arg, &["--protocol"]) { + cli.protocol = Some(value.to_string()); + continue; + } + if let Some(value) = inline_value(&arg, &["--external-references"]) { + cli.external_references = parse_bool("--external-references", value)?; + continue; + } + if let Some(value) = inline_value(&arg, &["--sequences"]) { + cli.sequences = parse_bool("--sequences", value)?; + continue; + } + if let Some(value) = inline_value(&arg, &["--progressive-widths"]) { + cli.progressive_widths = parse_bool("--progressive-widths", value)?; + continue; + } match arg.as_str() { "-h" | "--help" => return Ok(CliCommand::Help), @@ -356,6 +392,24 @@ impl Cli { "--log" => { cli.show_log = next_bool_value(&mut args, true)?; } + "--serve" => { + cli.serve = Some(next_value(&mut args, &arg)?); + } + "--connect" => { + cli.connect = Some(next_value(&mut args, &arg)?); + } + "--protocol" => { + cli.protocol = Some(next_value(&mut args, &arg)?); + } + "--external-references" => { + cli.external_references = next_bool_value(&mut args, true)?; + } + "--sequences" => { + cli.sequences = next_bool_value(&mut args, true)?; + } + "--progressive-widths" => { + cli.progressive_widths = next_bool_value(&mut args, true)?; + } "--" => { for value in args.by_ref() { set_positional_query(&mut cli, value)?; @@ -451,6 +505,21 @@ impl Cli { " List version-control tags and exit\n", " --log\n", " Print the transitions log and exit (implies --transactions)\n", + " --serve \n", + " Serve the database over TCP (e.g. 127.0.0.1:7878) until stopped;\n", + " each message may use the text or the binary LiNo protocol\n", + " --connect \n", + " Send the query to a clink server instead of opening a database;\n", + " an empty query reads every link\n", + " --protocol \n", + " 'text' or 'binary' for --connect (default: text); 'text', 'binary'\n", + " or 'any' for --serve (default: any)\n", + " --external-references\n", + " Binary protocol: send numbers and characters as external references\n", + " --sequences\n", + " Binary protocol: send lists in the variable-length sequence section\n", + " --progressive-widths\n", + " Binary protocol: widen references link by link instead of uniformly\n", " -h, --help\n", " Print help\n", " -V, --version\n", diff --git a/rust/src/main.rs b/rust/src/main.rs index d045bfc..35d6691 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -6,6 +6,10 @@ use anyhow::{anyhow, bail, Result}; use link_cli::cli::{Cli, CliCommand}; use link_cli::import_lino_file; +use link_cli::protocol::{ + AcceptedProtocols, BinaryLinoOptions, BinaryLinoProtocol, LinksClient, LinksServer, + LinoProtocol, ServerOptions, TextLinoProtocol, +}; use link_cli::{ make_triggers_database_filename, CommitMode, LogRetentionPolicy, NamedTypeLinks, NamedTypesDecorator, PersistentTransformationDecorator, PersistentTransformationKind, @@ -30,6 +34,13 @@ fn main() -> Result<()> { bail!("Only one of --always, --once, or --never can be used at a time."); } + if cli.serve.is_some() && cli.connect.is_some() { + bail!("--serve and --connect cannot be used together."); + } + if let Some(address) = &cli.connect { + return run_client(&cli, address); + } + let vc_requested = cli.vc_requested(); let transactions_requested = cli.transactions_requested(); @@ -97,6 +108,9 @@ where { if !persistent_transformations_enabled(cli) { let mut storage = storage; + if let Some(address) = &cli.serve { + return run_server(cli, address, &mut storage); + } run_query_pipeline(cli, &mut storage)?; storage.save()?; return Ok(()); @@ -110,6 +124,9 @@ where let mut storage = PersistentTransformationDecorator::new(storage, trigger_store, cli.trace) .with_auto_create_missing_references(cli.auto_create_missing_references); + if let Some(address) = &cli.serve { + return run_server(cli, address, &mut storage); + } run_query_pipeline_with(cli, &mut storage, run_trigger_command)?; storage.save()?; Ok(()) @@ -397,3 +414,71 @@ where Ok(()) } + +/// The binary options selected by `--external-references`, `--sequences` and +/// `--progressive-widths`. +fn binary_options(cli: &Cli) -> BinaryLinoOptions { + BinaryLinoOptions::default() + .with_external_references(cli.external_references) + .with_sequences(cli.sequences) + .with_progressive_widths(cli.progressive_widths) +} + +/// `--connect`: sends the query to a server and prints the reply. +/// +/// Any binary option implies `--protocol binary`. +fn run_client(cli: &Cli, address: &str) -> Result<()> { + let options = binary_options(cli); + let protocol: Box = match cli.protocol.as_deref().map(str::trim) { + None if options == BinaryLinoOptions::default() => Box::new(TextLinoProtocol::new()), + None => Box::new(BinaryLinoProtocol::with_options(options)), + Some(name) if name.eq_ignore_ascii_case("text") => { + if options != BinaryLinoOptions::default() { + bail!("Binary protocol options require --protocol binary."); + } + Box::new(TextLinoProtocol::new()) + } + Some(name) if name.eq_ignore_ascii_case("binary") => { + Box::new(BinaryLinoProtocol::with_options(options)) + } + Some(other) => bail!("Invalid --protocol value '{other}'. Use 'text' or 'binary'."), + }; + let query = cli.query.as_deref().or(cli.query_arg.as_deref()); + let mut client = LinksClient::connect(address, protocol)?; + let reply = client.query_text(query.unwrap_or_default())?; + if !reply.is_empty() { + println!("{reply}"); + } + Ok(()) +} + +/// `--serve`: serves `storage` until the process is stopped. +fn run_server(cli: &Cli, address: &str, storage: &mut S) -> Result<()> +where + S: NamedTypeLinks, +{ + if cli.query.is_some() || cli.query_arg.is_some() { + bail!("--serve does not take a query; send queries with --connect."); + } + let accept = match cli.protocol.as_deref().map(str::trim) { + None => AcceptedProtocols::Any, + Some(name) if name.eq_ignore_ascii_case("any") => AcceptedProtocols::Any, + Some(name) if name.eq_ignore_ascii_case("text") => AcceptedProtocols::Text, + Some(name) if name.eq_ignore_ascii_case("binary") => AcceptedProtocols::Binary, + Some(other) => { + bail!("Invalid --protocol value '{other}'. Use 'any', 'text' or 'binary'.") + } + }; + let options = ServerOptions { + trace: cli.trace, + auto_create_missing_references: cli.auto_create_missing_references, + accept, + ..ServerOptions::default() + }; + let server = LinksServer::bind(address, options)?; + println!("clink server listening on {}", server.local_addr()?); + std::io::Write::flush(&mut std::io::stdout())?; + server.serve(storage)?; + storage.save()?; + Ok(()) +} diff --git a/rust/tests/cli_tcp_tests.rs b/rust/tests/cli_tcp_tests.rs new file mode 100644 index 0000000..c11f059 --- /dev/null +++ b/rust/tests/cli_tcp_tests.rs @@ -0,0 +1,164 @@ +//! `clink --serve` and `clink --connect` end to end (issue #105). + +use link_cli::cli::{Cli, CliCommand}; +use std::io::{BufRead, BufReader}; +use std::process::{Child, Command, Output, Stdio}; +use tempfile::TempDir; + +fn parse_run(args: &[&str]) -> Cli { + match Cli::parse_from(args).expect("CLI arguments should parse") { + CliCommand::Run(cli) => *cli, + other => panic!("expected run command, got {other:?}"), + } +} + +#[test] +fn parses_tcp_options() { + let cli = parse_run(&[ + "clink", + "--serve", + "127.0.0.1:7878", + "--protocol=binary", + "--external-references", + "--sequences", + "true", + "--progressive-widths=on", + ]); + assert_eq!(cli.serve.as_deref(), Some("127.0.0.1:7878")); + assert_eq!(cli.protocol.as_deref(), Some("binary")); + assert!(cli.external_references); + assert!(cli.sequences); + assert!(cli.progressive_widths); + + let cli = parse_run(&["clink", "--connect=localhost:1", "() ((1 1))"]); + assert_eq!(cli.connect.as_deref(), Some("localhost:1")); + assert_eq!(cli.query_arg.as_deref(), Some("() ((1 1))")); + assert!(!cli.external_references && !cli.sequences && !cli.progressive_widths); + assert!(Cli::help_text().contains("--serve ")); + assert!(Cli::help_text().contains("--connect ")); +} + +struct Server { + child: Child, + address: String, +} + +impl Server { + fn start(directory: &TempDir, extra: &[&str]) -> Self { + let database = directory.path().join("served.links"); + let mut child = Command::new(env!("CARGO_BIN_EXE_clink")) + .arg("--db") + .arg(&database) + .args(["--serve", "127.0.0.1:0"]) + .args(extra) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let mut line = String::new(); + BufReader::new(child.stdout.take().unwrap()) + .read_line(&mut line) + .unwrap(); + let address = line + .trim() + .strip_prefix("clink server listening on ") + .unwrap_or_else(|| panic!("unexpected banner {line:?}")) + .to_string(); + Self { child, address } + } + + fn connect(&self, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_clink")) + .args(["--connect", &self.address]) + .args(args) + .output() + .unwrap() + } + + fn stdout(&self, args: &[&str]) -> String { + let output = self.connect(args); + assert!( + output.status.success(), + "{args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + } +} + +impl Drop for Server { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +#[test] +fn clients_query_a_served_database_over_both_protocols() { + let directory = TempDir::new().unwrap(); + let server = Server::start(&directory, &[]); + + assert_eq!(server.stdout(&["() ((1 1))"]), "() ((1: 1 1))\n"); + assert_eq!( + server.stdout(&["--protocol", "binary", "() ((2 2))"]), + "() ((2: 2 2))\n" + ); + assert_eq!( + server.stdout(&[ + "--external-references", + "--sequences", + "((1: 1 1)) ((1: 1 2))" + ]), + "((1: 1 1)) ((1: 1 2))\n" + ); + assert_eq!(server.stdout(&[]), "(1: 1 2)\n(2: 2 2)\n"); + assert_eq!( + server + .stdout(&["--protocol", "binary", "((2: 2 2)) ()"]) + .lines() + .count(), + 2 + ); + + let failure = server.connect(&["((99: 1 1)) ()"]); + assert!(!failure.status.success()); + assert!(String::from_utf8_lossy(&failure.stderr).contains("server error")); + + let conflicting = server.connect(&["--protocol", "text", "--sequences", ""]); + assert!(!conflicting.status.success()); +} + +#[test] +fn served_changes_are_persisted() { + let directory = TempDir::new().unwrap(); + { + let server = Server::start(&directory, &["--auto-create-missing-references"]); + assert_eq!( + server + .stdout(&["() ((child: father mother))"]) + .lines() + .last(), + Some("() ((child: father mother))") + ); + } + let output = Command::new(env!("CARGO_BIN_EXE_clink")) + .arg("--db") + .arg(directory.path().join("served.links")) + .arg("--after") + .output() + .unwrap(); + let listing = String::from_utf8(output.stdout).unwrap(); + assert!(listing.contains("(child: father mother)"), "{listing}"); +} + +#[test] +fn serve_rejects_a_query() { + let directory = TempDir::new().unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_clink")) + .arg("--db") + .arg(directory.path().join("x.links")) + .args(["--serve", "127.0.0.1:0", "() ((1 1))"]) + .output() + .unwrap(); + assert!(!output.status.success()); +} From 30a7f77fa1c9f9bfbd6e0499ef44fd024fe5373f Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 18:12:40 +0000 Subject: [PATCH 05/44] feat(csharp): add text and binary LiNo protocols, TCP server and client Port of the Rust protocol module: LinksPacket codec, LiNo <-> packet mapping with optional external references, sequence section and progressive widths, dot-stuffed text framing, protocol detection and a LinksServer/LinksClient pair that run substitution queries over TCP. --- .../Protocol/LinksClient.cs | 73 +++ .../Protocol/LinksPacket.cs | 450 +++++++++++++++ .../Protocol/LinksServer.cs | 307 +++++++++++ .../Protocol/LinoFormat.cs | 142 +++++ .../Protocol/LinoMapping.cs | 521 ++++++++++++++++++ .../Protocol/LinoProtocolException.cs | 67 +++ .../Protocol/LinoProtocols.cs | 263 +++++++++ .../Protocol/LinoStreamReader.cs | 96 ++++ 8 files changed, 1919 insertions(+) create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksClient.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoMapping.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocolException.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoStreamReader.cs diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksClient.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksClient.cs new file mode 100644 index 0000000..0f9d574 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksClient.cs @@ -0,0 +1,73 @@ +using System.Net.Sockets; + +using LinoLink = Link.Foundation.Links.Notation.Link; + +namespace Foundation.Data.Doublets.Cli.Protocol; + +///

Sends substitution queries to a LiNo server over any . +public sealed class LinksClient : IDisposable +{ + private readonly TcpClient _client; + private readonly LinoConnection _connection; + + private LinksClient(TcpClient client, ILinoProtocol protocol) + { + _client = client; + _connection = new LinoConnection(client.GetStream(), protocol); + } + + /// The protocol used for every message. + public ILinoProtocol Protocol => _connection.Protocol; + + /// Connects to : using . + public static LinksClient Connect(string host, int port, ILinoProtocol protocol) + { + ArgumentNullException.ThrowIfNull(protocol); + var client = new TcpClient(); + try + { + client.Connect(host, port); + client.NoDelay = true; + return new LinksClient(client, protocol); + } + catch (SocketException error) + { + client.Dispose(); + throw new LinoProtocolException(LinoProtocolErrorKind.Io, error.Message, error); + } + } + + /// Connects to an address written as host:port. + public static LinksClient Connect(string endPoint, ILinoProtocol protocol) + { + var (host, port) = LinksServer.ParseEndPoint(endPoint); + return Connect(host, port, protocol); + } + + /// + /// Sends a parsed document and returns the reply document. An + /// (error: 'message') reply raises a error. + /// + public IReadOnlyList Request(IReadOnlyList document) + { + var reply = _connection.Request(document); + if (LinksServer.ErrorMessage(reply) is { } message) + { + throw new LinoProtocolException(LinoProtocolErrorKind.Remote, message); + } + return reply; + } + + /// Runs a LiNo substitution query; the empty query reads every link. + public IReadOnlyList Query(string query) => Request(LinoFormat.ParseDocument(query)); + + /// Like , returning the reply as canonical text. + public string QueryText(string query) => LinoFormat.FormatDocument(Query(query)); + + /// + public void Dispose() + { + _connection.Dispose(); + _client.Dispose(); + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs new file mode 100644 index 0000000..1dfb96a --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs @@ -0,0 +1,450 @@ +using System.Buffers.Binary; + +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// One reference inside a . +/// True for an external value (a number or a code point). +/// The address inside the packet, or the external value. +public readonly record struct PacketReference(bool IsExternal, ulong Value) +{ + /// The null reference. + public static readonly PacketReference Null = Internal(LinksPacket.NullAddress); + + /// An address inside the packet (0 null, 1..5 markers, then links). + public static PacketReference Internal(ulong address) => new(false, address); + + /// An external value, e.g. a number or a Unicode code point. + public static PacketReference External(ulong value) => new(true, value); + + public override string ToString() => IsExternal ? $"External({Value})" : $"Internal({Value})"; +} + +/// Safety limits applied while decoding untrusted input. +public sealed record DecodeLimits +{ + /// Maximum N + M. + public ulong MaxLinks { get; init; } = 1UL << 22; + + /// Maximum total number of references inside all sequences. + public ulong MaxSequenceItems { get; init; } = 1UL << 24; + + /// Maximum number of LiNo nodes a packet may expand to. + public long MaxNodes { get; init; } = 1L << 22; + + /// Maximum LiNo nesting depth. + public int MaxDepth { get; init; } = 1024; + + /// Maximum size of a text message in bytes. + public long MaxTextBytes { get; init; } = 64L << 20; + + /// The default limits. + public static DecodeLimits Default { get; } = new(); +} + +/// +/// The binary links packet: the wire format of the binary LiNo protocol. +/// +/// +/// +/// byte 0 0x10 | flags high nibble 1 = format version 1 +/// bit 0 external references (Hybrid encoding) +/// bit 1 sequence section present +/// bits 2-3 log2 of the minimum width in bytes +/// LEB128 N number of fixed doublets +/// LEB128 M number of sequences (only when bit 1 is set) +/// N times source target one fixed doublet, refs width(a) bytes each +/// M times size ref_1 … ref_n one sequence, size and refs width(a) bytes each +/// +/// Addresses are implicit: 0 is null, 1..5 are the reserved marker points +/// (never transmitted), the fixed doublets occupy 6..6+N and the sequences +/// follow them. Every reference of the link at address a uses +/// width(a) = max(min_width, tier(a)) bytes, little-endian, where tier(a) is +/// the smallest of 1, 2, 4 and 8 bytes able to hold a. With external +/// references enabled the top bit marks a reference as external, exactly like +/// Platform.Data.Hybrid<T>, which halves every internal range. +/// The byte layout is identical to the Rust link_cli::protocol::LinksPacket. +/// +public sealed class LinksPacket +{ + /// The high nibble of the header byte; text never starts with 0x10..0x1F. + public const byte BinaryVersion1 = 0x10; + + /// Null link address. + public const ulong NullAddress = 0; + /// Marker 1: the unary one; 2^k = (2^(k-1) 2^(k-1)). + public const ulong One = 1; + /// Marker 2: (Number unary) is a non-negative integer. + public const ulong Number = 2; + /// Marker 3: (String code points…) is a Unicode string. + public const ulong String = 3; + /// Marker 4: (List elements…) is a list of links. + public const ulong List = 4; + /// Marker 5: (Identified id values…) is a link with an id. + public const ulong Identified = 5; + /// Address of the first transmitted link. + public const ulong FirstLinkAddress = 6; + + private const byte FlagExternalReferences = 0b0001; + private const byte FlagSequences = 0b0010; + private const int WidthShift = 2; + + /// The reference widths, in bytes, that a packet may use. + public static IReadOnlyList Widths { get; } = new byte[] { 1, 2, 4, 8 }; + + /// Header bit 0: references may be external (Hybrid encoding). + public bool ExternalReferences { get; set; } + + /// Header bit 1: the packet carries a sequence section (possibly empty). + public bool SequencesSection { get; set; } + + /// Minimum reference width in bytes: 1, 2, 4 or 8. + public byte MinWidth { get; set; } = 1; + + /// Fixed doublets at addresses 6..6+N. + public List<(PacketReference Source, PacketReference Target)> Doublets { get; } = new(); + + /// Variable-length sequences at addresses 6+N..6+N+M. + public List> Sequences { get; } = new(); + + /// Largest internal address that fits in bytes. + public static ulong InternalCapacity(byte width, bool externalReferences) + { + var bits = width * 8 - (externalReferences ? 1 : 0); + return bits >= 64 ? ulong.MaxValue : (1UL << bits) - 1; + } + + /// Largest external value that fits in bytes. + public static ulong ExternalCapacity(byte width) => (1UL << (width * 8 - 1)) - 1; + + /// Largest unsigned value (a sequence size) that fits in bytes. + public static ulong UnsignedCapacity(byte width) => InternalCapacity(width, false); + + /// The narrowest width able to hold the internal address. + public static byte AddressTier(ulong address, bool externalReferences) + { + foreach (var width in Widths) + { + if (InternalCapacity(width, externalReferences) >= address) + { + return width; + } + } + return 8; + } + + /// Encodes an external value at the way Hybrid<T> does. + public static ulong EncodeExternal(ulong value, byte width) => + value == 0 ? 1UL << (width * 8 - 1) : (0UL - value) & UnsignedCapacity(width); + + /// Decodes a raw value; returns true and the value for externals. + public static bool TryDecodeExternal(ulong raw, byte width, out ulong value) + { + var externalZero = 1UL << (width * 8 - 1); + value = 0; + if (raw == externalZero) + { + return true; + } + if (raw > externalZero) + { + value = (0UL - raw) & UnsignedCapacity(width); + return true; + } + return false; + } + + /// Address of the fixed doublet with zero-based index. + public ulong DoubletAddress(int index) => FirstLinkAddress + (ulong)index; + + /// Address of the sequence with zero-based index. + public ulong SequenceAddress(int index) => FirstLinkAddress + (ulong)Doublets.Count + (ulong)index; + + /// The highest address in the packet, or null for an empty packet. + public ulong? LastAddress + { + get + { + var count = Doublets.Count + Sequences.Count; + return count > 0 ? FirstLinkAddress + (ulong)count - 1 : null; + } + } + + /// Width used by every reference of the link at . + public byte WidthAt(ulong address) => Math.Max(MinWidth, AddressTier(address, ExternalReferences)); + + /// + /// The smallest min_width able to encode the packet; with + /// at least the tier of the last address, so every reference has the same width. + /// + public byte RequiredMinWidth(bool uniform) + { + byte needed = 1; + void Note(ulong address, byte need) + { + if (need > AddressTier(address, ExternalReferences)) + { + needed = Math.Max(needed, need); + } + } + for (var index = 0; index < Doublets.Count; index++) + { + var (source, target) = Doublets[index]; + Note(DoubletAddress(index), Math.Max(ReferenceNeed(source), ReferenceNeed(target))); + } + for (var index = 0; index < Sequences.Count; index++) + { + var items = Sequences[index]; + var need = Widths.FirstOrDefault(width => UnsignedCapacity(width) >= (ulong)items.Count, (byte)8); + foreach (var item in items) + { + need = Math.Max(need, ReferenceNeed(item)); + } + Note(SequenceAddress(index), need); + } + if (uniform && LastAddress is { } last) + { + needed = Math.Max(needed, AddressTier(last, ExternalReferences)); + } + return needed; + } + + private byte ReferenceNeed(PacketReference reference) + { + if (!reference.IsExternal) + { + return 1; + } + if (!ExternalReferences) + { + throw LinoProtocolException.Unencodable("external reference in a packet without external references"); + } + foreach (var width in Widths) + { + if (ExternalCapacity(width) >= reference.Value) + { + return width; + } + } + throw LinoProtocolException.Unencodable($"external value {reference.Value} exceeds 63 bits"); + } + + private byte HeaderByte() + { + var code = Widths.ToList().IndexOf(MinWidth); + if (code < 0) + { + throw LinoProtocolException.Unencodable($"invalid width {MinWidth}"); + } + var header = (byte)(BinaryVersion1 | (code << WidthShift)); + if (ExternalReferences) + { + header |= FlagExternalReferences; + } + if (SequencesSection) + { + header |= FlagSequences; + } + return header; + } + + private ulong RawReference(PacketReference reference, ulong address, byte width) + { + if (!reference.IsExternal) + { + if (reference.Value >= address) + { + throw LinoProtocolException.Unencodable($"link {address} refers forward to {reference.Value}"); + } + return reference.Value; + } + if (!ExternalReferences || reference.Value > ExternalCapacity(width)) + { + throw LinoProtocolException.Unencodable( + $"external value {reference.Value} does not fit {width} byte(s) at link {address}"); + } + return EncodeExternal(reference.Value, width); + } + + /// Serializes the packet. + public byte[] ToBytes() + { + if (!SequencesSection && Sequences.Count > 0) + { + throw LinoProtocolException.Unencodable("sequences in a packet without a sequence section"); + } + var output = new List { HeaderByte() }; + WriteLeb128(output, (ulong)Doublets.Count); + if (SequencesSection) + { + WriteLeb128(output, (ulong)Sequences.Count); + } + for (var index = 0; index < Doublets.Count; index++) + { + var address = DoubletAddress(index); + var width = WidthAt(address); + var (source, target) = Doublets[index]; + WriteRaw(output, RawReference(source, address, width), width); + WriteRaw(output, RawReference(target, address, width), width); + } + for (var index = 0; index < Sequences.Count; index++) + { + var address = SequenceAddress(index); + var width = WidthAt(address); + var items = Sequences[index]; + var size = (ulong)items.Count; + if (size > UnsignedCapacity(width)) + { + throw LinoProtocolException.Unencodable( + $"sequence size {size} does not fit {width} byte(s) at link {address}"); + } + WriteRaw(output, size, width); + foreach (var item in items) + { + WriteRaw(output, RawReference(item, address, width), width); + } + } + return output.ToArray(); + } + + /// Parses a complete packet; trailing bytes are an error. + public static LinksPacket FromBytes(byte[] bytes, DecodeLimits? limits = null) + { + var reader = LinoStreamReader.FromBytes(bytes); + var packet = ReadFrom(reader, limits) ?? throw LinoProtocolException.Malformed("empty input"); + if (!reader.AtEnd) + { + throw LinoProtocolException.Malformed("trailing bytes after the packet"); + } + return packet; + } + + /// Reads one packet; null on a clean end of stream. + public static LinksPacket? ReadFrom(LinoStreamReader reader, DecodeLimits? limits = null) + { + ArgumentNullException.ThrowIfNull(reader); + limits ??= DecodeLimits.Default; + var first = reader.ReadByte(); + if (first < 0) + { + return null; + } + var header = (byte)first; + if ((header & 0xF0) != BinaryVersion1) + { + throw LinoProtocolException.Malformed($"unsupported binary header byte 0x{header:X2}"); + } + var packet = new LinksPacket + { + ExternalReferences = (header & FlagExternalReferences) != 0, + SequencesSection = (header & FlagSequences) != 0, + MinWidth = Widths[(header >> WidthShift) & 0b11], + }; + var doubletCount = ReadLeb128(reader); + var sequenceCount = packet.SequencesSection ? ReadLeb128(reader) : 0; + if (doubletCount > limits.MaxLinks || sequenceCount > limits.MaxLinks - doubletCount) + { + throw LinoProtocolException.Limit( + $"packet declares {doubletCount} + {sequenceCount} links, limit is {limits.MaxLinks}"); + } + for (var index = 0; index < (int)doubletCount; index++) + { + var address = packet.DoubletAddress(index); + var width = packet.WidthAt(address); + var source = packet.ReadReference(reader, address, width); + var target = packet.ReadReference(reader, address, width); + packet.Doublets.Add((source, target)); + } + var itemsLeft = limits.MaxSequenceItems; + for (var index = 0; index < (int)sequenceCount; index++) + { + var address = packet.SequenceAddress(index); + var width = packet.WidthAt(address); + var size = ReadRaw(reader, width); + if (size > itemsLeft) + { + throw LinoProtocolException.Limit($"sequence items exceed the limit of {limits.MaxSequenceItems}"); + } + itemsLeft -= size; + var items = new List((int)Math.Min(size, 4096)); + for (ulong item = 0; item < size; item++) + { + items.Add(packet.ReadReference(reader, address, width)); + } + packet.Sequences.Add(items); + } + return packet; + } + + private PacketReference ReadReference(LinoStreamReader reader, ulong address, byte width) + { + var raw = ReadRaw(reader, width); + if (ExternalReferences && TryDecodeExternal(raw, width, out var value)) + { + return PacketReference.External(value); + } + if (raw >= address) + { + throw LinoProtocolException.Malformed($"link {address} refers to {raw}, which is not an earlier link"); + } + return PacketReference.Internal(raw); + } + + private static void WriteRaw(List output, ulong value, byte width) + { + for (var index = 0; index < width; index++) + { + output.Add((byte)(value >> (8 * index))); + } + } + + private static ulong ReadRaw(LinoStreamReader reader, byte width) + { + Span bytes = stackalloc byte[8]; + bytes.Clear(); + reader.ReadExactly(bytes[..width]); + return BinaryPrimitives.ReadUInt64LittleEndian(bytes); + } + + /// Appends as unsigned LEB128. + public static void WriteLeb128(List output, ulong value) + { + ArgumentNullException.ThrowIfNull(output); + while (true) + { + var current = (byte)(value & 0x7F); + value >>= 7; + if (value == 0) + { + output.Add(current); + return; + } + output.Add((byte)(current | 0x80)); + } + } + + /// Reads an unsigned LEB128 value of at most 64 bits. + public static ulong ReadLeb128(LinoStreamReader reader) + { + ArgumentNullException.ThrowIfNull(reader); + ulong value = 0; + for (var shift = 0; shift < 64; shift += 7) + { + var next = reader.ReadByte(); + if (next < 0) + { + throw LinoProtocolException.Malformed("unexpected end of packet"); + } + var payload = (ulong)(next & 0x7F); + if (shift == 63 && payload > 1) + { + throw LinoProtocolException.Malformed("LEB128 value overflows 64 bits"); + } + value |= payload << shift; + if ((next & 0x80) == 0) + { + return value; + } + } + throw LinoProtocolException.Malformed("LEB128 value overflows 64 bits"); + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs new file mode 100644 index 0000000..0016a03 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs @@ -0,0 +1,307 @@ +using System.Net; +using System.Net.Sockets; +using Platform.Data; +using Platform.Data.Doublets; + +using DoubletLink = Platform.Data.Doublets.Link; +using LinoLink = Link.Foundation.Links.Notation.Link; + +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// Which protocols a server accepts. +public enum AcceptedProtocols +{ + /// Detect the protocol of every message (the default). + Any, + /// Only messages. + Text, + /// Only messages. + Binary, +} + +/// Server configuration. +public sealed record LinksServerOptions +{ + /// Print every request and reply to stderr. + public bool Trace { get; init; } + + /// Passed to . + public bool AutoCreateMissingReferences { get; init; } + + /// Protocols the server answers. + public AcceptedProtocols Accept { get; init; } = AcceptedProtocols.Any; + + /// Limits applied to incoming messages. + public DecodeLimits Limits { get; init; } = DecodeLimits.Default; +} + +/// +/// A TCP server exposing a links store through the LiNo protocols. +/// +/// +/// Each connection gets its own thread that parses and formats messages; +/// requests are executed one at a time under a lock, so the store needs no +/// thread safety of its own. The protocol is detected per message and the +/// reply uses the same protocol, so text and binary clients can share one server. +/// +public sealed class LinksServer : IDisposable +{ + private readonly TcpListener _listener; + private readonly LinksServerOptions _options; + private readonly object _storeLock = new(); + private readonly HashSet _clients = new(); + private volatile bool _stopping; + + private LinksServer(TcpListener listener, LinksServerOptions options) + { + _listener = listener; + _options = options; + } + + /// The address the server listens on. + public IPEndPoint LocalEndPoint => (IPEndPoint)_listener.LocalEndpoint; + + /// Binds to : (use port 0 for an ephemeral port). + public static LinksServer Bind(string host, int port, LinksServerOptions? options = null) + { + ArgumentNullException.ThrowIfNull(host); + var address = IPAddress.TryParse(host, out var parsed) + ? parsed + : Dns.GetHostAddresses(host).First(); + var listener = new TcpListener(address, port); + listener.Start(); + return new LinksServer(listener, options ?? new LinksServerOptions()); + } + + /// Binds to an address written as host:port. + public static LinksServer Bind(string endPoint, LinksServerOptions? options = null) + { + ArgumentNullException.ThrowIfNull(endPoint); + var (host, port) = ParseEndPoint(endPoint); + return Bind(host, port, options); + } + + /// Splits host:port (or [ipv6]:port) into its parts. + public static (string Host, int Port) ParseEndPoint(string endPoint) + { + ArgumentNullException.ThrowIfNull(endPoint); + var separator = endPoint.LastIndexOf(':'); + if (separator <= 0 || !int.TryParse(endPoint[(separator + 1)..], out var port) || port is < 0 or > 65535) + { + throw new ArgumentException($"expected host:port, got '{endPoint}'", nameof(endPoint)); + } + return (endPoint[..separator].Trim('[', ']'), port); + } + + /// Serves until is called. + public void Serve(INamedTypesLinks links) + { + ArgumentNullException.ThrowIfNull(links); + var workers = new List(); + while (!_stopping) + { + TcpClient client; + try + { + client = _listener.AcceptTcpClient(); + } + catch (Exception error) when (error is SocketException or ObjectDisposedException or InvalidOperationException) + { + break; + } + lock (_clients) + { + if (_stopping) + { + client.Dispose(); + break; + } + _clients.Add(client); + } + var worker = new Thread(() => HandleConnection(client, links)) { IsBackground = true }; + workers.Add(worker); + worker.Start(); + workers.RemoveAll(thread => !thread.IsAlive); + } + Shutdown(); + foreach (var worker in workers) + { + worker.Join(); + } + } + + /// Stops and closes every open connection. + public void Shutdown() + { + lock (_clients) + { + _stopping = true; + foreach (var client in _clients) + { + client.Dispose(); + } + _clients.Clear(); + } + _listener.Stop(); + } + + /// + public void Dispose() => Shutdown(); + + private void HandleConnection(TcpClient client, INamedTypesLinks links) + { + try + { + client.NoDelay = true; + using var stream = client.GetStream(); + var reader = new LinoStreamReader(stream); + while (!_stopping) + { + (IReadOnlyList Document, MessageFormat Format)? message; + try + { + message = LinoProtocols.ReadAnyDocument(reader, _options.Limits); + } + catch (LinoProtocolException error) when (error.Kind != LinoProtocolErrorKind.Io) + { + // The stream may be out of sync; answer in text and hang up. + MessageFormat.Text.Protocol(_options.Limits).WriteDocument(stream, ErrorDocument(error.Message)); + Trace($"connection closed: {error.Message}"); + return; + } + if (message is not { } received) + { + return; + } + var (document, format) = received; + var reply = Accepts(format) ? Execute(links, document) : ErrorDocument("this server does not accept this protocol"); + if (reply is null) + { + return; + } + format.Protocol(_options.Limits).WriteDocument(stream, reply); + } + } + catch (Exception error) when (error is IOException or ObjectDisposedException or SocketException or LinoProtocolException) + { + Trace($"connection closed: {error.Message}"); + } + finally + { + lock (_clients) + { + _clients.Remove(client); + } + client.Dispose(); + } + } + + private bool Accepts(MessageFormat format) => _options.Accept switch + { + AcceptedProtocols.Text => !format.IsBinary, + AcceptedProtocols.Binary => format.IsBinary, + _ => true, + }; + + private IReadOnlyList? Execute(INamedTypesLinks links, IReadOnlyList document) + { + lock (_storeLock) + { + // No request may touch the store once Serve is returning. + if (_stopping) + { + return null; + } + Trace($"request: {LinoFormat.FormatDocument(document)}"); + var reply = ExecuteRequest(links, document, _options.AutoCreateMissingReferences); + Trace($"reply: {LinoFormat.FormatDocument(reply)}"); + return reply; + } + } + + private void Trace(string message) + { + if (_options.Trace) + { + Console.Error.WriteLine($"[server] {message}"); + } + } + + /// The reply document for a failed request: (error: 'message'). + public static IReadOnlyList ErrorDocument(string message) => + new[] { LinoFormat.Link("error", new List { LinoFormat.Reference(message) }) }; + + /// Returns the message of an (error: 'message') reply, or null. + public static string? ErrorMessage(IReadOnlyList document) + { + ArgumentNullException.ThrowIfNull(document); + return document is [{ Id: "error", Values: [{ Values: null, Id: var message }] }] ? message : null; + } + + /// + /// Executes one request against . + /// + /// + /// A non-empty document is a substitution query; the reply holds one + /// (before) (after) line per change, exactly like clink --changes. + /// The empty document asks for every link, one (index: source target) + /// per line. Failures produce an . + /// + public static IReadOnlyList ExecuteRequest( + INamedTypesLinks links, + IReadOnlyList document, + bool autoCreateMissingReferences = false) + { + ArgumentNullException.ThrowIfNull(links); + ArgumentNullException.ThrowIfNull(document); + try + { + if (document.Count == 0) + { + var any = links.Constants.Any; + return links.All(new DoubletLink(any, any, any)) + .Select(link => new DoubletLink(link)) + .OrderBy(link => link.Index) + .Select(link => LinkLino(links, link)) + .ToList(); + } + var changes = new List<(DoubletLink Before, DoubletLink After)>(); + AdvancedMixedQueryProcessor.ProcessQuery(links, new AdvancedMixedQueryProcessor.Options + { + Query = LinoFormat.FormatDocument(document), + AutoCreateMissingReferences = autoCreateMissingReferences, + ChangesHandler = (before, after) => + { + changes.Add((new DoubletLink(before), new DoubletLink(after))); + return links.Constants.Continue; + }, + }); + return ChangesSimplifier.SimplifyChanges(changes) + .Select(change => LinoFormat.Link(null, new List + { + ChangeSide(links, change.Before), + ChangeSide(links, change.After), + })) + .ToList(); + } + catch (Exception error) when (error is not OutOfMemoryException) + { + return ErrorDocument(error.Message); + } + } + + // `(index: source target)`, naming every reference that has a name. + private static LinoLink LinkLino(INamedTypesLinks links, DoubletLink link) => + LinoFormat.Link(ReferenceName(links, link.Index), new List + { + LinoFormat.Reference(ReferenceName(links, link.Source)), + LinoFormat.Reference(ReferenceName(links, link.Target)), + }); + + // `()` for a missing side of a change, `((index: source target))` otherwise. + private static LinoLink ChangeSide(INamedTypesLinks links, DoubletLink link) => + LinoFormat.Link(null, link.IsNull() ? new List() : new List { LinkLino(links, link) }); + + private static string ReferenceName(INamedTypesLinks links, uint id) => + links.GetName(id) ?? id.ToString(System.Globalization.CultureInfo.InvariantCulture); +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs new file mode 100644 index 0000000..03e8acd --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs @@ -0,0 +1,142 @@ +using Link.Foundation.Links.Notation; + +using LinoLink = Link.Foundation.Links.Notation.Link; + +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// +/// Canonical LiNo text for documents, shared by every protocol. +/// +/// +/// A document is the list of top-level links of a message. A reference is a +/// whose Values is null. The canonical model is +/// the one the Rust links-notation parser produces, so documents decoded from +/// the same bytes compare equal in both ports; the C# parser keeps one extra +/// wrapper around single-reference groups, which +/// removes. ParseDocument(FormatDocument(document)) always equals +/// document, which is what makes the text and binary protocols +/// interchangeable. +/// +public static class LinoFormat +{ + private static readonly char[] Quotes = { '\'', '"', '`' }; + + /// A reference: a link without values. + public static LinoLink Reference(string text) => new(text); + + /// A link with an optional id and the given values. + public static LinoLink Link(string? id, IList values) => new(id!, values); + + /// True when is a reference. + public static bool IsReference(LinoLink link) => link.Values is null; + + /// Parses LiNo text into a canonical document. Blank input is the empty document. + public static IReadOnlyList ParseDocument(string text) + { + ArgumentNullException.ThrowIfNull(text); + if (string.IsNullOrWhiteSpace(text)) + { + return Array.Empty(); + } + IList parsed; + try + { + parsed = new Parser().Parse(text); + } + catch (Exception exception) when (exception is not OutOfMemoryException) + { + throw new LinoProtocolException(LinoProtocolErrorKind.InvalidLino, exception.Message, exception); + } + return parsed.Select(Canonical).ToList(); + } + + /// + /// Converts a link produced by the C# parser into the canonical model: + /// an unnamed group holding exactly one reference is that reference. + /// + public static LinoLink Canonical(LinoLink link) + { + if (link.Values is null) + { + return link; + } + if (link.Id is null && link.Values.Count == 1 && link.Values[0].Values is null) + { + return link.Values[0]; + } + return Link(link.Id, link.Values.Select(Canonical).ToList()); + } + + /// + /// Formats a document as canonical LiNo text, one top-level link per line. + /// A top-level link without an id and with at least two values is written + /// without its outer parentheses: () ((1 1)). + /// + public static string FormatDocument(IEnumerable document) + { + ArgumentNullException.ThrowIfNull(document); + return string.Join("\n", document.Select(FormatTopLevel)); + } + + private static string FormatTopLevel(LinoLink link) => + link.Values is { Count: >= 2 } values && link.Id is null ? JoinValues(values) : FormatLink(link); + + /// Formats one link as it appears nested inside another link. + public static string FormatLink(LinoLink link) + { + if (link.Values is not { } values) + { + return FormatReference(link.Id ?? string.Empty); + } + if (link.Id is null) + { + // `(a)` parses back as the reference `a`, so a one-reference + // link needs a second pair of parentheses. + return values.Count == 1 && values[0].Values is null + ? $"(({FormatReference(values[0].Id ?? string.Empty)}))" + : $"({JoinValues(values)})"; + } + return values.Count == 0 + ? $"({FormatReference(link.Id)}:)" + : $"({FormatReference(link.Id)}: {JoinValues(values)})"; + } + + private static string JoinValues(IEnumerable values) => string.Join(" ", values.Select(FormatLink)); + + /// Quotes a reference when it would not survive parsing as a bare word. + /// + /// links-notation opens a quoted reference with a run of N equal quote + /// characters, closes it with the next run of exactly N, and reads 2N + /// quotes inside as N literal ones. The opening run is counted greedily, + /// so the chosen quote must differ from the first character; N is one + /// more than the longest run of that quote inside, and odd, because an + /// even delimiter run may be read as an empty reference. + /// + public static string FormatReference(string reference) + { + ArgumentNullException.ThrowIfNull(reference); + var needsQuotes = reference.Length == 0 + || reference.Any(character => char.IsWhiteSpace(character) || character is '(' or ')' or ':' or '\'' or '"' or '`'); + if (!needsQuotes) + { + return reference; + } + var (quote, count) = Quotes + .Where(quote => reference.Length == 0 || reference[0] != quote) + .Select(quote => (Quote: quote, Count: (LongestRun(reference, quote) + 1) | 1)) + .MinBy(candidate => candidate.Count); + var delimiter = new string(quote, count); + return $"{delimiter}{reference}{delimiter}"; + } + + private static int LongestRun(string text, char quote) + { + var (longest, current) = (0, 0); + foreach (var character in text) + { + current = character == quote ? current + 1 : 0; + longest = Math.Max(longest, current); + } + return longest; + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoMapping.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoMapping.cs new file mode 100644 index 0000000..1fccb72 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoMapping.cs @@ -0,0 +1,521 @@ +using System.Globalization; +using System.Text; + +using LinoLink = Link.Foundation.Links.Notation.Link; + +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// +/// Optional features of the binary LiNo protocol. Every feature is off by +/// default; each one can be switched on independently, like stacking a decorator. +/// +/// +/// Send numbers and code points as Hybrid external references instead of +/// in-band unary links. Halves the internal address range of each width. +/// +/// +/// Use the variable-length sequence section for lists, strings and links with +/// ids instead of cons chains of doublets. +/// +/// +/// Let the reference width grow with the address instead of using the width of +/// the largest address for every reference. +/// +public readonly record struct BinaryLinoOptions( + bool ExternalReferences = false, + bool Sequences = false, + bool ProgressiveWidths = false) +{ + /// Enables or disables external references. + public BinaryLinoOptions WithExternalReferences(bool enabled = true) => this with { ExternalReferences = enabled }; + + /// Enables or disables the sequence section. + public BinaryLinoOptions WithSequences(bool enabled = true) => this with { Sequences = enabled }; + + /// Enables or disables progressive reference widths. + public BinaryLinoOptions WithProgressiveWidths(bool enabled = true) => this with { ProgressiveWidths = enabled }; +} + +/// +/// Lossless mapping between LiNo documents and s, +/// byte-for-byte compatible with the Rust port. +/// +/// +/// +/// () is the null link 0. +/// A numeric reference n is (Number unary(n)), or an external reference when enabled. +/// Any other reference is (String code points…); code points are unary numbers or externals. +/// A link without an id and with exactly two values is a plain doublet. +/// A link without an id and any other number of values is a list. +/// A link with an id is (Identified id values…). +/// The document is a list of its top-level links, stored last (the root). +/// +/// Typed values and lists are a sequence [marker, elements…] when the +/// sequence section is enabled and the doublet (marker chain) otherwise, +/// where chain is the nil-terminated cons list (e1 (e2 (… (en 0)))). +/// Identical sub-links are emitted once and shared. +/// +public static class LinoMapping +{ + /// Converts a document into a packet. + public static LinksPacket EncodeDocument(IReadOnlyList document, BinaryLinoOptions options = default) + { + ArgumentNullException.ThrowIfNull(document); + var encoder = new Encoder(options); + if (document.Count > 0) + { + var items = document.Select(encoder.Encode).ToList(); + if (options.Sequences) + { + encoder.Sequences.Add(items); + } + else + { + var chain = encoder.Chain(items); + encoder.Doublets.Add((Node.Internal(LinksPacket.List), chain)); + } + } + return encoder.Finish(); + } + + /// Converts a packet back into a document. + public static IReadOnlyList DecodeDocument(LinksPacket packet, DecodeLimits? limits = null) + { + ArgumentNullException.ThrowIfNull(packet); + limits ??= DecodeLimits.Default; + if (packet.LastAddress is not { } root) + { + return Array.Empty(); + } + var decoder = new Decoder(packet, limits); + var view = decoder.View(PacketReference.Internal(root)); + IReadOnlyList items = view switch + { + { Kind: ViewKind.Sequence } when !StartsWithMarker(view.Items!) => view.Items!, + { Kind: ViewKind.Doublet, Source: { IsExternal: false, Value: LinksPacket.List } } => decoder.Chain(view.Target), + _ => throw LinoProtocolException.Malformed("the root link is not a list"), + }; + var budget = limits.MaxNodes; + return items.Select(item => decoder.Decode(item, 0, ref budget)).ToList(); + } + + /// Parses a canonical unsigned decimal number (no sign, no leading zeros). + public static bool TryParseCanonicalNumber(string text, out ulong value) + { + value = 0; + var canonical = !string.IsNullOrEmpty(text) + && text.All(character => character is >= '0' and <= '9') + && (text == "0" || text[0] != '0'); + return canonical && ulong.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value); + } + + private static bool IsMarker(ulong address) => address is >= LinksPacket.One and < LinksPacket.FirstLinkAddress; + + private static bool StartsWithMarker(IReadOnlyList items) => + items.Count > 0 && !items[0].IsExternal && IsMarker(items[0].Value); + + private enum NodeKind + { + Internal, + External, + Doublet, + Sequence, + } + + private readonly record struct Node(NodeKind Kind, ulong Value) + { + public static Node Internal(ulong address) => new(NodeKind.Internal, address); + + public static Node External(ulong value) => new(NodeKind.External, value); + } + + private sealed class NodeListComparer : IEqualityComparer> + { + public static readonly NodeListComparer Instance = new(); + + public bool Equals(List? x, List? y) => + ReferenceEquals(x, y) || (x is not null && y is not null && x.SequenceEqual(y)); + + public int GetHashCode(List obj) + { + var hash = new HashCode(); + foreach (var node in obj) + { + hash.Add(node); + } + return hash.ToHashCode(); + } + } + + private sealed class Encoder + { + private readonly BinaryLinoOptions _options; + private readonly Dictionary<(Node, Node), int> _doubletIndex = new(); + private readonly Dictionary, int> _sequenceIndex = new(NodeListComparer.Instance); + private readonly List _powers = new() { Node.Internal(LinksPacket.One) }; + + public Encoder(BinaryLinoOptions options) => _options = options; + + public List<(Node Source, Node Target)> Doublets { get; } = new(); + + public List> Sequences { get; } = new(); + + private Node Doublet(Node source, Node target) + { + if (_doubletIndex.TryGetValue((source, target), out var existing)) + { + return new Node(NodeKind.Doublet, (ulong)existing); + } + var index = Doublets.Count; + Doublets.Add((source, target)); + _doubletIndex[(source, target)] = index; + return new Node(NodeKind.Doublet, (ulong)index); + } + + private Node Sequence(List items) + { + if (_sequenceIndex.TryGetValue(items, out var existing)) + { + return new Node(NodeKind.Sequence, (ulong)existing); + } + var index = Sequences.Count; + Sequences.Add(items); + _sequenceIndex[items] = index; + return new Node(NodeKind.Sequence, (ulong)index); + } + + // Fixed doublets may only refer to fixed doublets, so a pair holding + // a sequence becomes a two-element sequence. + private Node Pair(Node first, Node second) => + first.Kind == NodeKind.Sequence || second.Kind == NodeKind.Sequence + ? Sequence(new List { first, second }) + : Doublet(first, second); + + public Node Chain(IReadOnlyList items) + { + var tail = Node.Internal(LinksPacket.NullAddress); + for (var index = items.Count - 1; index >= 0; index--) + { + tail = Doublet(items[index], tail); + } + return tail; + } + + private Node Typed(ulong marker, List elements) + { + if (_options.Sequences) + { + elements.Insert(0, Node.Internal(marker)); + return Sequence(elements); + } + return Doublet(Node.Internal(marker), Chain(elements)); + } + + private Node Power(int exponent) + { + while (_powers.Count <= exponent) + { + var previous = _powers[^1]; + _powers.Add(Doublet(previous, previous)); + } + return _powers[exponent]; + } + + private Node Unary(ulong value) + { + var powers = new List(); + for (var bit = 63; bit >= 0; bit--) + { + if ((value & (1UL << bit)) != 0) + { + powers.Add(Power(bit)); + } + } + if (powers.Count == 0) + { + return Node.Internal(LinksPacket.NullAddress); + } + var sum = powers[^1]; + for (var index = powers.Count - 2; index >= 0; index--) + { + sum = Doublet(powers[index], sum); + } + return sum; + } + + private Node Scalar(ulong value) => + _options.ExternalReferences && value <= LinksPacket.ExternalCapacity(8) ? Node.External(value) : Unary(value); + + private Node Reference(string text) + { + if (TryParseCanonicalNumber(text, out var value)) + { + if (_options.ExternalReferences && value <= LinksPacket.ExternalCapacity(8)) + { + return Node.External(value); + } + var unary = Unary(value); + return Doublet(Node.Internal(LinksPacket.Number), unary); + } + var codePoints = text.EnumerateRunes().Select(rune => Scalar((ulong)rune.Value)).ToList(); + return Typed(LinksPacket.String, codePoints); + } + + public Node Encode(LinoLink link) + { + if (link.Values is not { } values) + { + return Reference(link.Id ?? string.Empty); + } + if (link.Id is { } id) + { + var elements = new List { Reference(id) }; + elements.AddRange(values.Select(Encode)); + return Typed(LinksPacket.Identified, elements); + } + switch (values.Count) + { + case 0: + return Node.Internal(LinksPacket.NullAddress); + case 2: + var first = Encode(values[0]); + var second = Encode(values[1]); + return Pair(first, second); + default: + var items = values.Select(Encode).ToList(); + return _options.Sequences ? Sequence(items) : Typed(LinksPacket.List, items); + } + } + + public LinksPacket Finish() + { + var doubletCount = (ulong)Doublets.Count; + PacketReference Resolve(Node node) => node.Kind switch + { + NodeKind.Internal => PacketReference.Internal(node.Value), + NodeKind.External => PacketReference.External(node.Value), + NodeKind.Doublet => PacketReference.Internal(LinksPacket.FirstLinkAddress + node.Value), + _ => PacketReference.Internal(LinksPacket.FirstLinkAddress + doubletCount + node.Value), + }; + var packet = new LinksPacket + { + ExternalReferences = _options.ExternalReferences, + SequencesSection = _options.Sequences, + }; + packet.Doublets.AddRange(Doublets.Select(doublet => (Resolve(doublet.Source), Resolve(doublet.Target)))); + packet.Sequences.AddRange(Sequences.Select(items => items.Select(Resolve).ToList())); + packet.MinWidth = packet.RequiredMinWidth(!_options.ProgressiveWidths); + return packet; + } + } + + private enum ViewKind + { + Null, + Marker, + External, + Doublet, + Sequence, + } + + private readonly record struct View( + ViewKind Kind, + ulong Value = 0, + PacketReference Source = default, + PacketReference Target = default, + IReadOnlyList? Items = null); + + private sealed class Decoder + { + private readonly LinksPacket _packet; + private readonly DecodeLimits _limits; + // _unary[i] is the number doublet i denotes, if it is a unary number. + private readonly ulong?[] _unary; + + public Decoder(LinksPacket packet, DecodeLimits limits) + { + _packet = packet; + _limits = limits; + // Links only refer backwards, so one forward pass evaluates every + // unary number without recursion. + _unary = new ulong?[packet.Doublets.Count]; + for (var index = 0; index < packet.Doublets.Count; index++) + { + var (source, target) = packet.Doublets[index]; + if (UnaryValue(source) is { } sourceValue + && UnaryValue(target) is { } targetValue + && sourceValue <= ulong.MaxValue - targetValue) + { + _unary[index] = sourceValue + targetValue; + } + } + } + + private ulong? UnaryValue(PacketReference reference) + { + if (reference.IsExternal) + { + return null; + } + return reference.Value switch + { + LinksPacket.NullAddress => 0, + LinksPacket.One => 1, + >= LinksPacket.FirstLinkAddress when reference.Value - LinksPacket.FirstLinkAddress < (ulong)_unary.Length + => _unary[reference.Value - LinksPacket.FirstLinkAddress], + _ => null, + }; + } + + public View View(PacketReference reference) + { + if (reference.IsExternal) + { + return new View(ViewKind.External, reference.Value); + } + if (reference.Value == LinksPacket.NullAddress) + { + return new View(ViewKind.Null); + } + if (reference.Value < LinksPacket.FirstLinkAddress) + { + return new View(ViewKind.Marker, reference.Value); + } + var address = reference.Value - LinksPacket.FirstLinkAddress; + var doublets = (ulong)_packet.Doublets.Count; + if (address < doublets) + { + var (source, target) = _packet.Doublets[(int)address]; + return new View(ViewKind.Doublet, Source: source, Target: target); + } + if (address - doublets < (ulong)_packet.Sequences.Count) + { + return new View(ViewKind.Sequence, Items: _packet.Sequences[(int)(address - doublets)]); + } + throw LinoProtocolException.Malformed($"dangling reference {address}"); + } + + private ulong Number(PacketReference reference) + { + if (reference.IsExternal) + { + return reference.Value; + } + return UnaryValue(reference) ?? throw LinoProtocolException.Malformed("expected a unary number"); + } + + // The elements of a typed value given in doublet form: a cons chain, + // optionally ending in a sequence holding the remaining elements. + public List Chain(PacketReference tail) + { + var elements = new List(); + while (true) + { + var view = View(tail); + switch (view.Kind) + { + case ViewKind.Null: + return elements; + case ViewKind.Doublet: + if (elements.Count >= _limits.MaxNodes) + { + throw LinoProtocolException.Limit("chain too long"); + } + elements.Add(view.Source); + tail = view.Target; + break; + case ViewKind.Sequence: + elements.AddRange(view.Items!); + return elements; + default: + throw LinoProtocolException.Malformed("broken element chain"); + } + } + } + + private LinoLink Typed(ulong marker, IReadOnlyList elements, int depth, ref long budget) + { + switch (marker) + { + case LinksPacket.Number: + if (elements.Count != 1) + { + throw LinoProtocolException.Malformed("a number needs exactly one value"); + } + return LinoFormat.Reference(Number(elements[0]).ToString(CultureInfo.InvariantCulture)); + case LinksPacket.String: + var text = new StringBuilder(elements.Count); + foreach (var element in elements) + { + var codePoint = Number(element); + if (codePoint > int.MaxValue || !Rune.IsValid((int)codePoint)) + { + throw LinoProtocolException.Malformed($"invalid code point {codePoint}"); + } + text.Append(new Rune((int)codePoint).ToString()); + } + return LinoFormat.Reference(text.ToString()); + case LinksPacket.List: + return List(elements, 0, depth, ref budget); + case LinksPacket.Identified: + if (elements.Count == 0) + { + throw LinoProtocolException.Malformed("an identified link needs an id"); + } + var id = Decode(elements[0], depth + 1, ref budget); + if (id.Values is not null) + { + throw LinoProtocolException.Malformed("a link id must be a reference"); + } + var values = List(elements, 1, depth, ref budget).Values!; + return LinoFormat.Link(id.Id ?? string.Empty, values); + default: + throw LinoProtocolException.Malformed($"marker {marker} cannot start a typed value"); + } + } + + private LinoLink List(IReadOnlyList elements, int skip, int depth, ref long budget) + { + var values = new List(Math.Max(0, elements.Count - skip)); + for (var index = skip; index < elements.Count; index++) + { + values.Add(Decode(elements[index], depth + 1, ref budget)); + } + return LinoFormat.Link(null, values); + } + + public LinoLink Decode(PacketReference reference, int depth, ref long budget) + { + if (depth >= _limits.MaxDepth) + { + throw LinoProtocolException.Limit($"nesting deeper than {_limits.MaxDepth}"); + } + if (budget <= 0) + { + throw LinoProtocolException.Limit("too many LiNo nodes"); + } + budget--; + var view = View(reference); + switch (view.Kind) + { + case ViewKind.Null: + return LinoFormat.Link(null, new List()); + case ViewKind.External: + return LinoFormat.Reference(view.Value.ToString(CultureInfo.InvariantCulture)); + case ViewKind.Marker: + throw LinoProtocolException.Malformed($"marker {view.Value} used as a value"); + case ViewKind.Doublet when !view.Source.IsExternal && view.Source.Value == LinksPacket.Number: + return Typed(LinksPacket.Number, new[] { view.Target }, depth, ref budget); + case ViewKind.Doublet when !view.Source.IsExternal && IsMarker(view.Source.Value): + return Typed(view.Source.Value, Chain(view.Target), depth, ref budget); + case ViewKind.Doublet: + return List(new[] { view.Source, view.Target }, 0, depth, ref budget); + default: + var items = view.Items!; + return StartsWithMarker(items) + ? Typed(items[0].Value, items.Skip(1).ToList(), depth, ref budget) + : List(items, 0, depth, ref budget); + } + } + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocolException.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocolException.cs new file mode 100644 index 0000000..0c0628f --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocolException.cs @@ -0,0 +1,67 @@ +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// What went wrong while encoding, decoding or exchanging a LiNo message. +public enum LinoProtocolErrorKind +{ + /// The underlying transport failed. + Io, + /// The peer sent bytes that are not a valid message. + Malformed, + /// The message text is not valid LiNo. + InvalidLino, + /// The message exceeds one of the configured . + LimitExceeded, + /// The document cannot be represented with the chosen options. + Unencodable, + /// The server answered with an (error: …) document. + Remote, +} + +/// Raised by the LiNo network protocols (issue #105). +public sealed class LinoProtocolException : Exception +{ + public LinoProtocolException() + : this(LinoProtocolErrorKind.Malformed, "unknown protocol error") + { + } + + public LinoProtocolException(string message) + : this(LinoProtocolErrorKind.Malformed, message) + { + } + + public LinoProtocolException(string message, Exception innerException) + : this(LinoProtocolErrorKind.Io, message, innerException) + { + } + + public LinoProtocolException(LinoProtocolErrorKind kind, string detail, Exception? innerException = null) + : base($"{Describe(kind)}: {detail}", innerException) + { + Kind = kind; + Detail = detail; + } + + /// The category of the failure. + public LinoProtocolErrorKind Kind { get; } + + /// The message without the category prefix. + public string Detail { get; } + + internal static LinoProtocolException Malformed(string detail) => new(LinoProtocolErrorKind.Malformed, detail); + + internal static LinoProtocolException Limit(string detail) => new(LinoProtocolErrorKind.LimitExceeded, detail); + + internal static LinoProtocolException Unencodable(string detail) => new(LinoProtocolErrorKind.Unencodable, detail); + + private static string Describe(LinoProtocolErrorKind kind) => kind switch + { + LinoProtocolErrorKind.Io => "I/O error", + LinoProtocolErrorKind.Malformed => "malformed message", + LinoProtocolErrorKind.InvalidLino => "invalid LiNo", + LinoProtocolErrorKind.LimitExceeded => "limit exceeded", + LinoProtocolErrorKind.Unencodable => "cannot encode", + LinoProtocolErrorKind.Remote => "server error", + _ => kind.ToString(), + }; +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs new file mode 100644 index 0000000..ec7e29e --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs @@ -0,0 +1,263 @@ +using System.Text; + +using LinoLink = Link.Foundation.Links.Notation.Link; + +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// +/// Reads and writes whole LiNo documents on a byte stream. Both +/// and +/// implement it, so code written against it switches protocol by swapping one value. +/// +public interface ILinoProtocol +{ + /// Writes one message carrying . + void WriteDocument(Stream output, IReadOnlyList document); + + /// Reads one message; null when the stream ended cleanly. + IReadOnlyList? ReadDocument(LinoStreamReader input); +} + +/// +/// UTF-8 LiNo text, one message per block of lines ended by a line holding +/// only .. Lines starting with . get one extra . (SMTP dot-stuffing). +/// +/// +/// Lines may end with \n or \r\n; a carriage return right before +/// a line feed is dropped, so a quoted reference holding \r\n arrives as +/// \n (use to carry such references exactly). +/// +public sealed record TextLinoProtocol : ILinoProtocol +{ + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + /// Limits applied when reading. + public DecodeLimits Limits { get; init; } = DecodeLimits.Default; + + /// Frames already formatted LiNo text as one message. + public static void WriteText(Stream output, string text) + { + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(text); + var framed = new StringBuilder(text.Length + 4); + if (text.Length > 0) + { + foreach (var line in text.Split('\n')) + { + if (line.StartsWith('.')) + { + framed.Append('.'); + } + framed.Append(line).Append('\n'); + } + } + framed.Append(".\n"); + var bytes = StrictUtf8.GetBytes(framed.ToString()); + output.Write(bytes); + output.Flush(); + } + + /// Reads one framed message as raw text, without parsing it. + public string? ReadText(LinoStreamReader input) + { + ArgumentNullException.ThrowIfNull(input); + var text = new List(); + var line = new List(); + var first = true; + while (true) + { + line.Clear(); + var budget = Math.Max(0, Limits.MaxTextBytes - text.Count) + 2; + var read = input.ReadLine(line, budget); + if (read == 0) + { + if (first) + { + return null; + } + throw LinoProtocolException.Malformed("stream ended before the '.' terminator line"); + } + if (line[^1] != (byte)'\n') + { + if (read >= budget) + { + throw LinoProtocolException.Limit($"text message longer than {Limits.MaxTextBytes} bytes"); + } + throw LinoProtocolException.Malformed("stream ended before the '.' terminator line"); + } + line.RemoveAt(line.Count - 1); + // CRLF line endings (telnet, netcat on Windows) are accepted. + if (line.Count > 0 && line[^1] == (byte)'\r') + { + line.RemoveAt(line.Count - 1); + } + if (line.Count == 1 && line[0] == (byte)'.') + { + break; + } + if (!first) + { + text.Add((byte)'\n'); + } + first = false; + var skip = line.Count > 0 && line[0] == (byte)'.' ? 1 : 0; + text.AddRange(line.Skip(skip)); + } + try + { + return StrictUtf8.GetString(text.ToArray()); + } + catch (DecoderFallbackException) + { + throw LinoProtocolException.Malformed("text message is not valid UTF-8"); + } + } + + /// + public void WriteDocument(Stream output, IReadOnlyList document) => + WriteText(output, LinoFormat.FormatDocument(document)); + + /// + public IReadOnlyList? ReadDocument(LinoStreamReader input) => + ReadText(input) is { } text ? LinoFormat.ParseDocument(text) : null; +} + +/// +/// Binary links packets (see ); every message is +/// self-delimiting, so no extra framing is needed. +/// +public sealed record BinaryLinoProtocol : ILinoProtocol +{ + /// A binary protocol with every optional feature off. + public BinaryLinoProtocol() + { + } + + /// A binary protocol with the given optional features. + public BinaryLinoProtocol(BinaryLinoOptions options) => Options = options; + + /// Optional features used when writing. + public BinaryLinoOptions Options { get; init; } + + /// Limits applied when reading. + public DecodeLimits Limits { get; init; } = DecodeLimits.Default; + + /// Encodes a document into packet bytes. + public byte[] Encode(IReadOnlyList document) => LinoMapping.EncodeDocument(document, Options).ToBytes(); + + /// Decodes packet bytes into a document. + public IReadOnlyList Decode(byte[] bytes) => + LinoMapping.DecodeDocument(LinksPacket.FromBytes(bytes, Limits), Limits); + + /// + public void WriteDocument(Stream output, IReadOnlyList document) + { + ArgumentNullException.ThrowIfNull(output); + output.Write(Encode(document)); + output.Flush(); + } + + /// + public IReadOnlyList? ReadDocument(LinoStreamReader input) => + LinksPacket.ReadFrom(input, Limits) is { } packet ? LinoMapping.DecodeDocument(packet, Limits) : null; +} + +/// The wire format a message arrived in, so a reply can use the same one. +/// False for . +/// The binary options read from the packet header. +public readonly record struct MessageFormat(bool IsBinary, BinaryLinoOptions Options = default) +{ + /// The text format. + public static MessageFormat Text => new(false); + + /// The binary format with . + public static MessageFormat Binary(BinaryLinoOptions options) => new(true, options); + + /// A protocol that writes messages in this format. + public ILinoProtocol Protocol(DecodeLimits limits) => + IsBinary ? new BinaryLinoProtocol(Options) { Limits = limits } : new TextLinoProtocol { Limits = limits }; +} + +/// Detects which protocol a peer used. +public static class LinoProtocols +{ + /// True when starts a binary message rather than a text one. + public static bool IsBinaryStart(byte value) => (value & 0xF0) == LinksPacket.BinaryVersion1; + + /// Reads one message in whichever protocol the peer used; null at the end of the stream. + public static (IReadOnlyList Document, MessageFormat Format)? ReadAnyDocument( + LinoStreamReader input, + DecodeLimits? limits = null) + { + ArgumentNullException.ThrowIfNull(input); + limits ??= DecodeLimits.Default; + var first = input.PeekByte(); + if (first < 0) + { + return null; + } + if (!IsBinaryStart((byte)first)) + { + return new TextLinoProtocol { Limits = limits }.ReadDocument(input) is { } text + ? (text, MessageFormat.Text) + : null; + } + if (LinksPacket.ReadFrom(input, limits) is not { } packet) + { + return null; + } + var progressive = packet.LastAddress is { } last + && packet.MinWidth < LinksPacket.AddressTier(last, packet.ExternalReferences); + var options = new BinaryLinoOptions(packet.ExternalReferences, packet.SequencesSection, progressive); + return (LinoMapping.DecodeDocument(packet, limits), MessageFormat.Binary(options)); + } +} + +/// A byte stream decorated with an . +public sealed class LinoConnection : IDisposable +{ + private readonly Stream _stream; + private readonly LinoStreamReader _reader; + + /// Wraps ; disposing the connection disposes the stream. + public LinoConnection(Stream stream, ILinoProtocol protocol) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(protocol); + _stream = stream; + _reader = new LinoStreamReader(stream); + Protocol = protocol; + } + + /// The protocol in use. + public ILinoProtocol Protocol { get; } + + /// Sends one document. + public void Send(IReadOnlyList document) + { + using var buffer = new MemoryStream(); + Protocol.WriteDocument(buffer, document); + try + { + buffer.WriteTo(_stream); + _stream.Flush(); + } + catch (Exception error) when (error is IOException or ObjectDisposedException) + { + throw new LinoProtocolException(LinoProtocolErrorKind.Io, error.Message, error); + } + } + + /// Receives one document; null when the peer closed the stream. + public IReadOnlyList? Receive() => Protocol.ReadDocument(_reader); + + /// Sends and waits for the reply. + public IReadOnlyList Request(IReadOnlyList document) + { + Send(document); + return Receive() ?? throw LinoProtocolException.Malformed("connection closed before the reply"); + } + + /// + public void Dispose() => _stream.Dispose(); +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoStreamReader.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoStreamReader.cs new file mode 100644 index 0000000..da32e20 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoStreamReader.cs @@ -0,0 +1,96 @@ +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// +/// A buffered byte reader that can peek at the next byte, so a server can +/// tell a text message from a binary one before reading it. +/// +public sealed class LinoStreamReader +{ + private readonly Stream _stream; + private readonly byte[] _buffer; + private int _position; + private int _length; + + public LinoStreamReader(Stream stream, int bufferSize = 8192) + { + ArgumentNullException.ThrowIfNull(stream); + _stream = stream; + _buffer = new byte[bufferSize]; + } + + /// Wraps a byte array, e.g. a whole encoded message. + public static LinoStreamReader FromBytes(byte[] bytes) => new(new MemoryStream(bytes, writable: false)); + + /// The next byte without consuming it, or -1 at the end of the stream. + public int PeekByte() => Fill() ? _buffer[_position] : -1; + + /// The next byte, or -1 at the end of the stream. + public int ReadByte() => Fill() ? _buffer[_position++] : -1; + + /// True when every byte of the stream has been consumed. + public bool AtEnd => !Fill(); + + /// Fills ; a short stream is a malformed message. + public void ReadExactly(Span destination) + { + while (!destination.IsEmpty) + { + if (!Fill()) + { + throw LinoProtocolException.Malformed("unexpected end of packet"); + } + var count = Math.Min(destination.Length, _length - _position); + _buffer.AsSpan(_position, count).CopyTo(destination); + _position += count; + destination = destination[count..]; + } + } + + /// + /// Appends bytes up to and including the next \n to , + /// reading at most . Returns the number of bytes read. + /// + public int ReadLine(List line, long maxBytes) + { + var read = 0; + while (read < maxBytes && Fill()) + { + var available = _buffer.AsSpan(_position, (int)Math.Min(_length - _position, maxBytes - read)); + var newline = available.IndexOf((byte)'\n'); + var take = newline < 0 ? available.Length : newline + 1; + for (var index = 0; index < take; index++) + { + line.Add(available[index]); + } + _position += take; + read += take; + if (newline >= 0) + { + break; + } + } + return read; + } + + private bool Fill() + { + if (_position < _length) + { + return true; + } + try + { + _length = _stream.Read(_buffer, 0, _buffer.Length); + } + catch (IOException exception) + { + throw new LinoProtocolException(LinoProtocolErrorKind.Io, exception.Message, exception); + } + catch (ObjectDisposedException exception) + { + throw new LinoProtocolException(LinoProtocolErrorKind.Io, exception.Message, exception); + } + _position = 0; + return _length > 0; + } +} From e4456b532ed3428e0bcabd07eede1f5e1d2ab85e Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 18:34:39 +0000 Subject: [PATCH 06/44] fix: upgrade links-notation to 0.22.0 so deep nesting cannot stall a server links-notation 0.16.1 parsed nested groups in time exponential in the depth (link-foundation/links-notation#314, fixed in 0.21.3): a 90-byte message nested 40 deep kept a server thread busy for minutes. 0.22.0 is linear and rejects nesting deeper than 64. 0.22.0 wraps a lone reference in a group in both ports (`a` parses as `(a)`), so the Rust parse_document now removes that wrapper, as C# already did. Golden vectors are unchanged. Deep-nesting regression tests are in the Rust packet tests and the C# codec tests. --- ...oundation.Data.Doublets.Cli.Library.csproj | 2 +- .../Protocol/LinoFormat.cs | 10 +++---- rust/Cargo.lock | 4 +-- rust/Cargo.toml | 2 +- rust/src/protocol/format.rs | 26 +++++++++++++++++-- rust/tests/protocol_packet_tests.rs | 21 +++++++++++++++ 6 files changed, 54 insertions(+), 11 deletions(-) diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Foundation.Data.Doublets.Cli.Library.csproj b/csharp/Foundation.Data.Doublets.Cli.Library/Foundation.Data.Doublets.Cli.Library.csproj index 996ebfb..9edfa20 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Foundation.Data.Doublets.Cli.Library.csproj +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Foundation.Data.Doublets.Cli.Library.csproj @@ -30,7 +30,7 @@ - + diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs index 03e8acd..54c1dad 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs @@ -9,11 +9,11 @@ namespace Foundation.Data.Doublets.Cli.Protocol; /// /// /// A document is the list of top-level links of a message. A reference is a -/// whose Values is null. The canonical model is -/// the one the Rust links-notation parser produces, so documents decoded from -/// the same bytes compare equal in both ports; the C# parser keeps one extra -/// wrapper around single-reference groups, which -/// removes. ParseDocument(FormatDocument(document)) always equals +/// whose Values is null. links-notation keeps +/// one extra wrapper around single-reference groups (a parses as +/// (a)); removes it, exactly like the Rust +/// port, so documents decoded from the same bytes compare equal in both +/// ports. ParseDocument(FormatDocument(document)) always equals /// document, which is what makes the text and binary protocols /// interchangeable. /// diff --git a/rust/Cargo.lock b/rust/Cargo.lock index cd2718e..5b624da 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "links-notation" -version = "0.16.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0068a55e836850caf98abf96117045ab641af446ab0816557360b318fd9e5cfe" +checksum = "120d3b63178b82b0a416d7180f3d3c57170a8bbb00ffac4975d27779013fc4be" dependencies = [ "links-notation-macro", "nom", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 4412b55..6a57ab4 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -30,7 +30,7 @@ anyhow = "1.0.104" # Source: http://github.com/linksplatform/doublets-rs doublets = "0.5.0" # Source: http://github.com/link-foundation/links-notation -links-notation = "0.16.1" +links-notation = "0.22.0" # Source: http://github.com/link-foundation/lino-arguments lino-arguments = "0.3.0" diff --git a/rust/src/protocol/format.rs b/rust/src/protocol/format.rs index f6e7157..cb28c20 100644 --- a/rust/src/protocol/format.rs +++ b/rust/src/protocol/format.rs @@ -10,12 +10,34 @@ use super::error::{ProtocolError, ProtocolResult}; use super::mapping::LinoDocument; use links_notation::{parse_lino_to_links, LiNo}; -/// Parses LiNo text into a document. Blank input is the empty document. +/// Parses LiNo text into a canonical document. Blank input is the empty document. pub fn parse_document(text: &str) -> ProtocolResult { if text.trim().is_empty() { return Ok(Vec::new()); } - parse_lino_to_links(text).map_err(|error| ProtocolError::InvalidLino(error.to_string())) + let links = + parse_lino_to_links(text).map_err(|error| ProtocolError::InvalidLino(error.to_string()))?; + Ok(links.into_iter().map(canonical).collect()) +} + +/// Converts a parsed link into the canonical model: an unnamed group holding +/// exactly one reference is that reference, so `a` and `(a)` both parse as +/// the reference `a`. links-notation (since 0.17) and its C# port keep that +/// wrapper; removing it gives both ports the same document model. +pub fn canonical(link: LiNo) -> LiNo { + match link { + LiNo::Link { + id: None, + mut values, + } if values.len() == 1 && matches!(values[0], LiNo::Ref(_)) => { + values.pop().expect("one value") + } + LiNo::Link { id, values } => LiNo::Link { + id, + values: values.into_iter().map(canonical).collect(), + }, + reference => reference, + } } /// Formats a document as canonical LiNo text, one top-level link per line. diff --git a/rust/tests/protocol_packet_tests.rs b/rust/tests/protocol_packet_tests.rs index 373f7a7..f1cf343 100644 --- a/rust/tests/protocol_packet_tests.rs +++ b/rust/tests/protocol_packet_tests.rs @@ -523,3 +523,24 @@ fn every_short_reference_over_delimiters_round_trips() { } } } + +#[test] +fn deeply_nested_text_parses_in_linear_time() { + // links-notation before 0.21.3 took exponential time in the nesting depth + // (link-foundation/links-notation#314), so one short message could stall + // a server thread. Depth 40 took minutes there. + let nested = |depth: usize| format!("{}a{}", "(".repeat(depth), ")".repeat(depth)); + let started = std::time::Instant::now(); + let mut link = parse_document(&nested(40)).unwrap().remove(0); + let mut depth = 1; + while let links_notation::LiNo::Link { mut values, .. } = link { + link = values.remove(0); + depth += 1; + } + assert_eq!(depth, 40); + assert!(matches!( + parse_document(&nested(100_000)), + Err(ProtocolError::InvalidLino(_)) + )); + assert!(started.elapsed() < std::time::Duration::from_secs(10)); +} From 547efa3027a22e70a1f83b00936c1e457f6d8c3e Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 18:34:39 +0000 Subject: [PATCH 07/44] test(csharp): cover the LiNo protocol codecs with the Rust corpus and golden vectors Also add lino-structure-probe examples in C# and Rust that print how each links-notation port structures LiNo text, for checking that they agree. --- .../Protocol/LinoProtocolCodecTests.cs | 400 ++++++++++++++++++ .../LinoStructureProbe.csproj | 15 + examples/lino-structure-probe/Program.cs | 28 ++ rust/examples/lino_structure_probe.rs | 29 ++ 4 files changed, 472 insertions(+) create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs create mode 100644 examples/lino-structure-probe/LinoStructureProbe.csproj create mode 100644 examples/lino-structure-probe/Program.cs create mode 100644 rust/examples/lino_structure_probe.rs diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs new file mode 100644 index 0000000..a3e3ad1 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs @@ -0,0 +1,400 @@ +using Foundation.Data.Doublets.Cli.Protocol; +using Platform.Data; + +using LinoLink = Link.Foundation.Links.Notation.Link; + +namespace Foundation.Data.Doublets.Cli.Tests.Protocol; + +/// Binary and text LiNo protocol codecs (issue #105). +public sealed class LinoProtocolCodecTests +{ + private static readonly string[] Corpus = + { + "() ((1 1))", + "((1: 1 1)) ((1: 1 2))", + "((1 1)) ()", + "((1: 1 1)) ()", + "(($i: $s $t)) (($i: $s $t))", + "((($index: $source $target)) (($index: $target $source)))", + "(a b c d)", + "(name: 'with space' \"it's\")", + "((a))", + "(((a)))", + "(a (b c) ((d)))", + "1\n2\n3", + "hello", + "😀 привет 世界", + "0", + "007", + "9223372036854775807", + "9223372036854775808", + "18446744073709551615", + "18446744073709551616", + "'multi\nline'", + ".dot", + "'a''b\"c`d'", + "(1: (2: 3 4) 5)", + "() ()", + "(* *)", + "(type: type type)", + }; + + /// Golden vectors shared with the Rust test suite: both ports must produce exactly these bytes. + public static TheoryData GoldenVectors => new() + { + { "() ((1 1))", false, false, "10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b" }, + { "() ((1 1))", false, true, "12 02 03 02 01 06 06 01 07 02 00 08 01 09" }, + { "() ((1 1))", true, false, "11 06 ff ff 06 00 04 07 00 08 09 00 04 0a" }, + { "() ((1 1))", true, true, "13 01 03 ff ff 01 06 02 00 07 01 08" }, + { "hi", true, false, "11 05 97 00 98 06 03 07 08 00 04 09" }, + { "hi", true, true, "13 00 02 03 03 98 97 01 06" }, + { "", false, false, "10 00" }, + { "", true, true, "13 00 00" }, + }; + + internal static IEnumerable AllOptions() + { + foreach (var externalReferences in new[] { false, true }) + { + foreach (var sequences in new[] { false, true }) + { + foreach (var progressiveWidths in new[] { false, true }) + { + yield return new BinaryLinoOptions(externalReferences, sequences, progressiveWidths); + } + } + } + } + + private static string Hex(byte[] bytes) => string.Join(" ", bytes.Select(value => value.ToString("x2"))); + + private static byte[] Unhex(string text) => + text.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(value => Convert.ToByte(value, 16)).ToArray(); + + private static IReadOnlyList Parse(string text) => LinoFormat.ParseDocument(text); + + [Theory] + [MemberData(nameof(GoldenVectors))] + public void GoldenVectorsAreStable(string text, bool externalReferences, bool sequences, string expected) + { + var binary = new BinaryLinoProtocol(new BinaryLinoOptions(externalReferences, sequences)); + Assert.Equal(expected, Hex(binary.Encode(Parse(text)))); + Assert.Equal(Parse(text), new BinaryLinoProtocol().Decode(Unhex(expected))); + } + + [Fact] + public void EveryOptionSetRoundTripsTheCorpus() + { + foreach (var text in Corpus) + { + var document = Parse(text); + foreach (var options in AllOptions()) + { + var binary = new BinaryLinoProtocol(options); + var bytes = binary.Encode(document); + Assert.InRange(bytes[0], 0x10, 0x1F); + Assert.True(document.SequenceEqual(binary.Decode(bytes)), $"{text} with {options}"); + } + } + } + + [Fact] + public void CanonicalTextRoundTripsTheCorpus() + { + foreach (var text in Corpus) + { + var document = Parse(text); + var canonical = LinoFormat.FormatDocument(document); + Assert.True(document.SequenceEqual(Parse(canonical)), $"{text} -> {canonical}"); + } + Assert.Equal("() ((1 1))", LinoFormat.FormatDocument(Parse("(()((1 1)))"))); + Assert.Equal("((1: 1 1)) ((1: 1 2))", LinoFormat.FormatDocument(Parse("((1: 1 1)) ((1: 1 2))"))); + } + + [Fact] + public void ParsedDocumentsMatchTheRustModel() + { + // The C# parser wraps single references in an extra group; the + // canonical model drops it so both ports decode to equal documents. + // Expected structures are the output of rust/examples/lino_structure_probe.rs. + var a = LinoFormat.Reference("a"); + LinoLink Group(LinoLink value) => LinoFormat.Link(null, new List { value }); + Assert.Equal(new[] { a }, Parse("a")); + Assert.Equal(new[] { a }, Parse("(a)")); + Assert.Equal(new[] { Group(a) }, Parse("((a))")); + Assert.Equal(new[] { Group(Group(a)) }, Parse("(((a)))")); + Assert.Equal(new[] { LinoFormat.Link("a", new List { LinoFormat.Reference("b") }) }, Parse("(a: (b))")); + Assert.Equal(new[] { LinoFormat.Reference("1"), LinoFormat.Reference("2"), LinoFormat.Reference("3") }, Parse("1\n2\n3")); + } + + [Fact] + public void ReferencesAreQuotedOnlyWhenNeeded() + { + Assert.Equal("plain", LinoFormat.FormatReference("plain")); + Assert.Equal("$x", LinoFormat.FormatReference("$x")); + Assert.Equal("''", LinoFormat.FormatReference("")); + Assert.Equal("'a b'", LinoFormat.FormatReference("a b")); + Assert.Equal("\"it's\"", LinoFormat.FormatReference("it's")); + Assert.Equal("`'\"`", LinoFormat.FormatReference("'\"")); + Assert.Equal("\"\"\"'\"`\"\"\"", LinoFormat.FormatReference("'\"`")); + foreach (var reference in new[] + { + "", "a b", "a:b", "(x)", "it's", "'\"", "'\"`", "tab\there", "a\nb", "'x", "x'", "'", "''", "'''", + "\"'`x`'\"", "a''''b\"\"`", "`'\"\"\"'''``", "(a) ''", + }) + { + var formatted = LinoFormat.FormatReference(reference); + var document = Parse(formatted); + Assert.Equal(formatted, LinoFormat.FormatDocument(document)); + Assert.Equal(new[] { LinoFormat.Reference(reference) }, document); + } + } + + [Fact] + public void EveryShortReferenceOverDelimitersRoundTrips() + { + var alphabet = new[] { '\'', '"', '`', 'a', ' ', '(', ')', ':' }; + var stack = new Stack(); + stack.Push(string.Empty); + while (stack.TryPop(out var reference)) + { + if (reference.Length > 0) + { + var formatted = LinoFormat.FormatReference(reference); + var document = Parse(formatted); + Assert.True( + document.Count == 1 && document[0].Equals(LinoFormat.Reference(reference)), + $"{reference} as {formatted}"); + } + if (reference.Length < 5) + { + foreach (var character in alphabet) + { + stack.Push(reference + character); + } + } + } + } + + [Fact] + public void WidthTiersFollowTheNumberOfLinks() + { + Assert.Equal(1, LinksPacket.AddressTier(0, false)); + Assert.Equal(1, LinksPacket.AddressTier(255, false)); + Assert.Equal(2, LinksPacket.AddressTier(256, false)); + Assert.Equal(2, LinksPacket.AddressTier(65_535, false)); + Assert.Equal(4, LinksPacket.AddressTier(65_536, false)); + Assert.Equal(4, LinksPacket.AddressTier(uint.MaxValue, false)); + Assert.Equal(8, LinksPacket.AddressTier((ulong)uint.MaxValue + 1, false)); + // External references take the top bit, halving every range. + Assert.Equal(1, LinksPacket.AddressTier(127, true)); + Assert.Equal(2, LinksPacket.AddressTier(128, true)); + Assert.Equal(2, LinksPacket.AddressTier(32_767, true)); + Assert.Equal(4, LinksPacket.AddressTier(32_768, true)); + Assert.Equal(ulong.MaxValue, LinksPacket.InternalCapacity(8, false)); + Assert.Equal((ulong)long.MaxValue, LinksPacket.InternalCapacity(8, true)); + Assert.Equal(127UL, LinksPacket.ExternalCapacity(1)); + } + + [Fact] + public void ExternalReferencesMatchPlatformDataHybrid() + { + foreach (var width in LinksPacket.Widths) + { + foreach (var value in new ulong[] { 0, 1, 2, 100, LinksPacket.ExternalCapacity(width) }) + { + var raw = LinksPacket.EncodeExternal(value, width); + Assert.True(LinksPacket.TryDecodeExternal(raw, width, out var decoded)); + Assert.Equal(value, decoded); + } + Assert.False(LinksPacket.TryDecodeExternal(LinksPacket.InternalCapacity(width, true), width, out _)); + } + Assert.Equal(0xFFUL, LinksPacket.EncodeExternal(1, 1)); + Assert.Equal(0x80UL, LinksPacket.EncodeExternal(0, 1)); + foreach (var value in new uint[] { 0, 1, 5, 1000, int.MaxValue }) + { + Assert.Equal((ulong)new Hybrid(value, isExternal: true).Value, LinksPacket.EncodeExternal(value, 4)); + } + } + + private static string ManyLinks(int count) => + string.Join("\n", Enumerable.Range(0, count).Select(index => $"(n{index} m{index})")); + + [Fact] + public void UniformWidthGrowsPast256Addresses() + { + var small = LinoMapping.EncodeDocument(Parse(ManyLinks(3))); + Assert.True(small.LastAddress <= 255); + Assert.Equal(1, small.MinWidth); + + var document = Parse(ManyLinks(400)); + var uniform = LinoMapping.EncodeDocument(document); + Assert.True(uniform.LastAddress > 255); + Assert.Equal(2, uniform.MinWidth); + var uniformBytes = uniform.ToBytes(); + const int header = 1 + 2; // header byte + two-byte LEB128 count + Assert.Equal(header + uniform.Doublets.Count * 4, uniformBytes.Length); + + var progressive = LinoMapping.EncodeDocument(document, new BinaryLinoOptions().WithProgressiveWidths()); + Assert.Equal(1, progressive.MinWidth); + var progressiveBytes = progressive.ToBytes(); + Assert.True(progressiveBytes.Length < uniformBytes.Length); + foreach (var bytes in new[] { uniformBytes, progressiveBytes }) + { + Assert.Equal(document, LinoMapping.DecodeDocument(LinksPacket.FromBytes(bytes))); + } + } + + [Fact] + public void LargeExternalValuesRaiseTheMinimumWidth() + { + var options = new BinaryLinoOptions(ExternalReferences: true, ProgressiveWidths: true); + Assert.Equal(2, LinoMapping.EncodeDocument(Parse("(1000 1)"), options).MinWidth); + Assert.Equal(1, LinoMapping.EncodeDocument(Parse("(100 1)"), options).MinWidth); + // Beyond 63 bits the number falls back to in-band unary links. + Assert.True(LinoMapping.EncodeDocument(Parse("18446744073709551615"), options).Doublets.Count > 60); + } + + [Fact] + public void HandBuiltPacketsDecode() + { + // One doublet (One One) is 2^1 in unary, so `(Number 7)` is the number 2. + var packet = new LinksPacket(); + packet.Doublets.Add((PacketReference.Internal(1), PacketReference.Internal(1))); + packet.Doublets.Add((PacketReference.Internal(2), PacketReference.Internal(6))); + packet.Doublets.Add((PacketReference.Internal(7), PacketReference.Null)); + packet.Doublets.Add((PacketReference.Internal(4), PacketReference.Internal(8))); + var bytes = packet.ToBytes(); + Assert.Equal("10 04 01 01 02 06 07 00 04 08", Hex(bytes)); + Assert.Equal("2", LinoFormat.FormatDocument(new BinaryLinoProtocol().Decode(bytes))); + } + + private static LinoProtocolErrorKind DecodeError(byte[] bytes, DecodeLimits? limits = null) + { + var protocol = new BinaryLinoProtocol { Limits = limits ?? DecodeLimits.Default }; + return Assert.Throws(() => protocol.Decode(bytes)).Kind; + } + + [Theory] + [InlineData("")] + [InlineData("20 00")] // unknown version nibble + [InlineData("10 01 06 00")] // refers to itself + [InlineData("10 01 07 00")] // refers forward + [InlineData("10 02 00")] // truncated + [InlineData("10 00 00")] // trailing byte + [InlineData("10 ff ff ff ff ff ff ff ff ff 7f")] // LEB128 overflow + [InlineData("10 01 01 01")] // root is not a list + [InlineData("10 02 00 02 04 06")] // marker 2 used as a value + [InlineData("10 01 04 02")] // a bare marker in a list chain + public void MalformedPacketsAreRejected(string hex) + { + Assert.Equal(LinoProtocolErrorKind.Malformed, DecodeError(Unhex(hex))); + } + + [Fact] + public void InvalidCodePointsAreRejected() + { + // (String (0x110000)) is not a valid code point. + var packet = new LinksPacket { ExternalReferences = true, MinWidth = 4 }; + packet.Doublets.Add((PacketReference.External(0x11_0000), PacketReference.Null)); + packet.Doublets.Add((PacketReference.Internal(3), PacketReference.Internal(6))); + packet.Doublets.Add((PacketReference.Internal(7), PacketReference.Null)); + packet.Doublets.Add((PacketReference.Internal(4), PacketReference.Internal(8))); + Assert.Equal(LinoProtocolErrorKind.Malformed, DecodeError(packet.ToBytes())); + } + + [Fact] + public void HostilePacketsHitLimits() + { + Assert.Equal(LinoProtocolErrorKind.LimitExceeded, DecodeError(Unhex("10 09"), new DecodeLimits { MaxLinks = 8 })); + + // A doubling chain `d(k) = (d(k-1) d(k-1))` expands to 2^k nodes. + var bomb = new LinksPacket(); + bomb.Doublets.Add((PacketReference.Null, PacketReference.Null)); + for (ulong address = 6; address < 60; address++) + { + bomb.Doublets.Add((PacketReference.Internal(address), PacketReference.Internal(address))); + } + var last = 5 + (ulong)bomb.Doublets.Count; + bomb.Doublets.Add((PacketReference.Internal(last), PacketReference.Null)); + bomb.Doublets.Add((PacketReference.Internal(4), PacketReference.Internal(last + 1))); + Assert.Equal(LinoProtocolErrorKind.LimitExceeded, DecodeError(bomb.ToBytes(), new DecodeLimits { MaxNodes = 1 << 12 })); + + var deep = string.Concat(Enumerable.Repeat("(y ", 40)) + "x" + new string(')', 40); + var bytes = new BinaryLinoProtocol().Encode(Parse(deep)); + Assert.Equal(LinoProtocolErrorKind.LimitExceeded, DecodeError(bytes, new DecodeLimits { MaxDepth = 10 })); + } + + [Fact] + public void TextMessagesAreDotStuffed() + { + var document = Parse(".dot\n'first\n.second'\n(a b)"); + using var wire = new MemoryStream(); + new TextLinoProtocol().WriteDocument(wire, document); + Assert.Equal("..dot\n'first\n..second'\na b\n.\n", System.Text.Encoding.UTF8.GetString(wire.ToArray())); + var reader = LinoStreamReader.FromBytes(wire.ToArray()); + Assert.Equal(document, new TextLinoProtocol().ReadDocument(reader)); + Assert.Null(new TextLinoProtocol().ReadDocument(reader)); + } + + [Fact] + public void TextMessagesAcceptCrlfAndRejectTruncation() + { + var crlf = LinoStreamReader.FromBytes("() ((1 1))\r\n.\r\n"u8.ToArray()); + Assert.Equal("() ((1 1))", LinoFormat.FormatDocument(new TextLinoProtocol().ReadDocument(crlf)!)); + + Assert.Empty(new TextLinoProtocol().ReadDocument(LinoStreamReader.FromBytes(".\n"u8.ToArray()))!); + + var truncated = LinoStreamReader.FromBytes("() ((1 1))\n"u8.ToArray()); + var error = Assert.Throws(() => new TextLinoProtocol().ReadDocument(truncated)); + Assert.Equal(LinoProtocolErrorKind.Malformed, error.Kind); + + var tiny = new TextLinoProtocol { Limits = new DecodeLimits { MaxTextBytes = 8 } }; + var longMessage = LinoStreamReader.FromBytes(System.Text.Encoding.UTF8.GetBytes(new string('a', 100) + "\n.\n")); + Assert.Equal(LinoProtocolErrorKind.LimitExceeded, Assert.Throws(() => tiny.ReadDocument(longMessage)).Kind); + + var invalidUtf8 = LinoStreamReader.FromBytes(new byte[] { 0xC3, 0x28, (byte)'\n', (byte)'.', (byte)'\n' }); + Assert.Equal(LinoProtocolErrorKind.Malformed, Assert.Throws(() => new TextLinoProtocol().ReadDocument(invalidUtf8)).Kind); + } + + [Fact] + public void ProtocolsAreDetectedPerMessage() + { + var document = Parse("() ((1 1))"); + var options = new BinaryLinoOptions().WithExternalReferences().WithSequences(); + using var wire = new MemoryStream(); + new TextLinoProtocol().WriteDocument(wire, document); + new BinaryLinoProtocol(options).WriteDocument(wire, document); + new TextLinoProtocol().WriteDocument(wire, document); + + var reader = LinoStreamReader.FromBytes(wire.ToArray()); + var formats = new List(); + while (LinoProtocols.ReadAnyDocument(reader) is { } message) + { + Assert.Equal(document, message.Document); + formats.Add(message.Format); + } + Assert.Equal(new[] { MessageFormat.Text, MessageFormat.Binary(options), MessageFormat.Text }, formats); + } + + [Fact] + public void DeeplyNestedTextParsesInLinearTime() + { + // links-notation before 0.21.3 took exponential time in the nesting + // depth (link-foundation/links-notation#314), so one short message + // could stall a server thread. Depth 40 took minutes there. + static string Nested(int depth) => new string('(', depth) + "a" + new string(')', depth); + var started = System.Diagnostics.Stopwatch.StartNew(); + var link = LinoFormat.ParseDocument(Nested(40)).Single(); + var depth = 1; + while (link.Values is { } values) + { + link = values[0]; + depth++; + } + Assert.Equal(40, depth); + var error = Assert.Throws(() => LinoFormat.ParseDocument(Nested(100_000))); + Assert.Equal(LinoProtocolErrorKind.InvalidLino, error.Kind); + Assert.True(started.Elapsed < TimeSpan.FromSeconds(10), started.Elapsed.ToString()); + } +} diff --git a/examples/lino-structure-probe/LinoStructureProbe.csproj b/examples/lino-structure-probe/LinoStructureProbe.csproj new file mode 100644 index 0000000..e3a999f --- /dev/null +++ b/examples/lino-structure-probe/LinoStructureProbe.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + enable + enable + lino-structure-probe + + + + + + + diff --git a/examples/lino-structure-probe/Program.cs b/examples/lino-structure-probe/Program.cs new file mode 100644 index 0000000..ae805d5 --- /dev/null +++ b/examples/lino-structure-probe/Program.cs @@ -0,0 +1,28 @@ +// Prints how the C# links-notation parser structures the inputs the Rust +// protocol tests rely on, so both ports can be checked for agreement: +// `dotnet run --project examples/lino-structure-probe -- "((a))"`. The Rust +// counterpart is `cargo run --example lino_structure_probe -- "((a))"`. +using Link.Foundation.Links.Notation; + +static string Show(Link link) => + link.Values == null + ? $"Ref({link.Id ?? ""})" + : $"Link{{{(link.Id == null ? "None" : link.Id)},[{string.Join(",", link.Values.Select(Show))}]}}"; + +var inputs = args.Length > 0 ? args : new[] { + "() ((1 1))", "(a b)", "a b", "a", "(a)", "((a))", "(((a)))", "(a:)", "(a: b c)", + "((1: 1 1)) ((1: 1 2))", "a\nb", "(a b)\n(c d)", "(a b) (c d)", "''", "\"\"\"'\"`\"\"\"", + "'x'''", "(name: 'with space' \"it's\")", "1\n2\n3", "() ()", "(a (b c) ((d)))", "'multi\nline'", "a\r", +}; +foreach (var input in inputs) +{ + try + { + var links = new Parser().Parse(input); + Console.WriteLine($"{input.Replace("\n", "\\n").Replace("\r", "\\r")} => [{string.Join(", ", links.Select(Show))}]"); + } + catch (Exception e) + { + Console.WriteLine($"{input.Replace("\n", "\\n")} => ERR {e.GetType().Name}: {e.Message.Split('\n')[0]}"); + } +} diff --git a/rust/examples/lino_structure_probe.rs b/rust/examples/lino_structure_probe.rs new file mode 100644 index 0000000..ffabde6 --- /dev/null +++ b/rust/examples/lino_structure_probe.rs @@ -0,0 +1,29 @@ +//! Prints how links-notation structures LiNo text, in the same notation as +//! `examples/lino-structure-probe`, so the Rust and C# parsers can +//! be compared line by line: `cargo run --example lino_structure_probe -- 'a b'`. + +use links_notation::{parse_lino_to_links, LiNo}; + +fn show(link: &LiNo) -> String { + match link { + LiNo::Ref(reference) => format!("Ref({reference})"), + LiNo::Link { id, values } => format!( + "Link{{{},[{}]}}", + id.as_deref().unwrap_or("None"), + values.iter().map(show).collect::>().join(",") + ), + } +} + +fn main() { + for input in std::env::args().skip(1) { + let shown = input.replace('\n', "\\n").replace('\r', "\\r"); + match parse_lino_to_links(&input) { + Ok(links) => println!( + "{shown} => [{}]", + links.iter().map(show).collect::>().join(", ") + ), + Err(error) => println!("{shown} => ERR {error}"), + } + } +} From d81b8efe09692026f51c2db24e41de10c2b831b8 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 18:39:09 +0000 Subject: [PATCH 08/44] test(csharp): cover the LiNo TCP server and client Mirrors rust/tests/protocol_tcp_tests.rs: CRUD over text and all eight binary option sets, identical replies across protocols, remote errors, a raw CRLF text session, malformed headers, protocol restriction, concurrent clients and shutdown. --- .../Protocol/LinksServerTests.cs | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs new file mode 100644 index 0000000..2a9f83f --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs @@ -0,0 +1,195 @@ +using System.Net.Sockets; +using System.Text; +using Foundation.Data.Doublets.Cli.Protocol; + +namespace Foundation.Data.Doublets.Cli.Tests.Protocol; + +/// LiNo substitution operations over TCP (issue #105). +public sealed class LinksServerTests +{ + private sealed class RunningServer : IDisposable + { + private readonly string _databaseFilename = Path.GetTempFileName(); + private readonly LinksServer _server; + private readonly Thread _thread; + private Exception? _failure; + + public RunningServer(LinksServerOptions? options = null) + { + _server = LinksServer.Bind("127.0.0.1", 0, options); + _thread = new Thread(() => + { + try + { + using var links = new NamedTypesDecorator(_databaseFilename); + _server.Serve(links); + } + catch (Exception error) + { + _failure = error; + } + }) { IsBackground = true }; + _thread.Start(); + } + + public int Port => _server.LocalEndPoint.Port; + + public LinksClient Client(ILinoProtocol protocol) => LinksClient.Connect("127.0.0.1", Port, protocol); + + public void Stop() + { + _server.Shutdown(); + Assert.True(_thread.Join(TimeSpan.FromSeconds(30)), "the server did not stop"); + Assert.Null(_failure); + } + + public void Dispose() + { + Stop(); + File.Delete(_databaseFilename); + File.Delete(NamedTypesDecorator.MakeNamesDatabaseFilename(_databaseFilename)); + } + } + + private static IEnumerable Protocols() => + LinoProtocolCodecTests.AllOptions() + .Select(options => (ILinoProtocol)new BinaryLinoProtocol(options)) + .Prepend(new TextLinoProtocol()); + + private static TcpClient RawConnection(RunningServer server) + { + return new TcpClient("127.0.0.1", server.Port) { ReceiveTimeout = 10_000 }; + } + + [Fact] + public void CrudWorksOverEveryProtocol() + { + foreach (var protocol in Protocols()) + { + using var server = new RunningServer(); + using var client = server.Client(protocol); + + // Create. + Assert.Equal("() ((1: 1 1))", client.QueryText("() ((1 1))")); + Assert.Equal("() ((2: 2 2))", client.QueryText("() ((2 2))")); + // Read. + Assert.Equal("(1: 1 1)\n(2: 2 2)", client.QueryText("")); + Assert.Equal("((1: 1 1)) ((1: 1 1))", client.QueryText("((1: 1 1)) ((1: 1 1))")); + // Update. + Assert.Equal("((1: 1 1)) ((1: 1 2))", client.QueryText("((1: 1 1)) ((1: 1 2))")); + // Delete. + Assert.Equal("((1: 1 2)) ()", client.QueryText("((1: 1 2)) ()")); + Assert.Equal("(2: 2 2)", client.QueryText("")); + } + } + + [Fact] + public void TextAndBinaryRepliesAreIdenticalDocuments() + { + using var server = new RunningServer(new LinksServerOptions { AutoCreateMissingReferences = true }); + using var text = server.Client(new TextLinoProtocol()); + using var binary = server.Client(new BinaryLinoProtocol(new BinaryLinoOptions().WithExternalReferences().WithSequences())); + text.Query("() ((child: father mother))"); + binary.Query("() (('two words': 'it''s' \"x\"))"); + Assert.Equal(text.Query(""), binary.Query("")); + var listing = text.QueryText(""); + Assert.Contains("(child: father mother)", listing); + Assert.Contains("('two words': \"it's\" x)", listing); + } + + [Fact] + public void QueryErrorsComeBackAsRemoteErrors() + { + using var server = new RunningServer(); + foreach (var protocol in Protocols()) + { + using var client = server.Client(protocol); + var error = Assert.Throws(() => client.Query("((99: 1 1)) ()")); + Assert.Equal(LinoProtocolErrorKind.Remote, error.Kind); + Assert.False(string.IsNullOrEmpty(error.Detail)); + // The connection stays usable after an error reply. + Assert.Equal("", client.QueryText("")); + } + } + + [Fact] + public void ARawTextSessionWorksWithCrlf() + { + using var server = new RunningServer(); + using var connection = RawConnection(server); + var stream = connection.GetStream(); + stream.Write("() ((1 1))\r\n.\r\n"u8); + using var reader = new StreamReader(stream, new UTF8Encoding(false)); + Assert.Equal("() ((1: 1 1))", reader.ReadLine()); + Assert.Equal(".", reader.ReadLine()); + + stream.Write(".\n"u8); + Assert.Equal("(1: 1 1)", reader.ReadLine()); + } + + [Fact] + public void MalformedMessagesGetAnErrorAndTheConnectionCloses() + { + using var server = new RunningServer(); + using var connection = RawConnection(server); + var stream = connection.GetStream(); + // A binary header announcing more links than the default limit allows. + stream.Write(new byte[] { 0x10, 0xff, 0xff, 0xff, 0xff, 0x0f }); + using var reader = new StreamReader(stream, new UTF8Encoding(false)); + var reply = reader.ReadToEnd(); + Assert.StartsWith("(error: ", reply, StringComparison.Ordinal); + Assert.EndsWith("\n.\n", reply, StringComparison.Ordinal); + } + + [Fact] + public void ServersCanRestrictTheAcceptedProtocol() + { + using var server = new RunningServer(new LinksServerOptions { Accept = AcceptedProtocols.Binary }); + using var text = server.Client(new TextLinoProtocol()); + var error = Assert.Throws(() => text.Query("() ((1 1))")); + Assert.Equal(LinoProtocolErrorKind.Remote, error.Kind); + using var binary = server.Client(new BinaryLinoProtocol()); + Assert.Equal("() ((1: 1 1))", binary.QueryText("() ((1 1))")); + } + + [Fact] + public async Task ConcurrentClientsShareOneStore() + { + using var server = new RunningServer(); + var workers = Enumerable.Range(0, 4).Select(worker => Task.Run(() => + { + using var client = server.Client(worker % 2 == 0 ? new TextLinoProtocol() : new BinaryLinoProtocol()); + for (var item = 0; item < 5; item++) + { + // The C# query processor does not report named creations as + // changes (unlike the Rust one), so only success is checked here. + var name = $"w{worker}i{item}"; + client.Query($"() (({name}: {name} {name}))"); + } + })).ToArray(); + await Task.WhenAll(workers); + using var reader = server.Client(new TextLinoProtocol()); + var listing = reader.QueryText("").Split('\n'); + Assert.Equal(20, listing.Length); + Assert.Contains("(w3i4: w3i4 w3i4)", listing); + } + + [Fact] + public void ShutdownStopsTheServer() + { + using var server = new RunningServer(); + using var client = server.Client(new TextLinoProtocol()); + client.Query("() ((1 1))"); + server.Stop(); + Assert.ThrowsAny(() => client.Query("")); + } + + [Fact] + public void EndPointsAreParsed() + { + Assert.Equal(("127.0.0.1", 8080), LinksServer.ParseEndPoint("127.0.0.1:8080")); + Assert.Equal(("::1", 0), LinksServer.ParseEndPoint("[::1]:0")); + Assert.Throws(() => LinksServer.ParseEndPoint("localhost")); + Assert.Throws(() => LinksServer.ParseEndPoint("localhost:70000")); + } +} From c2052982ba795aea8cbd6ffa2f20935888b5bc9c Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 18:45:40 +0000 Subject: [PATCH 09/44] feat(csharp): add clink --serve and --connect The C# CLI now serves a database over TCP and queries a served one, matching the Rust options: --serve, --connect, --protocol and the binary decorators --external-references, --sequences and --progressive-widths. examples/tcp/run-interop.sh drives a server written in one language with a client written in the other. --- .../Protocol/CliTcpIntegrationTests.cs | 138 +++++++++++++++++ .../Foundation.Data.Doublets.Cli/Program.cs | 139 ++++++++++++++++++ examples/tcp/README.md | 54 +++++++ examples/tcp/run-interop.sh | 48 ++++++ 4 files changed, 379 insertions(+) create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs create mode 100644 examples/tcp/README.md create mode 100755 examples/tcp/run-interop.sh diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs new file mode 100644 index 0000000..5cb68dd --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs @@ -0,0 +1,138 @@ +using System.Diagnostics; + +namespace Foundation.Data.Doublets.Cli.Tests.Protocol; + +/// clink --serve and clink --connect end to end (issue #105). +public sealed class CliTcpIntegrationTests : IDisposable +{ + private const string Banner = "clink server listening on "; + private static readonly string Clink = Path.Combine(AppContext.BaseDirectory, "clink.dll"); + private readonly string _directory = Directory.CreateTempSubdirectory("clink-tcp-").FullName; + + public void Dispose() => Directory.Delete(_directory, recursive: true); + + private static ProcessStartInfo StartInfo(IEnumerable arguments) + { + var info = new ProcessStartInfo("dotnet") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + info.ArgumentList.Add(Clink); + foreach (var argument in arguments) + { + info.ArgumentList.Add(argument); + } + return info; + } + + private static async Task<(int ExitCode, string Stdout, string Stderr)> RunAsync(params string[] arguments) + { + using var process = Process.Start(StartInfo(arguments))!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + await process.WaitForExitAsync(timeout.Token); + return (process.ExitCode, (await stdout).Replace("\r\n", "\n"), await stderr); + } + + private sealed class Server : IDisposable + { + private readonly Process _process; + + private Server(Process process, string address) + { + _process = process; + Address = address; + } + + public string Address { get; } + + public static async Task StartAsync(string database, params string[] extra) + { + var process = Process.Start(StartInfo(new[] { "--db", database, "--serve", "127.0.0.1:0" }.Concat(extra)))!; + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var line = await process.StandardOutput.ReadLineAsync(timeout.Token); + if (line is null || !line.StartsWith(Banner, StringComparison.Ordinal)) + { + process.Kill(entireProcessTree: true); + process.Dispose(); + throw new InvalidOperationException($"unexpected banner '{line}'"); + } + return new Server(process, line[Banner.Length..]); + } + + public async Task StdoutAsync(params string[] arguments) + { + var (exitCode, stdout, stderr) = await RunAsync(new[] { "--connect", Address }.Concat(arguments).ToArray()); + Assert.True(exitCode == 0, $"{string.Join(' ', arguments)}: {stderr}"); + return stdout; + } + + public void Dispose() + { + _process.Kill(entireProcessTree: true); + _process.WaitForExit(); + _process.Dispose(); + } + } + + [Fact] + public async Task ClientsQueryAServedDatabaseOverBothProtocols() + { + using var server = await Server.StartAsync(Path.Combine(_directory, "served.links")); + + Assert.Equal("() ((1: 1 1))\n", await server.StdoutAsync("() ((1 1))")); + Assert.Equal("() ((2: 2 2))\n", await server.StdoutAsync("--protocol", "binary", "() ((2 2))")); + Assert.Equal( + "((1: 1 1)) ((1: 1 2))\n", + await server.StdoutAsync("--external-references", "--sequences", "((1: 1 1)) ((1: 1 2))")); + Assert.Equal("(1: 1 2)\n(2: 2 2)\n", await server.StdoutAsync()); + Assert.Equal( + "((1: 1 2)) ((1: 1 2))\n", + await server.StdoutAsync("--progressive-widths", "--query", "((1: 1 2)) ((1: 1 2))")); + // Deleting 2 also deletes 1, which refers to it, exactly like the Rust port. + Assert.Equal("((2: 2 2)) ()\n((1: 1 2)) ()\n", await server.StdoutAsync("--protocol", "binary", "((2: 2 2)) ()")); + Assert.Equal("", await server.StdoutAsync("--protocol", "text")); + + var failure = await RunAsync("--connect", server.Address, "((99: 1 1)) ()"); + Assert.NotEqual(0, failure.ExitCode); + Assert.Contains("server error", failure.Stderr, StringComparison.Ordinal); + } + + [Fact] + public async Task ServedChangesArePersisted() + { + var database = Path.Combine(_directory, "served.links"); + using (var server = await Server.StartAsync(database, "--auto-create-missing-references")) + { + await server.StdoutAsync("() ((child: father mother))"); + } + var (exitCode, stdout, _) = await RunAsync("--db", database, "--after"); + Assert.Equal(0, exitCode); + Assert.Contains("(child: father mother)", stdout, StringComparison.Ordinal); + } + + [Theory] + [InlineData("--serve", "127.0.0.1:0", "--connect", "127.0.0.1:1")] + [InlineData("--connect", "127.0.0.1:1", "--protocol", "text", "--sequences")] + [InlineData("--connect", "127.0.0.1:1", "--protocol", "udp")] + [InlineData("--serve", "127.0.0.1:0", "--protocol", "udp")] + [InlineData("--serve", "127.0.0.1:0", "() ((1 1))")] + public async Task InvalidCombinationsAreRejected(params string[] arguments) + { + var (exitCode, _, stderr) = await RunAsync(new[] { "--db", Path.Combine(_directory, "x.links") }.Concat(arguments).ToArray()); + Assert.NotEqual(0, exitCode); + Assert.False(string.IsNullOrWhiteSpace(stderr)); + } + + [Fact] + public async Task HelpListsTheTcpOptions() + { + var (_, stdout, _) = await RunAsync("--help"); + foreach (var option in new[] { "--serve", "--connect", "--protocol", "--external-references", "--sequences", "--progressive-widths" }) + { + Assert.Contains(option, stdout, StringComparison.Ordinal); + } + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli/Program.cs b/csharp/Foundation.Data.Doublets.Cli/Program.cs index 04bf1ed..9d24dbc 100644 --- a/csharp/Foundation.Data.Doublets.Cli/Program.cs +++ b/csharp/Foundation.Data.Doublets.Cli/Program.cs @@ -1,5 +1,6 @@ using System.CommandLine; using Foundation.Data.Doublets.Cli; +using Foundation.Data.Doublets.Cli.Protocol; using Platform.Data; using Platform.Data.Doublets; @@ -176,6 +177,39 @@ DefaultValueFactory = _ => false }; +var serveOption = new Option("--serve") +{ + Description = "Serve the database over TCP at host:port (port 0 picks a free port)" +}; + +var connectOption = new Option("--connect") +{ + Description = "Send the query to a clink server at host:port and print the reply" +}; + +var protocolOption = new Option("--protocol") +{ + Description = "Wire protocol: 'text' or 'binary' with --connect (default: text), 'any', 'text' or 'binary' with --serve (default: any)" +}; + +var externalReferencesOption = new Option("--external-references") +{ + Description = "Binary protocol: send numbers as external references (implies --protocol binary)", + DefaultValueFactory = _ => false +}; + +var sequencesOption = new Option("--sequences") +{ + Description = "Binary protocol: send lists as a variable-length sequence section (implies --protocol binary)", + DefaultValueFactory = _ => false +}; + +var progressiveWidthsOption = new Option("--progressive-widths") +{ + Description = "Binary protocol: grow reference widths with the address (implies --protocol binary)", + DefaultValueFactory = _ => false +}; + var rootCommand = new RootCommand("LiNo CLI Tool for managing links data store"); rootCommand.Options.Add(dbOption); rootCommand.Options.Add(queryOption); @@ -207,6 +241,12 @@ rootCommand.Options.Add(listBranchesOption); rootCommand.Options.Add(listTagsOption); rootCommand.Options.Add(logOption); +rootCommand.Options.Add(serveOption); +rootCommand.Options.Add(connectOption); +rootCommand.Options.Add(protocolOption); +rootCommand.Options.Add(externalReferencesOption); +rootCommand.Options.Add(sequencesOption); +rootCommand.Options.Add(progressiveWidthsOption); rootCommand.SetAction( parseResult => @@ -241,6 +281,13 @@ var listBranches = parseResult.GetValue(listBranchesOption); var listTags = parseResult.GetValue(listTagsOption); var showLog = parseResult.GetValue(logOption); + var serveAddress = parseResult.GetValue(serveOption); + var connectAddress = parseResult.GetValue(connectOption); + var protocolName = parseResult.GetValue(protocolOption)?.Trim(); + var binaryOptions = new BinaryLinoOptions( + parseResult.GetValue(externalReferencesOption), + parseResult.GetValue(sequencesOption), + parseResult.GetValue(progressiveWidthsOption)); var triggerCommandCount = new[] { always, once, never }.Count(value => value); if (triggerCommandCount > 1) @@ -249,6 +296,18 @@ return 1; } + if (serveAddress is not null && connectAddress is not null) + { + Console.Error.WriteLine("--serve and --connect cannot be used together."); + return 1; + } + + if (connectAddress is not null) + { + var clientQuery = !string.IsNullOrWhiteSpace(queryOptionValue) ? queryOptionValue : queryArgumentValue; + return RunClient(connectAddress, protocolName, binaryOptions, clientQuery); + } + var vcRequested = vc || !string.IsNullOrWhiteSpace(vcFile) || !string.IsNullOrWhiteSpace(branchName) @@ -484,9 +543,49 @@ int RunCli() return 0; } + if (serveAddress is not null) + { + return RunServer(); + } + return RunQueryPipeline(); } + // `--serve`: serves decoratedLinks until the process is interrupted. + int RunServer() + { + if (!string.IsNullOrWhiteSpace(queryOptionValue) || !string.IsNullOrWhiteSpace(queryArgumentValue)) + { + Console.Error.WriteLine("--serve does not take a query; send queries with --connect."); + return 1; + } + AcceptedProtocols accept; + switch (protocolName?.ToLowerInvariant()) + { + case null or "any": accept = AcceptedProtocols.Any; break; + case "text": accept = AcceptedProtocols.Text; break; + case "binary": accept = AcceptedProtocols.Binary; break; + default: + Console.Error.WriteLine($"Invalid --protocol value '{protocolName}'. Use 'any', 'text' or 'binary'."); + return 1; + } + using var server = LinksServer.Bind(serveAddress, new LinksServerOptions + { + Trace = trace, + AutoCreateMissingReferences = autoCreateMissingReferences, + Accept = accept, + }); + Console.CancelKeyPress += (_, eventArgs) => + { + eventArgs.Cancel = true; + server.Shutdown(); + }; + Console.WriteLine($"clink server listening on {server.LocalEndPoint}"); + Console.Out.Flush(); + server.Serve(decoratedLinks); + return 0; + } + bool TryResolveSequence(VersionControlDecorator vc, string point, out long sequence) { sequence = 0; @@ -614,6 +713,46 @@ int RunQueryPipeline() return rootCommand.Parse(args).Invoke(); +// `--connect`: sends the query to a server and prints the reply. +// Any binary option implies `--protocol binary`. +static int RunClient(string address, string? protocolName, BinaryLinoOptions binaryOptions, string query) +{ + ILinoProtocol protocol; + switch (protocolName?.ToLowerInvariant()) + { + case null: + protocol = binaryOptions == default ? new TextLinoProtocol() : new BinaryLinoProtocol(binaryOptions); + break; + case "text" when binaryOptions != default: + Console.Error.WriteLine("Binary protocol options require --protocol binary."); + return 1; + case "text": + protocol = new TextLinoProtocol(); + break; + case "binary": + protocol = new BinaryLinoProtocol(binaryOptions); + break; + default: + Console.Error.WriteLine($"Invalid --protocol value '{protocolName}'. Use 'text' or 'binary'."); + return 1; + } + try + { + using var client = LinksClient.Connect(address, protocol); + var reply = client.QueryText(query); + if (reply.Length > 0) + { + Console.WriteLine(reply); + } + return 0; + } + catch (Exception error) when (error is LinoProtocolException or System.Net.Sockets.SocketException or IOException or ArgumentException) + { + Console.Error.WriteLine($"Error: {error.Message}"); + return 1; + } +} + static void PrintAllLinks(INamedTypesLinks links) { LinoDatabaseOutput.WriteDatabase(links, Console.Out); diff --git a/examples/tcp/README.md b/examples/tcp/README.md new file mode 100644 index 0000000..6fee685 --- /dev/null +++ b/examples/tcp/README.md @@ -0,0 +1,54 @@ +# LiNo over TCP — examples + +Runnable demonstrations of `clink --serve` and `clink --connect`, added for +issue [#105](https://github.com/link-foundation/link-cli/issues/105). + +## Scripts + +| File | What it shows | +|------|---------------| +| `run-interop.sh` | One port serves a database and the other port queries it, over the text protocol and every binary option | +| `README.md` | This file | + +```bash +./examples/tcp/run-interop.sh # Rust server, C# client +./examples/tcp/run-interop.sh csharp # C# server, Rust client +``` + +Both directions print the same thing: + +``` +server: clink server listening on 127.0.0.1:40123 +[text] () ((1: 1 1)) +[--protocol binary] () ((2: 2 2)) +[--external-references] () ((3: 3 3)) +[--sequences] () ((4: 4 4)) +[--progressive-widths] () ((5: 5 5)) +[--external-references --sequences --progressive-widths] () ((6: 6 6)) +listing: +(1: 1 1) +(2: 2 2) +(3: 3 3) +(4: 4 4) +(5: 5 5) +(6: 6 6) +``` + +## Doing it by hand + +```bash +clink --db served.links --serve 127.0.0.1:7777 # terminal 1 +clink --connect 127.0.0.1:7777 '() ((1 1))' # terminal 2, text protocol +clink --connect 127.0.0.1:7777 --protocol binary # list every link over binary +``` + +Text messages end with a line holding a single `.`, so you can also talk to +the server with `nc 127.0.0.1 7777`: + +``` +() ((2 2)) +. +``` + +The wire formats are described in +[`docs/case-studies/issue-105`](../../docs/case-studies/issue-105/README.md). diff --git a/examples/tcp/run-interop.sh b/examples/tcp/run-interop.sh new file mode 100755 index 0000000..47e2ddf --- /dev/null +++ b/examples/tcp/run-interop.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Serves a database with one port and queries it with the other, over the +# text protocol and every binary protocol option (issue #105). +# +# Usage: +# ./examples/tcp/run-interop.sh # Rust server, C# client +# ./examples/tcp/run-interop.sh csharp # C# server, Rust client + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +work_dir="$(mktemp -d)" +server_pid="" +cleanup() { + if [[ -n "$server_pid" ]]; then kill "$server_pid" 2>/dev/null || true; fi + rm -rf "$work_dir" +} +trap cleanup EXIT + +cargo build --manifest-path "$repo_root/rust/Cargo.toml" --quiet +dotnet build "$repo_root/csharp/Foundation.Data.Doublets.Cli" --nologo --verbosity quiet >/dev/null +rust_clink=("$repo_root/rust/target/debug/clink") +csharp_clink=(dotnet "$repo_root/csharp/Foundation.Data.Doublets.Cli/bin/Debug/net10.0/clink.dll") + +if [[ "${1:-rust}" == "csharp" ]]; then + server=("${csharp_clink[@]}") client=("${rust_clink[@]}") +else + server=("${rust_clink[@]}") client=("${csharp_clink[@]}") +fi + +"${server[@]}" --db "$work_dir/served.links" --serve 127.0.0.1:0 >"$work_dir/banner" & +server_pid=$! +for _ in $(seq 100); do + grep -q "listening" "$work_dir/banner" && break + sleep 0.1 +done +address="$(sed -n 's/^clink server listening on //p' "$work_dir/banner")" +echo "server: $(head -n 1 "$work_dir/banner")" + +index=0 +for flags in "" "--protocol binary" "--external-references" "--sequences" \ + "--progressive-widths" "--external-references --sequences --progressive-widths"; do + index=$((index + 1)) + # shellcheck disable=SC2086 # flags are meant to split + echo "[${flags:-text}] $("${client[@]}" --connect "$address" $flags "() (($index $index))")" +done +echo "listing:" +"${client[@]}" --connect "$address" --protocol binary --sequences From a3ca5662665c166d6a12997d78c5fd83f01f7055 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 18:45:59 +0000 Subject: [PATCH 10/44] chore: add release notes for LiNo over TCP --- csharp/.changeset/issue-105-lino-over-tcp.md | 12 ++++++++++++ .../20261004_120000_issue_105_tcp_protocols.md | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 csharp/.changeset/issue-105-lino-over-tcp.md diff --git a/csharp/.changeset/issue-105-lino-over-tcp.md b/csharp/.changeset/issue-105-lino-over-tcp.md new file mode 100644 index 0000000..8955a7e --- /dev/null +++ b/csharp/.changeset/issue-105-lino-over-tcp.md @@ -0,0 +1,12 @@ +--- +'Foundation.Data.Doublets.Cli': minor +--- + +Added LiNo substitution operations over TCP (issue #105). There are two protocols, and either can replace the other: + +- `TextLinoProtocol`: dot-stuffed UTF-8 messages. +- `BinaryLinoProtocol`: each packet carries its link count, and reference widths grow from 8 to 16, 32 and 64 bits. Optional decorators add external (Hybrid) references, a variable-length sequence section and progressive widths. + +`LinksServer` and `LinksClient` run create, read, update and delete queries over either protocol. The new `clink --serve` and `--connect` options do the same from the command line, together with `--protocol`, `--external-references`, `--sequences` and `--progressive-widths`. The wire format is byte-for-byte the same as the Rust port's, so C# and Rust servers and clients work with each other. + +Upgraded Link.Foundation.Links.Notation to 0.22.0. In earlier versions, parse time grew exponentially with nesting depth, so one short message could stall a server. diff --git a/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md b/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md index 8ce1d3b..e3f5a08 100644 --- a/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md +++ b/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md @@ -3,3 +3,5 @@ bump: minor --- Added LiNo substitution operations over TCP (issue #105): a dot-stuffed UTF-8 `TextLinoProtocol` and an interchangeable `BinaryLinoProtocol` whose packets carry the number of links and grow reference widths from 8 to 16, 32 and 64 bits, with optional external (Hybrid) references, a variable-length sequence section and progressive widths. `LinksServer`/`LinksClient` and the new `clink --serve`/`--connect` options (`--protocol`, `--external-references`, `--sequences`, `--progressive-widths`) run create, read, update and delete queries over either protocol. + +Upgraded `links-notation` to 0.22.0. In earlier versions, parse time grew exponentially with nesting depth, so one short message could stall a server. From feff091425d72a50ea3cd4f6ea0642bd158968cb Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 18:49:02 +0000 Subject: [PATCH 11/44] docs: add case study for issue #105 --- docs/case-studies/issue-105/README.md | 412 ++++++++++++++++++ .../github-data/issue-105-comments.json | 1 + .../issue-105/github-data/issue-105.json | 1 + .../issue-105/github-data/pr-106.json | 1 + docs/case-studies/issue-105/research-notes.md | 282 ++++++++++++ 5 files changed, 697 insertions(+) create mode 100644 docs/case-studies/issue-105/README.md create mode 100644 docs/case-studies/issue-105/github-data/issue-105-comments.json create mode 100644 docs/case-studies/issue-105/github-data/issue-105.json create mode 100644 docs/case-studies/issue-105/github-data/pr-106.json create mode 100644 docs/case-studies/issue-105/research-notes.md diff --git a/docs/case-studies/issue-105/README.md b/docs/case-studies/issue-105/README.md new file mode 100644 index 0000000..ca32d88 --- /dev/null +++ b/docs/case-studies/issue-105/README.md @@ -0,0 +1,412 @@ +# Case study: LiNo substitution operations over TCP (issue #105) + +Issue: [#105 "LINO API or LINKQL over TCP"](https://github.com/link-foundation/link-cli/issues/105) +Pull request: [#106](https://github.com/link-foundation/link-cli/pull/106) + +This folder contains: + +- `README.md`: this analysis. It covers the requirements, prior art, design, wire format, verification and the remaining limits. +- [`research-notes.md`](research-notes.md): raw notes on the linksplatform, link-foundation and link-assistant repositories and on public wire protocols, with quotes and links. +- `github-data/`: the issue, its comments and the pull request as JSON, captured on 2026-10-04. + +## 1. The problem + +link-cli already runs LiNo *substitution* queries, `(before) (after)`, against +a doublets store. Those queries cover create, read, update and delete. Until +now they only ran in-process, from the command line or through the library. +Other DBMSs expose their query language over a TCP socket: PostgreSQL on 5432, +MySQL on 3306, Redis on 6379 and Neo4j Bolt on 7687. Issue #105 asks for the +same thing for LiNo, with two protocols: + +- A UTF-8 text protocol. +- A compact binary protocol whose reference width grows with the number of links in the message. + +The protocols must be interchangeable and every optional feature must be a +decorator, in both the C# and the Rust port. + +## 2. Requirements + +| # | Requirement (from the issue text) | Where it is addressed | +|---|---|---| +| R1 | "lightweight protocols to transfer links notation representing substitution operations to allow all CRUD operations over TCP/IP" | `LinksServer` / `LinksClient` in both ports. A request is a substitution query and the empty request reads everything (§4.4). `clink --serve` / `--connect` (§4.5) | +| R2 | "UTF-8 based text only links notation" | `TextLinoProtocol` (§4.1) | +| R3 | "binary version of links notation, that will have number of links in a message/packet" | `BinaryLinoProtocol` and `LinksPacket`. The header carries N (doublets) and M (sequences) as LEB128 (§4.2) | +| R4 | "0-256 links have 8 bit size. 257-65536 links have 16 bits size and so on" (8/16/32/64) | The width is the tier of the largest address: 1, 2, 4 or 8 bytes. `--progressive-widths` grows the width per address instead (§4.2.3) | +| R5 | The binary protocol is "implemented as another decorator, for both C# and Rust" and can be "used in the same manner as unicode version … easily switchable" | Both protocols implement `LinoProtocol` (Rust trait) / `ILinoProtocol` (C# interface). `LinoConnection` decorates any byte stream with either one, so switching means swapping one value. The server detects the protocol of each message and answers in kind (§4.3) | +| R6 | "representation of numbers as defined in links themselves (see how we define numbers in our linksplatform projects)" | Default mapping: a number is `(Number unary(n))`, built from powers of two as in `Platform.Data.Doublets.Numbers.Unary` (§4.2.4) | +| R7 | "Or … use external references to directly support encoding of unicode sequences": "0-128 links have 8 bit size. 128-32768 … 16 bits" | `--external-references` / `BinaryLinoOptions.ExternalReferences`. Numbers and code points become `Platform.Data.Hybrid` external references, and the internal range per width is halved (§4.2.5) | +| R8 | "at the end we can have improved format for storing variable length sequences … size, reference_1, reference_2 … Each variable link sequences is addressable itself by last link + 1" | `--sequences` / `BinaryLinoOptions.Sequences`. A section of `size ref…` records follows the fixed doublets, and sequence *k* has address `6 + N + k` (§4.2.6) | +| R9 | "all these protocols options are optional, and come as decorators" | Every option is off by default and can be switched on independently, with `With…` builders. `AcceptedProtocols` lets a server restrict the protocol (§4.3) | +| R10 | "Reuse all the best experience … from other repositories of link-assistant, link-foundation, and linksplatform" | §3 and `research-notes.md`. Hybrid encoding, unary numbers, the linksql CRUD table and the RFC 9457-like error shape were reused | +| R11 | "collect data … to `./docs/case-studies/issue-{id}` … search online … list of each and all requirements … propose possible solutions and solution plans … check known existing components/libraries" | This folder | +| R12 | "plan and execute everything in this single pull request" | PR #106 | + +### Reading of ambiguous points + +- **"0-256" / "0-128" boundaries.** An 8-bit field holds 256 values. Address + `0` is null, so plain 8-bit references reach addresses `0..=255`. With the + Hybrid top bit, 8-bit internal references reach `0..=127`. The issue's + boundaries are each off by one. We follow the arithmetic, which is also what + `Platform.Data.Hybrid` does. +- **"number of links in a message".** The header stores the number of fixed + doublets N and, when the sequence section is on, the number of sequences M. + The width follows from the largest address, `5 + N + M`, not from N alone. + Addresses `1..=5` are reserved marker points (§4.2.2), so a packet always + knows its own maximum width before it reads any reference. +- **"decorator".** In C#, `NamedTypesDecorator`, `PinnedTypesDecorator` and the + others wrap an `ILinks`. A protocol here wraps a byte stream + (`LinoConnection`), and each binary option is one flag on an options value + that changes the encoder. Neither option needs the other. This keeps the + options composable without a separate wrapper type for every combination of + three flags. A reader still accepts every combination, because the header + announces the options in use. + +## 3. Prior art + +Full notes with quotes and file paths are in [`research-notes.md`](research-notes.md). + +### Inside the organisations + +| Project | What it has | What we took | +|---|---|---| +| [linksplatform/Data](https://github.com/linksplatform/Data) `Hybrid` | External references: top bit set means external, value `v` stored as `-v` (two's complement) and `0` stored as `2^(n-1)` (`ExternalZero`) | The exact external encoding at every width (8/16/32/64), so the 32-bit wire value equals `Hybrid` | +| [linksplatform/Numbers](https://github.com/linksplatform/Numbers) and `Data.Doublets` unary converters | `2^0 = 1`, `2^k = (2^(k-1) 2^(k-1))`, and a number is a nested sum of powers of two | The in-band number representation (R6) | +| `Data.Doublets.Sequences` `BalancedVariantConverter`, `StringToUnicodeSequenceConverter` | Strings as sequences of code points under a type marker | Strings as `(String code points…)` | +| [link-foundation/linksql](https://github.com/link-foundation/linksql) | HTTP + SSE only, with "Links Notation as the wire protocol" and the CRUD table `() ((s t))` / `((a b)) ((a c))` / `((a b)) ()` | The request semantics. No TCP transport existed to reuse | +| [link-foundation/lino-rest-api](https://github.com/link-foundation/lino-rest-api) | HTTP REST with `text/lino-line` and RFC 9457 problem details written in LiNo | The `(error: 'message')` reply, a minimal form of the same idea | +| [linksplatform/IdDistributorServer](https://github.com/linksplatform/IdDistributorServer), IdClient | A C TCP ping-pong benchmark with fixed 32-byte messages and `TCP_NODELAY` | Only the low-latency stance: `NoDelay` is set on every connection | +| [linksplatform/Protocols](https://github.com/linksplatform/Protocols) | UDP string sender and receiver (`Platform.Protocols.Udp`) | Nothing reusable for TCP framing | +| [link-foundation/links-notation](https://github.com/link-foundation/links-notation) | The LiNo parser and formatter (Rust crate and NuGet package) | Used directly. Upgraded 0.16.1 → 0.22.0 (§6.1) | + +No project in these organisations defined a raw TCP protocol for LiNo, so the +framing and the binary layout are new. Where possible they reuse the +encodings above. + +### Public protocols + +| Protocol | Relevant idea | Influence | +|---|---|---| +| CBOR, [RFC 8949](https://www.rfc-editor.org/rfc/rfc8949.html) | Integer arguments inline, or in 1/2/4/8 following bytes | The 1/2/4/8-byte tiers, chosen once per link rather than once per value | +| MessagePack, PackStream (Bolt) | Width escalation through marker bytes (`0xcc..0xcf`, `INT_8..INT_64`) | Same as above | +| LEB128 / protobuf varints | 7 bits per byte, little-endian groups | Header counts N and M | +| [PostgreSQL](https://www.postgresql.org/docs/current/protocol-overview.html), [MySQL](https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_packets.html) | Self-delimiting messages on a TCP stream | Every binary packet is self-delimiting: counts first, then fixed-size or size-prefixed records | +| SMTP (RFC 5321 §4.5.2) | Messages end with a line holding a single `.`, and leading dots are doubled | Text framing. It can be typed by hand in `nc` or `telnet` | +| Redis RESP | A text protocol that humans can type, where the first byte tells the type | Protocol detection by the first byte (§4.3) | +| LZW | Code width grows as the dictionary grows | `--progressive-widths` | + +### Libraries considered + +- **links-notation** (Rust `links-notation`, NuGet `Link.Foundation.Links.Notation`) is used for all text parsing and formatting. +- **lino-objects-codec** encodes JSON-like objects as LiNo. It is not needed, because requests and replies are already LiNo documents. +- **tokio / async-std** were not used. The Rust store is single-threaded, and a thread per connection with a store lock is simpler and enough for a CLI. The same model is used in C# with `TcpListener` and threads. +- **protobuf, Cap'n Proto, FlatBuffers** were rejected. They need a schema and would encode a LiNo tree as generic messages. The issue asks for a links-native layout with address-sized references. + +## 4. Design + +Both ports implement the same specification. The golden vectors (§5) prove +that they produce identical bytes. + +### 4.1 Text protocol + +```text +message = *line "." CRLF-or-LF +line = (escaped LiNo text) CRLF-or-LF ; a line starting with "." gets one more "." +``` + +- The payload is canonical LiNo with one top-level link per line, encoded as UTF-8. +- Readers accept `\n` and `\r\n`. +- A request and its reply use the same framing. An empty message (just `.`) is the empty document. +- Caveat: a quoted reference that contains `\r\n` arrives as `\n`. The binary protocol carries such references exactly. + +A session in `nc`: + +```text +() ((1 1)) +. +() ((1: 1 1)) +. +``` + +### 4.2 Binary protocol + +#### 4.2.1 Packet layout + +```text +byte 0 0x10 | flags high nibble 1 = format version 1 + bit 0 external references (Hybrid encoding) + bit 1 sequence section present + bits 2-3 log2 of the minimum width in bytes +LEB128 N number of fixed doublets +LEB128 M number of sequences (only when bit 1 is set) +N times source target one fixed doublet, refs width(a) bytes each +M times size ref_1 … ref_n one sequence, size and refs width(a) bytes each +``` + +All references are little-endian. A packet is self-delimiting, so it needs no +length prefix. + +#### 4.2.2 Addresses + +| Address | Meaning | +|---|---| +| `0` | null, the empty link `()` | +| `1` | `One`, the unary 1 | +| `2` | `Number` marker | +| `3` | `String` marker | +| `4` | `List` marker | +| `5` | `Identified` marker (a link with an id) | +| `6 … 5+N` | fixed doublets | +| `6+N … 5+N+M` | sequences ("last link + 1", R8) | + +Markers are never transmitted. A link may only refer to links before it, so a +decoder resolves everything in one pass. The last link is the document root. + +#### 4.2.3 Width tiers (R4) + +`tier(a)` is the smallest of 1, 2, 4 and 8 bytes that can hold address `a`. +Every reference of the link at address `a` uses +`width(a) = max(min_width, tier(a))` bytes. + +- **Default (uniform).** `min_width` is the tier of the highest address, so + one width is used for the whole packet: + + | Highest address | Plain | With external references | + |---|---|---| + | up to 255 / 127 | 8-bit | 8-bit | + | up to 65 535 / 32 767 | 16-bit | 16-bit | + | up to 2³²−1 / 2³¹−1 | 32-bit | 32-bit | + | larger | 64-bit | 64-bit | + + The first column gives the limit for plain references, the second for references with external references on. + +- **`--progressive-widths`.** `min_width` is 1, so the first links use 8-bit + references even in a large packet. The width only grows when the address + grows. Since a link at address `a` can only refer to addresses below `a`, + `tier(a)` always fits. + +#### 4.2.4 Numbers as links (R6, the default) + +| LiNo | Links | +|---|---| +| `()` | `0` | +| number `n` | `(Number unary(n))`. `unary(0) = 0`, `2^0 = One`, `2^k = (2^(k-1) 2^(k-1))`, and other values are right-nested sums of powers of two from the highest bit down | +| other reference | `(String code-point…)`, with each code point a unary number | +| link of two values, no id | a plain doublet | +| link of any other arity, no id | `(List e…)` | +| link with an id | `(Identified id value…)` | +| document | `(List top-level-link…)`, the root | + +Identical sub-links are emitted once, because links are content-addressed. + +#### 4.2.5 External references (R7, `--external-references`) + +Numbers and code points travel as Hybrid external references instead of +unary links. At width `w` bits: + +- value `v ≥ 1` is sent as `2^w − v` +- `0` is sent as `2^(w−1)` + +This is exactly `Platform.Data.Hybrid`. The top bit marks a reference as +external, so internal addresses lose one bit: 8-bit references reach `0..127`. + +#### 4.2.6 Sequence section (R8, `--sequences`) + +Without this option, lists, strings and identified links are cons chains of +doublets: `(marker (e1 (e2 (… (en 0)))))`. With it, they become one +variable-length record `size marker e1 … en`, and a plain list has no marker. + +Fixed doublets may only refer to earlier fixed doublets. So a two-value link +that holds a sequence becomes a two-element sequence. + +#### 4.2.7 Decoding untrusted input + +`DecodeLimits` caps the following: + +- the number of links: 2²² +- the total sequence items: 2²⁴ +- the expanded LiNo nodes +- the nesting depth +- the text message size: 64 MiB + +The decoder also rejects: + +- a forward reference or a reference to itself +- an unsupported header byte, meaning an unknown version +- a width too small for an address +- a truncated packet + +### 4.3 Decorators and switching (R5, R9) + +```rust +let mut connection = LinoConnection::new(stream, BinaryLinoProtocol::with_options( + BinaryLinoOptions::default().with_external_references(true).with_sequences(true))); +// …or TextLinoProtocol::new(); the rest of the code is unchanged. +``` + +```csharp +using var client = LinksClient.Connect("127.0.0.1", 7777, + new BinaryLinoProtocol(new BinaryLinoOptions().WithExternalReferences().WithSequences())); +var reply = client.QueryText("() ((1 1))"); +``` + +Protocol detection: a text message never starts with a byte in `0x10..0x1F`, +and a binary packet always does. `LinksServer` therefore reads each message, +detects its protocol and replies in the same protocol, with the same options. +`AcceptedProtocols.Text` / `Binary` (`--protocol` on `--serve`) restricts the +server to one protocol. + +### 4.4 Request semantics + +| Request | Reply | +|---|---| +| empty document | every link, `(index: source target)` per line, named where a name exists | +| substitution query | one `(before) (after)` line per change, as with `clink --changes` | +| failing query | `(error: 'message')`. The connection stays usable | +| malformed message | `(error: 'message')` in text, then the server closes the connection, because the stream may be out of sync | + +Requests run one at a time under a store lock. Each connection has its own +thread for parsing and formatting. + +### 4.5 Command line + +```bash +clink --db data.links --serve 127.0.0.1:7777 [--protocol text|binary] [--auto-create-missing-references] [--trace] +clink --connect 127.0.0.1:7777 '() ((1 1))' # text +clink --connect 127.0.0.1:7777 --protocol binary # list all links over binary +clink --connect 127.0.0.1:7777 --external-references --sequences --progressive-widths '((1: 1 1)) ((1: 1 2))' +``` + +On the client, any binary option implies `--protocol binary`, and +`--protocol text` together with a binary option is an error. Using `--serve` +and `--connect` together is also an error. + +## 5. Verification + +### Golden vectors + +The Rust and C# tests both assert these bytes. The vectors use the uniform +width. + +| Document | ext | seq | Bytes | +|---|---|---|---| +| `() ((1 1))` | – | – | `10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b` | +| `() ((1 1))` | – | ✓ | `12 02 03 02 01 06 06 01 07 02 00 08 01 09` | +| `() ((1 1))` | ✓ | – | `11 06 ff ff 06 00 04 07 00 08 09 00 04 0a` | +| `() ((1 1))` | ✓ | ✓ | `13 01 03 ff ff 01 06 02 00 07 01 08` | +| `hi` | ✓ | – | `11 05 97 00 98 06 03 07 08 00 04 09` | +| `hi` | ✓ | ✓ | `13 00 02 03 03 98 97 01 06` | +| (empty) | – | – | `10 00` | +| (empty) | ✓ | ✓ | `13 00 00` | + +How the first vector decodes: + +- header `0x10`, N = 7 +- `6 = (Number One)` = 1 +- `7 = (6 6)` = `(1 1)` +- `8 = (7 0)` +- `9 = (List 8)` = `((1 1))` +- `10 = (0 9)` = `() ((1 1))` +- `11 = (10 0)` +- `12 = (List 11)`, the document + +### Sizes + +Output of `cargo run --example lino_binary_dump -- ''`, in bytes. The +text size includes the `\n.\n` terminator. + +| Document | text | plain | seq | ext | ext+seq | +|---|---|---|---|---|---| +| `() ((1 1))` | 13 | 16 | 14 | 14 | 12 | +| `((1: 1 1)) ((1: 1 2))` | 24 | 38 | 28 | 32 | 22 | +| `() ((child: father mother))` | 30 | 132 | 118 | 52 | 38 | +| `() ((1000 70000))` | 20 | 70 | 68 | 50 | 39 | + +The pure-links mapping (R6) is not meant to be smaller than text. It +transmits a document as nothing but links, which a links store can import +without any parser. External references and sequences are the options that +make the packet compact, and with both on, the binary form is the smallest +for numeric links. Text stays smaller for documents made of names or large +numbers, because: + +- each name costs a sequence with a marker and one reference per code point; +- each number needs a whole external reference at the packet's width. + +The binary protocol's advantage is that it needs no parsing and has a fixed +structure, not that it is smaller. + +### Automated tests + +- Rust: + - `rust/tests/protocol_packet_tests.rs`: codecs, the golden vectors, the corpus round trip under all 8 option sets, limits, malformed input, and deep nesting in linear time. + - `rust/tests/protocol_tcp_tests.rs`: CRUD over every protocol, mixed clients, errors, CRLF, malformed input, protocol restriction, concurrency and shutdown. + - `rust/tests/cli_tcp_tests.rs`: `clink --serve` / `--connect` end to end. +- C#: + - `LinoProtocolCodecTests` (34 tests): the same corpus and golden vectors. + - `LinksServerTests` (9 tests): CRUD over every protocol, identical text and binary replies, errors, CRLF, malformed input, protocol restriction, concurrent clients and shutdown. + - `CliTcpIntegrationTests` (8 tests): `clink --serve` / `--connect` end to end. +- Cross-language interop: [`examples/tcp/run-interop.sh`](../../../examples/tcp/run-interop.sh) runs a Rust server with a C# client, and a C# server with a Rust client. It covers text and every binary option, and both directions print identical results. + +## 6. Findings along the way + +### 6.1 links-notation 0.16.1 parsed nested input in exponential time + +The time to parse `((((…a…))))` doubled with every level of nesting. A +request of a few dozen bytes could keep a server thread busy for minutes, +which would be a denial of service for anything listening on a socket. This +is upstream issue +[links-notation#314](https://github.com/link-foundation/links-notation/issues/314), +fixed in 0.21.3. + +Both ports now use **0.22.0**. Its parse time is linear and it rejects deeper +nesting than its limit. The regression test (`deeply_nested_text_parses_in_linear_time` +in Rust and `DeeplyNestedTextParsesInLinearTime` in C#) parses 40 levels +correctly, and rejects 100 000 levels quickly instead of hanging. + +### 6.2 One canonical model in both ports + +links-notation represents `a` and `(a)` the same way: an unnamed group +holding one reference. `parse_document` / `LinoFormat.ParseDocument` +collapse such a group to the reference itself, recursively. +`format_document` writes `((a))` when a one-reference link must survive. +With this rule, `parse(format(doc)) == doc` holds in both ports, and a +document decoded from the same bytes compares equal in Rust and C#. The tool +in `examples/lino-structure-probe` and `rust/examples/lino_structure_probe.rs` +prints the raw parser output, which was used to confirm this. + +### 6.3 Behaviour differences between the ports' query processors + +These are pre-existing behaviours, not introduced by the protocols. The +protocols carry whatever the processors return. + +- The C# `AdvancedMixedQueryProcessor` does not report the creation of a + *named* composite link as a change. As a result: + - `() ((a: a a))` gets an empty reply from a C# server. + - The same query gets `() ((a: a a))` from a Rust server. + - The link is created in both cases, and the listing shows it. +- Rust reports a failing query as `Query error: `, while C# reports + ``. +- Deleting a link also deletes the links that refer to it, in both ports. For + example, `((2: 2 2)) ()` with `(1: 1 2)` present replies with two + deletions. + +## 7. Risks and remaining limits + +- **No authentication or encryption.** The server is meant for trusted + networks or localhost, like a development database. For anything else, put + it behind TLS (stunnel or an SSH tunnel) or bind it to `127.0.0.1`. Adding + TLS would mean wrapping the stream, which the `LinoConnection` design + already allows. +- **Requests are serialized.** Requests run one at a time under a store lock, + so throughput is limited by the store, not by the network. +- **No streaming of huge results.** A reply is one message. The decode limits + bound what a peer can send. +- **Version 1 only.** The high nibble of the header is the version, and + `0x2_` … `0xF_` are reserved for future layouts, for example a different + marker set. + +## 8. Possible next steps + +1. A TLS decorator (`TlsLinoConnection`) once a use case needs it. +2. Subscriptions over the same connection: server-pushed `(before) (after)` messages, similar to linksql's SSE `GET /subscribe`. +3. Exposing the protocol from linksql's engine, so linksql gains the TCP transport its spec lacks. +4. A `Platform.Protocols.Tcp` package extracted from `LinksServer` / `LinksClient` if other projects need it. diff --git a/docs/case-studies/issue-105/github-data/issue-105-comments.json b/docs/case-studies/issue-105/github-data/issue-105-comments.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/docs/case-studies/issue-105/github-data/issue-105-comments.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/docs/case-studies/issue-105/github-data/issue-105.json b/docs/case-studies/issue-105/github-data/issue-105.json new file mode 100644 index 0000000..e06f479 --- /dev/null +++ b/docs/case-studies/issue-105/github-data/issue-105.json @@ -0,0 +1 @@ +{"author":{"id":"MDQ6VXNlcjE0MzE5MDQ=","is_bot":false,"login":"konard","name":"Konstantin Diachenko"},"body":"We need to have lightweight protocols to transfer links notation representing substitution operations to allow all CRUD operations over TCP/IP as other DBMS systems do.\n\nWe must support UTF-8 based text only links notation, and binary version of links notation, that will have number of links in a message/packet, first 256 links should have number of links or ids or addresses 8 bit in size, next we get 16 bit addresses, 32 addresses and 64 bit addresses.\n\nSo 0-256 links have 8 bit size.\n\n257-65536 links have 16 bits size and so on.\n\nSo for the binary protocol implementation, which should be implemented as another decorator, for both C# and Rust, must be able to be used in the same manner as unicode version of protocol, and be easily switchable between them.\n\nWe also should support representation of numbers as defined in links themselves (see how we define numbers in our linksplatform projects on GitHub and so on). Or we should be able to use external references to directly support encoding of unicode sequences if needed.\n\nIn that case it will go:\n\nSo 0-128 links have 8 bit size.\n\n128-32768 links have 16 bits size and so on.\n\nAlso, once we read all links, at the end we can have improved format for storing variable length sequences.\n\nSo first section of the protocol is fixed length doublets, and after we recorded all of them we can have additional links with variable lenght.\n\nIt goes like this:\n\nsize, reference_1, reference_2 and so on (so size defines how to read sequences), as we don't increase size of links, we can reference all links from previous fixed range here.\n\nEach variable link sequences is addressable itself by last link + 1 address or id.\n\nMake sure all these protocols options are optional, and come as decorators in our implementation.\n\nReuse all the best experience on the topic from other repositories of link-assistant, link-foundation, and linksplatform, so we reuse all the best approaches, if needed to modify these, we can copy the code here, and only later use dependencies, if some components cannot be used as is directly.\n\nWe need to collect data related about the issue to this repository, make sure we compile that data to `./docs/case-studies/issue-{id}` folder, and use it to do deep case study analysis (also make sure to search online for additional facts and data), list of each and all requirements from the issue, and propose possible solutions and solution plans for each requirement (we should also check known existing components/libraries, that solve similar problem or can help in solutions).\n\nPlease plan and execute everything in this single pull request, you have unlimited time and context, as context auto-compacts and you can continue indefinitely, until each and every requirement fully addressed, and everything is totally done.\n\n","createdAt":"2026-10-03T23:27:49Z","labels":[{"id":"LA_kwDONXCAbs8AAAAB0ixEPA","name":"documentation","description":"Improvements or additions to documentation","color":"0075ca"},{"id":"LA_kwDONXCAbs8AAAAB0ixEQA","name":"enhancement","description":"New feature or request","color":"a2eeef"}],"number":105,"state":"OPEN","title":"LINO API or LINKQL over TCP"} diff --git a/docs/case-studies/issue-105/github-data/pr-106.json b/docs/case-studies/issue-105/github-data/pr-106.json new file mode 100644 index 0000000..5e85b89 --- /dev/null +++ b/docs/case-studies/issue-105/github-data/pr-106.json @@ -0,0 +1 @@ +{"author":{"id":"MDQ6VXNlcjE0MzE5MDQ=","is_bot":false,"login":"konard","name":"Konstantin Diachenko"},"baseRefName":"main","body":"## 🤖 AI-Powered Solution Draft\n\nThis pull request is being automatically generated to solve issue #105.\n\n### 📋 Issue Reference\nFixes #105\n\n### 🚧 Status\n**Work in Progress** - The AI assistant is currently analyzing and implementing the solution draft.\n\n### 📝 Implementation Details\n_Details will be added as the solution draft is developed..._\n\n---\n*This PR was created automatically by the AI issue solver*","createdAt":"2026-10-04T17:27:40Z","headRefName":"issue-105-5190ec6fb31a","isDraft":true,"number":106,"state":"OPEN","title":"[WIP] LINO API or LINKQL over TCP"} diff --git a/docs/case-studies/issue-105/research-notes.md b/docs/case-studies/issue-105/research-notes.md new file mode 100644 index 0000000..b5ed594 --- /dev/null +++ b/docs/case-studies/issue-105/research-notes.md @@ -0,0 +1,282 @@ + + +# Research notes for link-cli issue #105 ("LINO API or LINKQL over TCP") + +Clones live in `/tmp/research/*`. Also cloned here: `Data`, `Data.Doublets`, `doublets-rs`, +`Converters`, `data-rs` (Rust `platform-data`), and `links-notation-0.16` (tag `rust_0.16.1` = +`csharp_0.16.1`, commit `f0193ce`). The main `links-notation` clone is at 0.22.0. + +Key takeaways: + +- No linksplatform or link-foundation project defines a raw TCP protocol for LiNo yet. linksql only + has HTTP + SSE (JS only). IdDistributorServer/IdClient is a TCP ping-pong benchmark with fixed + 32-byte messages, not a real protocol. link-cli itself has no networking code (`grep TcpListener|TcpStream|tokio|Socket` → nothing). +- Hybrid / external references: **top bit set ⇔ external**. The value is stored as **two's-complement + negation** (`-v mod 2^N`), not as flag | magnitude. Raw value 0 is a special case, encoded as + `ExternalZero = 2^(N-1)`. +- CBOR's argument widths (inline <24, then 1/2/4/8 following bytes) and MySQL's length-encoded + integers are the closest prior art to the issue's width-escalation scheme. + +--- + +## 1. linksql (`/tmp/research/linksql`, v0.1.0, spec "Version 0.1 (draft)") + +Ports: `js/` (reference: engine, HTTP server, client, CLI), `rust/`, `python/`, `csharp/` (engine, +wire codec, schema only). Spec §13: *"`rust/`, `python/`, and `csharp/` ... do not ship the HTTP +server, client or CLI."* + +### Transport +- **There is no TCP/binary transport.** The only transport is §11 "HTTP protocol" (OPTIONAL): + +| Method + path | Request | Response | +|---|---|---| +| `GET /` | — | `{ name, version, links }` | +| `POST /query` | JSON `{ "query": "…" }` or raw LiNo body | query report (§6) | +| `GET /links` | — | `{ links: [...] }` | +| `GET /introspect` | — | introspection snapshot | +| `POST /import` | raw LiNo body | `{ imported }` | +| `GET /export` | — | `text/plain` canonical LiNo | +| `GET /subscribe?pattern=` | restriction | `text/event-stream` (SSE) | + + Schema servers add `GET /schema`, `POST /query/`, `GET /subscribe/`. +- Errors: *"Unknown routes return `404` with `{ error }`. Any thrown error returns `400` with `{ error: message }`."* + (`js/src/server.js:199-201`: `sendData(req, res, 400, { error: error.message })`). +- SSE (§11.1): opens with the comment `: linksql stream open`, then one `data: ` frame per change, + where the payload is `{ operation, matching }`. +- Server: `js/src/server.js` uses Node's `http.createServer` with no framework. Client: `js/src/client.js` uses `fetch`. + +### Wire format (§2.5 "Links Notation as the wire protocol") +> "Every structured value that crosses a network boundary (query reports, link lists, introspection +> and schema documents, subscription events) is encoded as Links Notation text rather than JSON." +- The default content type is `application/lino`. JSON is opt-in via `Accept: application/json`. If a client accepts both, LiNo wins. +- Codec: `lino-objects-codec` (`js/src/protocol.js`: `jsonToLino({ json: value })` / `linoToJson({ lino })`). + Rust: `rust/src/protocol.rs` (`LINO_CONTENT_TYPE`, `enum Value {Null,Bool,Int,Float,Str,Array,Object}`). + C#: `csharp/src/LinksQL/Protocol.cs`, `public static class Protocol` with `Encode`, `Decode`, + `PrefersJson`, `EncodeReport`, `EncodeIntrospection`, `EncodeSchema`. +- Object encoding convention: `((key value) (key value))`; an array becomes `(a b c)`; an empty object or array becomes `()`. + Example: `"((operation update) (created ()) ...)"`. + +### Query model (worth reusing for the text protocol) +- A query is one or two top-level LiNo nodes: `(restriction)` is a read; `(restriction) (substitution)` is a write. + The spec's CRUD table: + ``` + create () ((s t)) + read ((p)) + update ((a b)) ((a c)) + delete ((a b)) () + ``` +- Stored link canonical form: `(index: source target)`, e.g. `(3: 1 2)`. Indices start at `1`. +- Pattern slots: 2 values gives `{id?, source, target}`. `(x)` with 1 value and no id addresses by identity only. + +### QueryReport (§6), the response shape to mirror +```json +{ "operation": "update", + "matched": [ { "links": [{ "index": 5, "source": 3, "target": 4 }], "binding": {} } ], + "created": [], "updated": [{ "index": 5, "source": 3, "target": 6 }], "deleted": [] } +``` +- `operation` ∈ `read | create | update | delete | mixed | noop` (§6.1). The rule: read-only query → `read`; + none of created/updated/deleted non-empty → `noop`; exactly one → that one; more than one → `mixed`. +- Error taxonomy (`rust/src/lib.rs:60-90`, `csharp/src/LinksQL/Errors.cs`): `LinoSyntaxError` (carries a + message and a 0-based `position`, `-1` if none; the message is rendered as `"{message} (at position {n})"`), `LinkIntegrityError`, + `UnknownNameError`, `SubstitutionError`, `QueryError`, `SchemaError`. These map well to binary error codes. +- Subscriptions (§9) push `{ operation, matching }`. Triggers (§10) have modes `never|once|always`, and link-cli's + flags of the same names are cited. +- CLI (§12): `linksql serve --port --host `. The spec gives no default port. + +Reuse candidates: the `application/lino` content type name, the report field names, the error class names, +the operation classification rule, and the "LiNo first, JSON opt-in" stance. + +## 2. lino-rest-api (`/tmp/research/lino-rest-api`, spec `docs/spec/README.md`) +- **HTTP REST**. Packages: JS (`node:http`/Express), Python (ASGI), Rust (axum server, reqwest client). +- Media types (§2): `text/lino` (default, indented), `text/lino-line` (one value per line, used for streaming/NDJSON-like + bodies), `text/lino-compact` (type-tagged, base64; *"the only form that can carry shared object identity and cycles"*), + `application/json` (fallback). *"A server MUST include `charset=utf-8` when it emits a LINO media type."* +- Negotiation: `Accept` with q-values, `406`/`415`, `Vary: Accept`. A missing or `*/*` Accept selects `text/lino`. +- Errors: RFC 9457 problem details written in LiNo, served as `text/lino` or `application/problem+lino`: + ```lino + ( type "https://link-foundation.github.io/lino-rest-api/errors/not-found" + title "Not Found" status 404 detail "Item 42 does not exist" instance "/items/42" ) + ``` + `type`, `title` and `status` MUST be present. `errors` is reserved for field-level failures. +- Collections: `(items (...) page (limit 20 offset 0 total 2 count 2))`. Also ETag/304/412/428, CORS, + `/.well-known/lino-api`, OpenAPI 3.1. +- Value model (§3): `null`, `true/false`, int, float, `"quoted"`, `(a b c)`, `(k v)` pairs, via lino-objects-codec. +- Relevance: the `text/lino-line` idea (one message per line) gives a natural newline-delimited framing for a + text TCP protocol. RFC 9457-in-LiNo is a ready-made error message shape. + +## 3. IdDistributorServer / IdClient (C, TCP) +- README: *"Identifier distributor TCP/IP server implementation. A prototype for LinksPlatform cluster's core service."* +- **It has no real message layout.** It is an echo / ping-pong throughput benchmark: + - `Server.c`: `#define BUFSIZE 32`. Its loop is `recv(sock, buffer, BUFSIZE, 0); send(sock, buffer, BUFSIZE, 0); requestsCount++;`. + - `Client.c`: `send(*clientSocket, buffer, BUFSIZE, 0); recv(*clientSocket, buffer, BUFSIZE, 0);` (BUFSIZE 32). + - Messages are fixed 32 bytes with no length prefix, no type byte, and no ID content. No ID counter is implemented. + - Bug: the server declares `char buffer[8];` but receives `BUFSIZE` (32) bytes, which is a stack overflow (`Server.c:42,68`). +- Host and port are CLI args (`main(argc, argv)`: `hostname = arguments[1]; port = arguments[2]`). No default port. + `TCP_NODELAY` is set and `BACKLOG 100`. The Linux build uses `select()` (`-DSERVER_SELECT`, `MAX_SOCKETS 1000`); Windows uses Winsock. +- Takeaways: the intent was a minimal, low-latency TCP request/response loop with Nagle disabled. Nothing in it + can be reused as a format. + +## 4. Protocols (`linksplatform/Protocols`, NuGet `Platform.Protocols`, namespace `Platform.Protocols`) +(The issue calls it Platform.Communication. The repo and namespace are now `Protocols` / `Platform.Protocols`. +The `cpp/` folder only has `conanfile.txt`.) +- `Udp/UdpSender.cs`: `class UdpSender : DisposableBase`. Constructors: `(IPEndPoint)`, `(IPAddress, int port)`, + `(string hostname, int port)`, `(int port)` → loopback. Method: `int Send(string message)`. +- `Udp/UdpReceiver.cs`: `delegate void MessageHandlerCallback(string message)`. `class UdpReceiver : DisposableBase` + has `private const int DefaultPort = 15000`, constructors `(int listenPort, bool autoStart, MessageHandlerCallback)`, + `(int, cb)`, `(cb)`, `()`, and members `Start()`, `Stop()`, `string Receive()`, `ReceiveAndHandle()`, `bool Available`. + It runs a background thread that polls `Available` and else calls `ThreadHelpers.Sleep()`; exceptions are ignored. +- `Udp/UdpClientExtensions.cs`: `DefaultEncoding = Encoding.UTF8`; `SendString(this UdpClient, IPEndPoint, string)`; + `ReceiveString(this UdpClient)`. One datagram carries one UTF-8 string, with no framing. +- Also contains `Gexf/` (graph XML) and `Xml/Serializer.cs`. +- Takeaways: the API is string-message oriented and uses the callback name `MessageHandlerCallback`. Port 15000 is a historical default. + A TCP counterpart could mirror `Sender/Receiver(port, handler)`. + +## 5. Number representation inside links (Hybrid / external references) + +### C# `Platform.Data.Hybrid` (`/tmp/research/Data/csharp/Platform.Data/Hybrid.cs`) +```csharp +public static readonly TLinkAddress HalfOfNumberValuesRange = (NumericType.MaxValue) / TLinkAddress.CreateTruncating(2); +public static readonly TLinkAddress ExternalZero = (HalfOfNumberValuesRange + TLinkAddress.CreateTruncating(1)); +public bool IsInternal => SignedValue > 0; +public bool IsExternal => (Value == ExternalZero) || SignedValue < 0; +public long SignedValue => _addressToInt64Converter.Convert(Value); // UncheckedSignExtendingConverter +public long AbsoluteValue => (Value == ExternalZero) ? 0 : Math.Abs(SignedValue); +public Hybrid(TLinkAddress value, bool isExternal) { if (value == default && isExternal) Value = ExternalZero; + else Value = isExternal ? -(value) : value; } +public override string ToString() => IsExternal ? $"<{AbsoluteValue}>" : Value.ToString(); +``` +- So for an N-bit address: `H = 2^(N-1) - 1`, `ExternalZero = 2^(N-1)` (only the MSB set). +- **External = MSB set** (signed interpretation is negative, or the value is exactly ExternalZero). + The encoding is **two's-complement negation**: raw value `v ≥ 1` is stored as `2^N - v`, and `v = 0` is stored as `2^(N-1)`. + It is NOT `0x80 | v`. Example (8-bit): raw 1 → `0xFF`, raw 2 → `0xFE`, raw 127 → `0x81`, raw 0 → `0x80`. +- Tests confirm this (`Data/csharp/Platform.Data.Tests/HybridTests.cs`): + `new Hybrid(unchecked((byte)128)).AbsoluteValue == 0`, `new Hybrid(unchecked((byte)-1)).AbsoluteValue == 1`. + `LinksConstantsTests` uses internal `(1, half)` and external `(half+1, Max)` for byte/ushort/uint/ulong. +- `Numbers/Raw/AddressToRawNumberConverter.cs`: `Convert(source) => new Hybrid(source, isExternal: true)`. + `Numbers/Raw/RawNumberToAddressConverter.cs`: `Convert(source) => new Hybrid(source).AbsoluteValue`. + Note the naming: "AddressToRawNumber" means wrapping a plain number into its external (raw) form. + +### `LinksConstants` (`Data/csharp/Platform.Data/LinksConstants.cs`) +- With external refs enabled: internal range `(1, H)`, external range `(ExternalZero, MaxValue)`. +- Special constants are taken from the **top of the internal range**: `Continue = H`, `Break = H-1`, `Skip = H-2`, + `Any = H-3`, `Itself = H-4`, `Error = H-5`. The usable `InternalReferencesRange = (1, H-6)`. `Null = 0`. + When external refs are disabled, the same constants sit at `MaxValue`, `MaxValue-1`, and so on. + +### Resulting ranges (external refs enabled) +| width | internal link refs (raw / usable after 6 constants) | external raw numbers | encoded external values | +|---|---|---|---| +| 8 | 1..127 / 1..121 | 0..127 (128 values) | 0x80..0xFF | +| 16 | 1..32767 / 1..32761 | 0..32767 | 0x8000..0xFFFF | +| 32 | 1..2^31-1 / 1..2^31-7 | 0..2^31-1 | 0x8000_0000..0xFFFF_FFFF | +| 64 | 1..2^63-1 / 1..2^63-7 | 0..2^63-1 | 0x8000…..0xFFFF… | + +- Without the hybrid flag, an N-bit unsigned field holds 0..2^N-1. With 0 = null and 1-based indices that is 1..255 for 8-bit. + The issue's "0-256 → 8-bit" and "0-128 → 8-bit" are each off by one at the boundary. +- Widening: sign extension from N to 2N bits preserves the meaning, **except** ExternalZero. For example, `0x80` must map to `0x8000`, + because sign-extending would give `0xFF80`, which means raw 128. Narrowing is valid only if the value fits. +- Numbers wider than N-1 bits are chunked: `Data.Doublets.Sequences/.../Numbers/Raw/NumberToLongRawNumberSequenceConverter.cs` + has `_bitsPerRawNumber = NumericType.BitsSize - 1` and recursively does `Links.GetOrCreate(convertedLowPart, Convert(source >> bitsPerRawNumber))`. + `BigIntegerToRawNumberSequenceConverter` emits (N-1)-bit chunks, builds a sequence, and marks negatives with `GetOrCreate(NegativeNumberMarker, seq)`. + `Byte/BytesToRawNumberSequenceConverter` maps each byte to a raw number, then builds a list→sequence. +- Unary numbers (`Numbers/Unary/`): `PowerOf2ToUnaryNumberConverter` sets `2^0 = 1` (link 1) and `2^k = GetOrCreate(2^(k-1), 2^(k-1))`. + `AddressToUnaryNumberConverter` ORs together the set bits as nested `GetOrCreate(pow2(i), acc)`. Numbers become pure link structure, with no external refs. +- Strings: `CharToUnicodeSymbolConverter` creates `GetOrCreate(AddressToRaw(utf16CodeUnit), UnicodeSymbolType)`. + `BalancedVariantConverter` pairs neighbours layer by layer into a balanced binary tree of doublets. + `StringToUnicodeSequenceConverter` creates `GetOrCreate(balancedTree, UnicodeSequenceType)`. Empty string → the type link itself. + +### Rust `platform-data` (`/tmp/research/data-rs/src/hybrid.rs`) differs from C# +```rust +pub fn half() -> T { T::MAX / T::from_byte(2) } +fn extend_value(value: T) -> T { (T::MAX - value).wrapping_add(&T::from_byte(1)) } // = -value +pub fn is_internal(&self) -> bool { self.value < Self::half() } +``` +- `external(0)` yields `0`, not ExternalZero. The `default_external` range is `half()..=MAX`, which overlaps the internal + range `1..=half()` at `half`. Treat it as non-normative. The C# behaviour is the reference. + +### This repo's port (link-cli) +- `rust/src/hybrid_reference.rs` matches C# `Hybrid` exactly (u32 only): + ```rust + const EXTERNAL_ZERO: u32 = (u32::MAX / 2) + 1; + pub fn external(value: u32) -> Self { encoded: if value == 0 { EXTERNAL_ZERO } else { 0u32.wrapping_sub(value) } } + pub fn absolute_value(self) -> Option { if encoded == EXTERNAL_ZERO {Some(0)} else if encoded >= EXTERNAL_ZERO {Some(0u32.wrapping_sub(encoded))} else {None} } + ``` +- `rust/src/sequences/`: `AddressToRawNumberConverter::convert(u32) -> external_reference(address)`. + `RawNumberToAddressConverter::convert` returns `external_reference_value(raw).unwrap_or(raw)` (passes internal values through). + Also `BalancedVariantConverter`, `CharToUnicodeSymbolConverter` (UTF-16 code units), `StringToUnicodeSequenceConverter`, + `UnicodeSequenceToStringConverter`, `RightSequenceWalker`, `CachingConverterDecorator`, `TargetMatcher`, `DefaultStack`. +- C#: `csharp/Foundation.Data.Doublets.Cli.Library/NamedLinks.cs` does `new Hybrid(link, isExternal: true)` for + names of external references. `UnicodeStringStorage.cs` uses `AddressToRawNumberConverter` / `RawNumberToAddressConverter`. +- Dependencies: C# uses `Platform.Data 0.16.1`, `Platform.Data.Doublets 0.18.1`, `Platform.Data.Doublets.Sequences 0.6.5`, + `Link.Foundation.Links.Notation 0.16.1`. Rust uses `links-notation = "0.16.1"`. Rust `LinkStorage` is `HashMap`. +- Existing decorator pattern (for new protocol options): `NamedLinksDecorator`, `PinnedTypesDecorator`, + `NamedTypesDecorator`, `TransactionsDecorator`, `SimpleLinksDecorator`, `VersionControlDecorator`, + `PersistentTransformationDecorator` (C#). In Rust: `sequences::CachingConverterDecorator`. + +## 6. links-notation 0.16.1 API (`/tmp/research/links-notation-0.16`) +### Rust crate `links-notation` (lib name `links_notation`) +- `pub enum LiNo { Link { id: Option, values: Vec }, Ref(T) }` with `is_ref()`, `is_link()`, + `LiNo::new(id, values)`, `LiNo::anonymous(values)`, `LiNo::reference(v)`, `format_with_config(&FormatConfig)`, + and `impl Display`. +- Free functions: `parse_lino(&str) -> Result, ParseError>`, + `parse_lino_to_links(&str) -> Result>, ParseError>`, `format_links(&[LiNo]) -> String`, + `format_links_with_config(&[LiNo], &FormatConfig) -> String`. +- `LiNoBuilder::new().id(..).value(..).values(..).lino(..).build()`. Also `lino!` macro (re-export from `links-notation-macro`), + `FormatConfig { less_parentheses, max_line_length, indent_long_lines, max_inline_refs, group_consecutive, indent_string, prefer_inline }` + with `FormatConfig::builder()`, `parser::Link`, and tuple `From` impls for 2 to 10 elements. +- link-cli uses `use links_notation::{parse_lino_to_links, LiNo};` (`rust/src/parser.rs:8`). +### C# package `Link.Foundation.Links.Notation` (namespace the same) +- Parser: `Parser.peg` (Pegasus), `@classname Parser`, start rule `document >>`. Usage: `new Parser().Parse(text)` + (link-cli: `BasicQueryProcessor.cs:15-16`, `MixedQueryProcessor.cs:34`). +- `public struct Link` with `Id` and `Values` (an `IList>`). Constructors `(id, values)`, `(values)`, `(params Link[])`, `(id)`. + `ToString()`, `Simplify()`, `Combine()`, `EscapeReference(string)`, `ToLinkOrIdString()`, and tuple implicit conversions, + including `(id, source, target)`. +- Formatting: `IListExtensions.Format(this IList>)` (newline-joined), `Format(links, bool lessParentheses)`, + `Format(links, FormatOptions)`, `LinkFormatExtensions.FormatWithOptions(this Link, FormatOptions)`. + `FormatOptions { LessParentheses, MaxLineLength=80, IndentLongLines, MaxInlineRefs, ... }`. Also `LinksGroup`, `FormatConfig`. +- Other related libraries: `lino-objects-codec` (NuGet `Lino.Objects.Codec`, crate `lino-objects-codec`; JS `jsonToLino`/`linoToJson`), + `Data.Doublets.Lino` (C# `LinoImporter`/`LinoExporter`/`ILinoStorage`), and `lino-tokenizer` (*"tokenize Unicode String as sequence of references"*). + +## 7. Known wire protocols (V = verified via web fetch on 2026-10-04; K = from model knowledge) +- **PostgreSQL** (V) https://www.postgresql.org/docs/current/protocol-overview.html : + *"The first byte of a message identifies the message type, and the next four bytes specify the length of the rest of the + message (this length count includes itself, but not the message-type byte)."* The startup message has no type byte. + The length is Int32 big-endian. Default port 5432 (K). Message formats: https://www.postgresql.org/docs/current/protocol-message-formats.html +- **MySQL** (V) https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_packets.html : header is + `int<3> payload_length` (little-endian) + `int<1> sequence_id`. Payloads of `2^24-1` bytes or more are split, with `ff ff ff` marking a continuation. + The sequence id resets at each command. (K) Length-encoded integer (https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_dt_integers.html): + values below 251 take 1 byte; `0xFC` prefix + 2 bytes; `0xFD` + 3 bytes; `0xFE` + 8 bytes (`0xFB` is NULL in rows). This is width escalation by a prefix byte. Port 3306. +- **Redis RESP2/RESP3** (K) https://redis.io/docs/latest/develop/reference/protocol-spec/ : text, CRLF-terminated, and + the first byte is the type. RESP2 types: `+` simple string, `-` error, `:` integer, `$` bulk string, `*` array. + RESP3 adds `_` null, `#` bool, `,` double, `(` big number, `!` bulk error, `=` verbatim, `%` map, `~` set, + `>` push, `|` attribute. `HELLO 3` switches the version. Port 6379. Inline commands are also accepted. This is a good model for a human-typable text mode. +- **SQLite varint** (K) https://www.sqlite.org/fileformat2.html#varint : 1 to 9 bytes, big-endian. Each of the first 8 bytes carries + 7 bits and a high-bit continuation flag, and the 9th byte carries all 8 bits. This lets it reach 64 bits in 9 bytes. +- **LEB128 / protobuf varint** (K) https://protobuf.dev/programming-guides/encoding/ and https://en.wikipedia.org/wiki/LEB128 : + 7 bits per byte, least significant group first, and MSB = continuation. Up to 10 bytes for 64-bit. ZigZag (`(n<<1)^(n>>63)`) is used for signed values. + Protobuf messages have no self-delimitation, so streams use a varint length prefix (`writeDelimitedTo`). +- **CBOR, RFC 8949** (V) https://www.rfc-editor.org/rfc/rfc8949.html : the initial byte is a 3-bit major type plus a 5-bit additional information field. + *"Less than 24: The argument's value is the value of the additional information. 24, 25, 26, or 27: The argument's value + is held in the following 1, 2, 4, or 8 bytes, respectively, in network byte order."* A value of 31 means indefinite length (types 2-5). + The major types are 0 uint, 1 negative int, 2 bytes, 3 UTF-8 text, 4 array, 5 map, 6 tag, and 7 float/simple. **This is the closest match to the issue's + 8/16/32/64 width scheme.** Major types 0 and 1 also mirror "internal vs external" as two disjoint integer spaces. + CBOR sequences (RFC 8742) concatenate items with no framing. +- **MessagePack** (K) https://github.com/msgpack/msgpack/blob/master/spec.md : positive fixint `0x00-0x7f`, + negative fixint `0xe0-0xff`, `uint8/16/32/64 = 0xcc/0xcd/0xce/0xcf`, `int8..64 = 0xd0..0xd3`, fixarray `0x90-0x9f`, + `array16 0xdc`, `array32 0xdd`, `str8/16/32 0xd9/0xda/0xdb`, `bin8/16/32 0xc4/0xc5/0xc6`. Big-endian. +- **Cap'n Proto** (K) https://capnproto.org/encoding.html : 64-bit words, little-endian. The stream framing is a segment table: + `uint32 (segmentCount-1)`, then `uint32` size in words for each segment, padded to 8 bytes, then the segments. Pointers are offsets and the format is zero-copy. +- **FlatBuffers** (K) https://flatbuffers.dev/internals/ : little-endian, 32-bit `uoffset_t` offsets, vtables for tables, + and zero-copy access. There is no built-in stream framing; a size prefix is optional (`FinishSizePrefixed`). +- **Neo4j Bolt / PackStream** (K, graph-DB precedent) https://neo4j.com/docs/bolt/current/ : TCP port 7687. Messages are split into + chunks, each with a 2-byte big-endian length, and a `00 00` chunk ends the message. The handshake starts with the magic `60 60 B0 17` plus 4 version proposals. + PackStream integers use `TINY_INT` inline (-16..127), then markers `0xC8/0xC9/0xCA/0xCB` for INT_8/16/32/64. This is again width escalation by a marker. + +### Design hints derived from the above +- Binary frame idea: a magic/version handshake (Bolt), then per message `type byte + length` (PostgreSQL) or a varint length. +- The header can carry `linkCount`, and the reference width follows from it: `count ≤ 2^(w-1)-1` (hybrid) or `≤ 2^w-1` (plain). + This is equivalent to choosing a CBOR-like 1/2/4/8-byte argument once per message instead of once per value. +- The sequence section's `size` field could be a varint (LEB128) or reuse the per-message width. Sequence ids of + `count+1, count+2, ...` sit in the same reference space and must be counted when choosing the width. +- Text mode: one LiNo query per line (`text/lino-line` style) or length-prefixed UTF-8. Responses use linksql's QueryReport encoded via + lino-objects-codec (`((operation ...) (matched ...) (created ...) (updated ...) (deleted ...))`). Errors use + linksql error names or the RFC 9457-in-LiNo shape. From b4cb6dbad1bdfdceebc1b5b9e364b3393ac52952 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 18:50:18 +0000 Subject: [PATCH 12/44] docs: describe clink --serve / --connect and the LiNo protocols --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ csharp/README.md | 13 +++++++++++++ docs/ARCHITECTURE.md | 16 +++++++++++++++- docs/HOW-IT-WORKS.md | 27 +++++++++++++++++++++++++++ docs/REQUIREMENTS.md | 3 ++- rust/README.md | 13 +++++++++++++ 6 files changed, 111 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 055c588..41cbaaf 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ package built from `doublets-rs`. - [docs/case-studies/issue-92/README.md](docs/case-studies/issue-92/README.md): evidence and analysis behind the dual CLI + library packaging and unified API documentation site. - [docs/case-studies/issue-94/README.md](docs/case-studies/issue-94/README.md): evidence and analysis for the optional transactions and version-control layers. - [docs/case-studies/issue-98/README.md](docs/case-studies/issue-98/README.md): evidence and analysis for making the libraries reusable as a doublets-backed transactional store. +- [docs/case-studies/issue-105/README.md](docs/case-studies/issue-105/README.md): requirements, prior art and the wire formats of the LiNo text and binary protocols over TCP. ### API references @@ -416,6 +417,40 @@ See [rust/README.md](rust/README.md#use-as-a-library), [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md#embedding-the-library) for the details. +## Serve a database over TCP + +`--serve` exposes a database over TCP. `--connect` sends one query to a +served database and prints the reply: + +```bash +clink --db db.links --serve 127.0.0.1:7777 # terminal 1 +clink --connect 127.0.0.1:7777 '() ((1 1))' # terminal 2 +() ((1: 1 1)) +clink --connect 127.0.0.1:7777 # no query: list every link +(1: 1 1) +``` + +A request is a substitution query, so it covers create, read, update and +delete. The reply has one `(before) (after)` line per change, the same as +`--changes`. A failed query gets `(error: 'message')` back. + +There are two interchangeable protocols: + +- **text**: UTF-8 LiNo. Each message ends with a line holding a single `.`, + so you can also talk to the server with `nc`. +- **binary**: `--protocol binary`. Each packet carries its link count, and + references are 8, 16, 32 or 64 bits wide depending on that count. Three + optional decorators can each be switched on independently: + - `--external-references`: numbers and characters travel as Hybrid external + references. + - `--sequences`: a variable-length sequence section. + - `--progressive-widths`: the reference width grows with the address. + +The server detects the protocol of each message and replies in the same one. +C# and Rust servers and clients work with each other: see +[`examples/tcp/`](examples/tcp) and +[docs/case-studies/issue-105](docs/case-studies/issue-105/README.md). + ## Update single link Update link with index 1 and source 1 and target 1, changing target to 2. @@ -586,6 +621,12 @@ Both the C# NuGet tool and the Rust CLI support every option below. | `--triggers` | bool | `false` | _None_ | Enable persistent transformation triggers for the command | | `--triggers-file` | string | `.triggers.links` | _None_ | Path to the persistent transformation trigger links database | | `--embed-triggers` | bool | `false` | _None_ | Store persistent transformation triggers in the main links database | +| `--serve` | string | _None_ | _None_ | Serve the database over TCP at `host:port` (port `0` picks a free port) | +| `--connect` | string | _None_ | _None_ | Send the query to a `clink --serve` server at `host:port` and print the reply | +| `--protocol` | string | detected | _None_ | `text` or `binary` (with `--serve`, also `any`, which is the default) | +| `--external-references` | bool | `false` | _None_ | Binary protocol: send numbers and characters as external references | +| `--sequences` | bool | `false` | _None_ | Binary protocol: use the variable-length sequence section | +| `--progressive-widths` | bool | `false` | _None_ | Binary protocol: grow the reference width with the address | The query can be passed as the first positional argument or through `--query`, `--apply`, or `--do`. In the Rust CLI, `--query` takes precedence when both diff --git a/csharp/README.md b/csharp/README.md index 9b978f7..b3ffd21 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -76,6 +76,19 @@ library API for explicit batches. End-to-end demo scripts live in [`examples/transactions/`](../examples/transactions) and [`examples/version-control/`](../examples/version-control). +### Serve Over TCP + +```bash +clink --db data.links --serve 127.0.0.1:7777 # serve a database +clink --connect 127.0.0.1:7777 '() ((1 1))' # text protocol +clink --connect 127.0.0.1:7777 --protocol binary --sequences # binary protocol, list all links +``` + +In the library, `LinksServer` serves any `INamedTypesLinks`, and +`LinksClient` queries a server through either `TextLinoProtocol` or +`BinaryLinoProtocol`. The wire formats are described in +[docs/case-studies/issue-105](../docs/case-studies/issue-105/README.md). + ### Use as a library `Foundation.Data.Doublets.Cli.Library` is usable as an embedded diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 29c3611..088f746 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -46,6 +46,12 @@ Key files: - `VersionControlDecorator.cs`: optional version-control layer that sits above the transactions decorator and provides branching, tagging, and time-travel checkout. +- `Protocol/`: LiNo over TCP (#105). This folder contains: + - `TextLinoProtocol` and `BinaryLinoProtocol` (`LinoProtocols.cs`) + - the binary `LinksPacket` + - `LinoMapping` (document ↔ packet) + - `LinoFormat` (canonical text) + - `LinksServer` and `LinksClient` - `LinksFileLock.cs`: advisory locking of a database's `.lock` sidecar plus the `StorageRevision` fingerprint used to detect writes by other processes. @@ -79,11 +85,19 @@ Key files: transitions-log, retention-policy, and commit-mode types. - `rust/src/version_control/`: optional version-control decorator with branching, tagging, and time-travel checkout. +- `rust/src/protocol/`: LiNo over TCP (#105). The module mirrors the C# + `Protocol/` folder file for file: + - `protocols.rs` + - `packet.rs` + - `mapping.rs` + - `format.rs` + - `server.rs` + - `client.rs` Main Rust dependencies: - `doublets = "0.5.0"` for links storage foundations. -- `links-notation = "0.16.1"` for LiNo parsing. +- `links-notation = "0.22.0"` for LiNo parsing. - `lino-arguments = "0.3.0"` for argument initialization compatibility. - `anyhow` and `thiserror` for error handling. diff --git a/docs/HOW-IT-WORKS.md b/docs/HOW-IT-WORKS.md index 90edfae..43d05c2 100644 --- a/docs/HOW-IT-WORKS.md +++ b/docs/HOW-IT-WORKS.md @@ -281,6 +281,33 @@ rewinds everything; checkout to a higher seq replays as needed. When no version-control flag is passed, no `versioncontrol.links` sidecar is created. +## Serving Over TCP + +`clink --serve host:port` keeps a database open and answers LiNo requests +over TCP. `clink --connect host:port [query]` sends one request and prints +the reply. + +- A request is the same substitution expression the CLI takes. The reply is + the list of `(before) (after)` changes. An empty request lists every link. +- Each connection has its own thread. Requests run one at a time under a + store lock. +- The protocol is detected per message, and the reply uses the same one: + - **Text**: UTF-8 LiNo. A line holding a single `.` ends a message, and + leading dots are doubled. + - **Binary**: a packet whose first byte is `0x10 | flags`. It continues + with the number of fixed doublets and sequences, then the doublets + `source target` and the sequences `size ref…`. Addresses are implicit, + and a reference is 1, 2, 4 or 8 bytes wide, depending on the highest + address. +- Numbers and strings are encoded as links by default: unary numbers and + code-point lists under marker points. Three options change the encoding: + - `--external-references` sends them as Hybrid external references instead. + - `--sequences` replaces cons chains with variable-length records. + - `--progressive-widths` lets early links use narrower references. + +The full wire format, golden vectors and design notes are in +[case-studies/issue-105](case-studies/issue-105/README.md). + ## Browser Runtime The WebAssembly workbench uses the Rust query processor in the browser. diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index cf389bd..afe66d2 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -50,6 +50,7 @@ the core CLI behavior and is also used by the WebAssembly browser workbench. | Document and test crash safety. | #98 | Durability is stated per backend: a file-mapped store survives a process crash with no explicit save and needs `flush()` only to survive a machine crash, while in-memory stores need `flush()`/`save()` for anything. Tests crash a file-backed store mid-rebuild and reopen it; recovery replays committed-but-unapplied transitions, rolls back uncommitted ones, and is idempotent. `transactions::FileTransitionLog` discards a tail torn by a crash. | | Support multi-process access to one database. | #98 | Both implementations take advisory locks on a `.lock` sidecar (shared for readers, exclusive for writers, blocking and non-blocking) and expose a `StorageRevision` fingerprint that answers "has anyone else written since I last looked?" without reparsing. File-mapped stores are mutated in place, so the inode never changes and other processes' mappings stay valid. | | Keep dependencies current across every language. | #98 | Rust, C#, and JavaScript dependencies were refreshed together, keeping shared libraries on the same version across languages (notably `links-notation` 0.16.1 in both Rust and C#). | +| LiNo substitution operations over TCP. | #105 | C# and Rust expose `--serve` / `--connect` and the `LinksServer` / `LinksClient` library types. Requests are substitution queries covering CRUD. There are two interchangeable protocols, a dot-terminated UTF-8 text protocol and a binary links packet:
  • the binary header carries the link count;
  • references are 8, 16, 32 or 64 bits wide depending on that count;
  • three optional decorators, external (Hybrid) references, a variable-length sequence section and progressive widths, can each be switched on independently.
Both ports produce identical bytes, and Rust and C# servers and clients work with each other. `links-notation` was upgraded to 0.22.0 in both languages, because older versions parsed nested input in exponential time. | | Separate code by implementation language. | #63, #64, #77, #79 | C# code and release helpers live under `csharp/`; Rust code, release helpers, and the WebAssembly wrapper crate live under `rust/`; the browser app and JavaScript lockfile live under `js/`. | | Provide Rust parity for core behavior. | #63, #67, #68 | Rust mirrors query processing, names, import/export, structure formatting, and Unicode sequence support. | | Run in a browser. | #12, #52, #69, #70 | The Rust query processor is wrapped with `wasm-bindgen` and surfaced through a React/Vite workbench. | @@ -60,7 +61,7 @@ the core CLI behavior and is also used by the WebAssembly browser workbench. | Requirement | Source | Current state | |-------------|--------|---------------| | Unify shell quoting across platforms. | #18, PR #49 | Open. Documentation still uses Unix-style single quotes in many examples because that is the current repository convention. | -| LiNo protocol server mode. | #30, PR #41 | Open. Proposed server mode is not part of the merged CLI. | +| LiNo protocol server mode. | #30, PR #41 | Partly covered by #105: `clink --serve` exposes a database over TCP with text and binary LiNo protocols. | | Benchmark CLI versus protocol server access. | #31, PR #40 | Open. Depends on server mode. | | LINO REST API. | #32, PR #39 | Open. Not part of the merged CLI. | | LINO GRPC-style API. | #33, PR #38 | Open. Not part of the merged CLI. | diff --git a/rust/README.md b/rust/README.md index 5a06446..6e82e59 100644 --- a/rust/README.md +++ b/rust/README.md @@ -59,6 +59,19 @@ End-to-end demo scripts live in [`examples/transactions/`](../examples/transactions) and [`examples/version-control/`](../examples/version-control). +### Serve Over TCP + +```bash +clink --db data.links --serve 127.0.0.1:7777 # serve a database +clink --connect 127.0.0.1:7777 '() ((1 1))' # text protocol +clink --connect 127.0.0.1:7777 --protocol binary --sequences # binary protocol, list all links +``` + +In the library, `link_cli::protocol::LinksServer` serves any `NamedTypeLinks` store, and +`LinksClient` queries a server through either `TextLinoProtocol` or +`BinaryLinoProtocol`. The wire formats are described in +[docs/case-studies/issue-105](../docs/case-studies/issue-105/README.md). + ## Use as a library `link_cli` is usable as an embedded, doublets-backed transactional store, not From fcb1fecf12468ed7849ac0796ccd2d3acabbd9ec Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 18:52:05 +0000 Subject: [PATCH 13/44] fix(wasm): lock links-notation 0.22.0 in the wasm crate too --- rust/wasm/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/wasm/Cargo.lock b/rust/wasm/Cargo.lock index 7fcbd35..35c1b12 100644 --- a/rust/wasm/Cargo.lock +++ b/rust/wasm/Cargo.lock @@ -361,9 +361,9 @@ dependencies = [ [[package]] name = "links-notation" -version = "0.16.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0068a55e836850caf98abf96117045ab641af446ab0816557360b318fd9e5cfe" +checksum = "120d3b63178b82b0a416d7180f3d3c57170a8bbb00ffac4975d27779013fc4be" dependencies = [ "links-notation-macro", "nom", From ccdd4135e309542f0e8bbe1c3e5a91600bfecc0f Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 18:54:39 +0000 Subject: [PATCH 14/44] style(csharp): apply dotnet format --- .../Protocol/LinksServerTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs index 2a9f83f..72f9245 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs @@ -28,7 +28,8 @@ public RunningServer(LinksServerOptions? options = null) { _failure = error; } - }) { IsBackground = true }; + }) + { IsBackground = true }; _thread.Start(); } From fbcbfe85e3d10e4c562c43e2673048af8fefff63 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 23:39:23 +0000 Subject: [PATCH 15/44] feat(rust): RemoteLinks decorator implementing Links, Doublets and NamedTypeLinks over the LiNo protocols --- rust/src/protocol/links_operations.rs | 429 ++++++++++++++++++++++++++ rust/src/protocol/mod.rs | 4 + rust/src/protocol/remote_links.rs | 311 +++++++++++++++++++ rust/src/protocol/server.rs | 4 + rust/tests/common/mod.rs | 77 +++++ rust/tests/protocol_tcp_tests.rs | 76 +---- 6 files changed, 832 insertions(+), 69 deletions(-) create mode 100644 rust/src/protocol/links_operations.rs create mode 100644 rust/src/protocol/remote_links.rs create mode 100644 rust/tests/common/mod.rs diff --git a/rust/src/protocol/links_operations.rs b/rust/src/protocol/links_operations.rs new file mode 100644 index 0000000..0c2c825 --- /dev/null +++ b/rust/src/protocol/links_operations.rs @@ -0,0 +1,429 @@ +//! The links interface (`ILinks` in C#, [`doublets::Links`] in Rust) as LiNo +//! documents, so a store behind a [`LinksServer`](super::LinksServer) is used +//! exactly like a local one through [`RemoteLinks`](super::RemoteLinks). +//! +//! Every request is one top-level link named after the operation and holding +//! exactly one value. The substitution query language ignores that shape (a +//! query needs a restriction *and* a substitution), so an operation never +//! collides with a query sent to the same server. +//! +//! | Request | Reply | +//! |----------------------------------|----------------------------------------| +//! | `(count: (index source target))` | `(count: N)` | +//! | `(each: (index source target))` | one `(index: source target)` per match | +//! | `(create: (source target))` | `() ((index: source target))` | +//! | `(update: (index source target))`| `((index: s t)) ((index: source target))` | +//! | `(delete: index)` | `((index: source target)) ()` per removed link | +//! | `(get-name: link)` | `(name: 'text')`, or nothing | +//! | `(set-name: (link 'text'))` | `(link: N)` | +//! | `(get-by-name: 'text')` | `(link: N)`, or nothing | +//! | `(remove-name: link)` | nothing | +//! +//! A restriction holds up to three parts, matched like the raw links +//! interface matches a query: none matches every link, `(index)` one link, +//! `(index value)` links of that index whose source or target is `value`, +//! and `(index source target)` matches part by part. `*` matches anything. +//! Every reference in a reply is a number; failures are `(error: 'message')`. + +use super::error::{ProtocolError, ProtocolResult}; +use super::mapping::LinoDocument; +use crate::link::Link; +use crate::named_type_links::NamedTypeLinks; +use links_notation::LiNo; + +/// The wire spelling of "any value" in a restriction. +pub const ANY: &str = "*"; + +/// One part of a restriction: `None` matches any value. +pub type Part = Option; + +/// One call of the links interface. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LinksOperation { + /// Number of links matching the restriction. + Count(Vec), + /// Every link matching the restriction, ordered by index. + Each(Vec), + /// Creates a link. + Create { source: u32, target: u32 }, + /// Points an existing link at a new source and target. + Update { + index: u32, + source: u32, + target: u32, + }, + /// Deletes a link, and every link that still refers to it. + Delete(u32), + /// The name of a link. + GetName(u32), + /// Names a link. + SetName(u32, String), + /// The link with a name. + GetByName(String), + /// Removes the name of a link. + RemoveName(u32), +} + +/// A `(before, after)` change; a missing side is [`Link::null`]. +pub type Change = (Link, Link); + +impl LinksOperation { + /// The request document for this operation. + pub fn to_document(&self) -> LinoDocument { + let (name, argument) = match self { + Self::Count(restriction) => ("count", restriction_lino(restriction)), + Self::Each(restriction) => ("each", restriction_lino(restriction)), + Self::Create { source, target } => ("create", numbers_lino(&[*source, *target])), + Self::Update { + index, + source, + target, + } => ("update", numbers_lino(&[*index, *source, *target])), + Self::Delete(index) => ("delete", number(*index)), + Self::GetName(index) => ("get-name", number(*index)), + Self::SetName(index, name) => ( + "set-name", + group(vec![number(*index), LiNo::Ref(name.clone())]), + ), + Self::GetByName(name) => ("get-by-name", LiNo::Ref(name.clone())), + Self::RemoveName(index) => ("remove-name", number(*index)), + }; + vec![named(name, vec![argument])] + } + + /// Recognises an operation request; `Ok(None)` for any other document, + /// such as a substitution query. + pub fn from_document(document: &[LiNo]) -> ProtocolResult> { + let [LiNo::Link { + id: Some(name), + values, + }] = document + else { + return Ok(None); + }; + let [argument] = values.as_slice() else { + return Ok(None); + }; + let operation = match name.as_str() { + "count" => Self::Count(parse_restriction(argument)?), + "each" => Self::Each(parse_restriction(argument)?), + "create" => { + let [source, target] = parse_numbers::<2>(argument)?; + Self::Create { source, target } + } + "update" => { + let [index, source, target] = parse_numbers::<3>(argument)?; + Self::Update { + index, + source, + target, + } + } + "delete" => Self::Delete(parse_number(argument)?), + "get-name" => Self::GetName(parse_number(argument)?), + "set-name" => match parts(argument) { + [index, LiNo::Ref(name)] => Self::SetName(parse_number(index)?, name.clone()), + _ => return Err(malformed_argument("set-name", argument)), + }, + "get-by-name" => match argument { + LiNo::Ref(name) => Self::GetByName(name.clone()), + _ => return Err(malformed_argument("get-by-name", argument)), + }, + "remove-name" => Self::RemoveName(parse_number(argument)?), + _ => return Ok(None), + }; + Ok(Some(operation)) + } + + /// Runs the operation against `storage` and returns the reply document. + pub fn execute( + &self, + storage: &mut S, + ) -> anyhow::Result { + Ok(match self { + Self::Count(restriction) => { + let count = matching(storage, restriction).len(); + vec![named("count", vec![LiNo::Ref(count.to_string())])] + } + Self::Each(restriction) => matching(storage, restriction) + .iter() + .map(link_lino) + .collect(), + Self::Create { source, target } => { + let index = storage.create(*source, *target); + let created = storage + .get_link(index) + .unwrap_or_else(|| Link::new(index, *source, *target)); + storage.save()?; + changes_document(&[(Link::null(), created)]) + } + Self::Update { + index, + source, + target, + } => { + let before = storage.update(*index, *source, *target)?; + let after = storage + .get_link(*index) + .unwrap_or_else(|| Link::new(*index, *source, *target)); + storage.save()?; + changes_document(&[(before, after)]) + } + Self::Delete(index) => { + let mut changes = Vec::new(); + storage + .delete_observed(*index, &mut |before, after| changes.push((before, after)))?; + storage.save()?; + changes_document(&changes) + } + Self::GetName(index) => match storage.get_name(*index)? { + Some(name) => vec![named("name", vec![LiNo::Ref(name)])], + None => Vec::new(), + }, + Self::SetName(index, name) => { + let link = storage.set_name(*index, name)?; + storage.save()?; + link_reply(Some(link)) + } + Self::GetByName(name) => link_reply(storage.get_by_name(name)?), + Self::RemoveName(index) => { + storage.remove_name(*index)?; + storage.save()?; + Vec::new() + } + }) + } +} + +/// Whether `link` matches `restriction`, with the shapes of the raw links +/// interface: see the [module documentation](self). +pub fn matches(link: &Link, restriction: &[Part]) -> bool { + let is = |part: Part, value: u32| part.is_none_or(|part| part == value); + match *restriction { + [] => true, + [index] => is(index, link.index), + [index, value] => { + is(index, link.index) && (is(value, link.source) || is(value, link.target)) + } + [index, source, target] => { + is(index, link.index) && is(source, link.source) && is(target, link.target) + } + _ => false, + } +} + +fn matching(storage: &mut S, restriction: &[Part]) -> Vec { + let mut links = match restriction.first() { + Some(Some(index)) => storage.get_link(*index).into_iter().collect(), + _ => storage.all_links(), + }; + links.retain(|link| matches(link, restriction)); + links.sort_by_key(|link| link.index); + links +} + +/// The `count` of a `(count: N)` reply. +pub fn parse_count(document: &[LiNo]) -> ProtocolResult { + match document { + [LiNo::Link { + id: Some(id), + values, + }] if id == "count" => match values.as_slice() { + [count] => parse_number(count), + _ => Err(malformed_reply("count", document)), + }, + _ => Err(malformed_reply("count", document)), + } +} + +/// The links of an `each` reply. +pub fn parse_links(document: &[LiNo]) -> ProtocolResult> { + document.iter().map(parse_link).collect() +} + +/// The `(before) (after)` pairs of a `create`, `update` or `delete` reply. +pub fn parse_changes(document: &[LiNo]) -> ProtocolResult> { + document + .iter() + .map(|change| match parts(change) { + [before, after] => Ok((parse_change_side(before)?, parse_change_side(after)?)), + _ => Err(malformed_reply("change", document)), + }) + .collect() +} + +/// The name of a `get-name` reply. +pub fn parse_name(document: &[LiNo]) -> ProtocolResult> { + match document { + [] => Ok(None), + [LiNo::Link { + id: Some(id), + values, + }] if id == "name" => match values.as_slice() { + [LiNo::Ref(name)] => Ok(Some(name.clone())), + _ => Err(malformed_reply("name", document)), + }, + _ => Err(malformed_reply("name", document)), + } +} + +/// The link of a `set-name` or `get-by-name` reply. +pub fn parse_link_reply(document: &[LiNo]) -> ProtocolResult> { + match document { + [] => Ok(None), + [LiNo::Link { + id: Some(id), + values, + }] if id == "link" => match values.as_slice() { + [link] => parse_number(link).map(Some), + _ => Err(malformed_reply("link", document)), + }, + _ => Err(malformed_reply("link", document)), + } +} + +/// A reply listing `changes`, one `(before) (after)` line each. +pub fn changes_document(changes: &[Change]) -> LinoDocument { + changes + .iter() + .map(|(before, after)| group(vec![change_side(before), change_side(after)])) + .collect() +} + +/// `(index: source target)` with plain numbers. +pub fn link_lino(link: &Link) -> LiNo { + named( + &link.index.to_string(), + vec![number(link.source), number(link.target)], + ) +} + +fn link_reply(link: Option) -> LinoDocument { + link.map(|link| vec![named("link", vec![number(link)])]) + .unwrap_or_default() +} + +fn change_side(link: &Link) -> LiNo { + if link.is_null() { + group(Vec::new()) + } else { + group(vec![link_lino(link)]) + } +} + +fn parse_change_side(side: &LiNo) -> ProtocolResult { + match side { + LiNo::Link { id: None, values } if values.is_empty() => Ok(Link::null()), + LiNo::Link { id: None, values } if values.len() == 1 => parse_link(&values[0]), + // `((index: source target))` loses its wrapper when the side is the + // single named link itself. + link @ LiNo::Link { id: Some(_), .. } => parse_link(link), + _ => Err(ProtocolError::malformed(format!( + "expected a change side, found {side:?}" + ))), + } +} + +fn parse_link(link: &LiNo) -> ProtocolResult { + match link { + LiNo::Link { + id: Some(index), + values, + } => match values.as_slice() { + [source, target] => Ok(Link::new( + parse_text_number(index)?, + parse_number(source)?, + parse_number(target)?, + )), + _ => Err(ProtocolError::malformed(format!( + "expected (index: source target), found {link:?}" + ))), + }, + _ => Err(ProtocolError::malformed(format!( + "expected (index: source target), found {link:?}" + ))), + } +} + +fn restriction_lino(restriction: &[Part]) -> LiNo { + group( + restriction + .iter() + .map(|part| part.map_or_else(|| LiNo::Ref(ANY.to_string()), number)) + .collect(), + ) +} + +fn parse_restriction(argument: &LiNo) -> ProtocolResult> { + let parts = parts(argument); + if parts.len() > 3 { + return Err(malformed_argument("restriction", argument)); + } + parts + .iter() + .map(|part| match part { + LiNo::Ref(text) if text == ANY => Ok(None), + part => parse_number(part).map(Some), + }) + .collect() +} + +fn numbers_lino(numbers: &[u32]) -> LiNo { + group(numbers.iter().copied().map(number).collect()) +} + +fn parse_numbers(argument: &LiNo) -> ProtocolResult<[u32; N]> { + let numbers = parts(argument) + .iter() + .map(parse_number) + .collect::>>()?; + numbers + .try_into() + .map_err(|_| malformed_argument(&format!("{N} numbers"), argument)) +} + +/// The values of an unnamed group; a lone reference is a group of one, since +/// the canonical document model unwraps `(x)` to `x`. +fn parts(argument: &LiNo) -> &[LiNo] { + match argument { + LiNo::Link { id: None, values } => values, + reference => std::slice::from_ref(reference), + } +} + +fn parse_number(value: &LiNo) -> ProtocolResult { + match value { + LiNo::Ref(text) => parse_text_number(text), + _ => Err(ProtocolError::malformed(format!( + "expected a number, found {value:?}" + ))), + } +} + +fn parse_text_number(text: &str) -> ProtocolResult { + text.parse() + .map_err(|_| ProtocolError::malformed(format!("expected a number, found '{text}'"))) +} + +fn number(value: u32) -> LiNo { + LiNo::Ref(value.to_string()) +} + +fn named(id: &str, values: Vec>) -> LiNo { + LiNo::Link { + id: Some(id.to_string()), + values, + } +} + +fn group(values: Vec>) -> LiNo { + LiNo::Link { id: None, values } +} + +fn malformed_argument(expected: &str, argument: &LiNo) -> ProtocolError { + ProtocolError::malformed(format!("expected {expected}, found {argument:?}")) +} + +fn malformed_reply(expected: &str, document: &[LiNo]) -> ProtocolError { + ProtocolError::malformed(format!("expected a {expected} reply, found {document:?}")) +} diff --git a/rust/src/protocol/mod.rs b/rust/src/protocol/mod.rs index 27e1b9d..c758758 100644 --- a/rust/src/protocol/mod.rs +++ b/rust/src/protocol/mod.rs @@ -36,20 +36,24 @@ mod client; mod error; mod format; +pub mod links_operations; mod mapping; pub mod packet; mod protocols; +mod remote_links; mod server; pub use client::LinksClient; pub use error::{ProtocolError, ProtocolResult}; pub use format::{format_document, format_link, format_reference, parse_document}; +pub use links_operations::LinksOperation; pub use mapping::{decode_document, encode_document, BinaryLinoOptions, LinoDocument}; pub use packet::{DecodeLimits, LinksPacket, Reference}; pub use protocols::{ is_binary_start, read_any_document, BinaryLinoProtocol, LinoConnection, LinoProtocol, MessageFormat, TextLinoProtocol, }; +pub use remote_links::RemoteLinks; pub use server::{ error_document, error_message, execute_request, AcceptedProtocols, LinksServer, ServerOptions, ShutdownHandle, diff --git a/rust/src/protocol/remote_links.rs b/rust/src/protocol/remote_links.rs new file mode 100644 index 0000000..7d25fc0 --- /dev/null +++ b/rust/src/protocol/remote_links.rs @@ -0,0 +1,311 @@ +//! [`RemoteLinks`]: a links store that lives behind a [`LinksServer`](super::LinksServer). +//! +//! It implements the same interfaces as a local store — [`doublets::Links`], +//! [`doublets::Doublets`] and [`NamedTypeLinks`] — so code written against +//! them, the [`QueryProcessor`](crate::QueryProcessor) included, switches +//! from a local file to a server by swapping one value: +//! +//! ```no_run +//! use link_cli::protocol::{RemoteLinks, TextLinoProtocol}; +//! use link_cli::{NamedTypeLinks, NamedTypesDecorator, QueryProcessor}; +//! +//! fn run(store: &mut impl NamedTypeLinks) -> anyhow::Result<()> { +//! QueryProcessor::new(false).process_query(store, "() ((1 1))")?; +//! Ok(()) +//! } +//! +//! run(&mut NamedTypesDecorator::new("db.links", false)?)?; +//! run(&mut RemoteLinks::connect("127.0.0.1:8080", TextLinoProtocol::new())?)?; +//! # Ok::<(), anyhow::Error>(()) +//! ``` +//! +//! Every call is one [`LinksOperation`] round trip. The methods of the +//! upstream and CLI traits that cannot return an error panic when the +//! connection fails, the way a local store panics when its file does; the +//! inherent methods return every failure as a [`ProtocolError`]. + +use super::client::LinksClient; +use super::error::{ProtocolError, ProtocolResult}; +use super::links_operations::{ + parse_changes, parse_count, parse_link_reply, parse_links, parse_name, Change, LinksOperation, + Part, +}; +use super::mapping::LinoDocument; +use super::protocols::LinoProtocol; +use crate::link::Link; +use crate::link_storage::ChangeObserver; +use crate::link_storage_doublets::link_storage_constants; +use crate::named_type_links::NamedTypeLinks; +use doublets::data::{Flow, LinksConstants, ReadHandler, WriteHandler}; +use doublets::{Doublets, Error, Link as DoubletsLink, Links}; +use std::net::ToSocketAddrs; +use std::sync::{Mutex, PoisonError}; + +/// A links store served by a [`LinksServer`](super::LinksServer), usable +/// wherever a local store is. +#[derive(Debug)] +pub struct RemoteLinks { + client: Mutex, +} + +impl RemoteLinks { + /// Connects to the server at `address`. + pub fn connect( + address: impl ToSocketAddrs, + protocol: impl LinoProtocol + 'static, + ) -> ProtocolResult { + Ok(Self::new(LinksClient::connect(address, protocol)?)) + } + + /// Uses an existing connection. + pub fn new(client: LinksClient) -> Self { + Self { + client: Mutex::new(client), + } + } + + /// Runs one operation on the server and returns its reply. + pub fn execute(&self, operation: &LinksOperation) -> ProtocolResult { + self.client + .lock() + .unwrap_or_else(PoisonError::into_inner) + .request(&operation.to_document()) + } + + /// Number of links matching `restriction`. + pub fn count_matching(&self, restriction: &[Part]) -> ProtocolResult { + parse_count(&self.execute(&LinksOperation::Count(restriction.to_vec()))?) + } + + /// Links matching `restriction`, ordered by index. + pub fn links_matching(&self, restriction: &[Part]) -> ProtocolResult> { + parse_links(&self.execute(&LinksOperation::Each(restriction.to_vec()))?) + } + + /// Runs a `create`, `update` or `delete` and returns its changes. + pub fn changes(&self, operation: &LinksOperation) -> ProtocolResult> { + parse_changes(&self.execute(operation)?) + } + + /// The link stored at `index`. + pub fn link(&self, index: u32) -> ProtocolResult> { + Ok(self.links_matching(&[Some(index)])?.pop()) + } + + fn create_remote(&self, source: u32, target: u32) -> ProtocolResult { + self.changes(&LinksOperation::Create { source, target })? + .into_iter() + .find_map(|(before, after)| (before.is_null() && !after.is_null()).then_some(after)) + .ok_or_else(|| ProtocolError::malformed("the create reply holds no created link")) + } + + fn removed_link(changes: &[Change], index: u32) -> ProtocolResult { + changes + .iter() + .find(|(before, _)| before.index == index) + .map(|(before, _)| *before) + .ok_or_else(|| { + ProtocolError::malformed(format!("the reply holds no change of {index}")) + }) + } + + fn parts(&self, query: &[u32]) -> Vec { + let any = self.constants().any; + query + .iter() + .map(|&part| (part != any).then_some(part)) + .collect() + } +} + +/// Unwraps the result of a call whose interface has no way to report a +/// failure. +fn connected(result: ProtocolResult) -> T { + result.unwrap_or_else(|error| panic!("remote links store failed: {error}")) +} + +fn doublets_link(link: &Link) -> DoubletsLink { + DoubletsLink::new(link.index, link.source, link.target) +} + +fn doublets_error(error: ProtocolError) -> Error { + Error::Other(Box::new(error)) +} + +/// Feeds `changes` to `handler` until it asks to stop. +fn replay(changes: &[Change], handler: WriteHandler<'_, u32>) -> Flow { + for (before, after) in changes { + if handler(doublets_link(before), doublets_link(after)) == Flow::Break { + return Flow::Break; + } + } + Flow::Continue +} + +fn part(query: &[u32], index: usize) -> u32 { + query.get(index).copied().unwrap_or(0) +} + +impl Links for RemoteLinks { + fn constants(&self) -> &LinksConstants { + link_storage_constants() + } + + fn count_links(&self, query: &[u32]) -> u32 { + connected(self.count_matching(&self.parts(query))) + } + + /// Creates an empty `(index: 0 0)` link, like every `doublets` store. + fn create_links( + &mut self, + _query: &[u32], + handler: WriteHandler<'_, u32>, + ) -> Result> { + let changes = self + .changes(&LinksOperation::Create { + source: 0, + target: 0, + }) + .map_err(doublets_error)?; + Ok(replay(&changes, handler)) + } + + fn each_links(&self, query: &[u32], handler: ReadHandler<'_, u32>) -> Flow { + for link in connected(self.links_matching(&self.parts(query))) { + if handler(doublets_link(&link)) == Flow::Break { + return Flow::Break; + } + } + Flow::Continue + } + + fn update_links( + &mut self, + query: &[u32], + change: &[u32], + handler: WriteHandler<'_, u32>, + ) -> Result> { + let changes = self + .changes(&LinksOperation::Update { + index: part(query, 0), + source: part(change, 1), + target: part(change, 2), + }) + .map_err(doublets_error)?; + Ok(replay(&changes, handler)) + } + + fn delete_links( + &mut self, + query: &[u32], + handler: WriteHandler<'_, u32>, + ) -> Result> { + let changes = self + .changes(&LinksOperation::Delete(part(query, 0))) + .map_err(doublets_error)?; + Ok(replay(&changes, handler)) + } +} + +impl Doublets for RemoteLinks { + fn get_link(&self, index: u32) -> Option> { + connected(self.link(index)).as_ref().map(doublets_link) + } +} + +impl NamedTypeLinks for RemoteLinks { + fn create(&mut self, source: u32, target: u32) -> u32 { + connected(self.create_remote(source, target)).index + } + + /// Asks for links until the server hands out `id`, then deletes the ones + /// handed out on the way, exactly like [`LinkStorage::ensure_created`](crate::LinkStorage::ensure_created). + fn ensure_created(&mut self, id: u32) -> u32 { + if id == 0 || self.exists(id) { + return id; + } + let mut passed_over = Vec::new(); + loop { + let created = NamedTypeLinks::create(self, 0, 0); + if created == id { + break; + } + passed_over.push(created); + } + for address in passed_over { + connected(self.changes(&LinksOperation::Delete(address))); + } + id + } + + fn get_link(&mut self, id: u32) -> Option { + connected(self.link(id)) + } + + fn exists(&mut self, id: u32) -> bool { + connected(self.link(id)).is_some() + } + + fn update(&mut self, id: u32, source: u32, target: u32) -> anyhow::Result { + let changes = self.changes(&LinksOperation::Update { + index: id, + source, + target, + })?; + Ok(Self::removed_link(&changes, id)?) + } + + fn delete(&mut self, id: u32) -> anyhow::Result { + self.delete_observed(id, &mut |_, _| {}) + } + + fn delete_observed(&mut self, id: u32, observer: ChangeObserver<'_>) -> anyhow::Result { + let changes = self.changes(&LinksOperation::Delete(id))?; + for (before, after) in &changes { + observer(*before, *after); + } + Ok(Self::removed_link(&changes, id)?) + } + + fn all_links(&mut self) -> Vec { + connected(self.links_matching(&[])) + } + + fn search(&mut self, source: u32, target: u32) -> Option { + connected(self.links_matching(&[None, Some(source), Some(target)])) + .first() + .map(|link| link.index) + } + + fn get_or_create(&mut self, source: u32, target: u32) -> u32 { + match self.search(source, target) { + Some(index) => index, + None => NamedTypeLinks::create(self, source, target), + } + } + + fn get_name(&mut self, id: u32) -> anyhow::Result> { + Ok(parse_name(&self.execute(&LinksOperation::GetName(id))?)?) + } + + fn set_name(&mut self, id: u32, name: &str) -> anyhow::Result { + parse_link_reply(&self.execute(&LinksOperation::SetName(id, name.to_string()))?)? + .ok_or_else(|| anyhow::anyhow!("the set-name reply holds no link")) + } + + fn get_by_name(&mut self, name: &str) -> anyhow::Result> { + Ok(parse_link_reply( + &self.execute(&LinksOperation::GetByName(name.to_string()))?, + )?) + } + + fn remove_name(&mut self, id: u32) -> anyhow::Result<()> { + self.execute(&LinksOperation::RemoveName(id))?; + Ok(()) + } + + /// The server saves after every change, so there is nothing to flush. + fn save(&mut self) -> anyhow::Result<()> { + Ok(()) + } +} diff --git a/rust/src/protocol/server.rs b/rust/src/protocol/server.rs index 25df0bd..2806174 100644 --- a/rust/src/protocol/server.rs +++ b/rust/src/protocol/server.rs @@ -8,6 +8,7 @@ use super::error::{ProtocolError, ProtocolResult}; use super::format::format_document; +use super::links_operations::LinksOperation; use super::mapping::LinoDocument; use super::packet::DecodeLimits; use super::protocols::{read_any_document, MessageFormat}; @@ -272,6 +273,9 @@ fn try_execute_request( processor: &QueryProcessor, document: &[LiNo], ) -> anyhow::Result { + if let Some(operation) = LinksOperation::from_document(document)? { + return operation.execute(storage); + } if document.is_empty() { let mut links = storage.all_links(); links.sort_by_key(|link| link.index); diff --git a/rust/tests/common/mod.rs b/rust/tests/common/mod.rs new file mode 100644 index 0000000..14ac958 --- /dev/null +++ b/rust/tests/common/mod.rs @@ -0,0 +1,77 @@ +//! Helpers shared by the integration tests that talk to a [`LinksServer`]. + +#![allow(dead_code)] // Every test crate compiles this module but uses only part of it. + +use link_cli::protocol::{ + BinaryLinoOptions, BinaryLinoProtocol, LinksClient, LinksServer, LinoProtocol, ServerOptions, + TextLinoProtocol, +}; +use link_cli::{LinkStorage, NamedTypesDecorator}; +use std::net::SocketAddr; +use std::thread::{self, JoinHandle}; +use tempfile::NamedTempFile; + +/// A [`LinksServer`] serving a temporary store on an ephemeral port; dropping +/// it stops the server. +pub struct RunningServer { + pub address: SocketAddr, + pub shutdown: link_cli::protocol::ShutdownHandle, + pub thread: Option>, + _database: NamedTempFile, +} + +impl RunningServer { + pub fn start(options: ServerOptions, named: bool) -> Self { + let database = NamedTempFile::new().unwrap(); + let path = database.path().to_path_buf(); + let server = LinksServer::bind("127.0.0.1:0", options).unwrap(); + let address = server.local_addr().unwrap(); + let shutdown = server.shutdown_handle().unwrap(); + let thread = thread::spawn(move || { + if named { + let mut storage = NamedTypesDecorator::new(&path, false).unwrap(); + server.serve(&mut storage).unwrap(); + } else { + let mut storage = LinkStorage::new(&path, false).unwrap(); + server.serve(&mut storage).unwrap(); + } + }); + Self { + address, + shutdown, + thread: Some(thread), + _database: database, + } + } + + pub fn client(&self, protocol: impl LinoProtocol + 'static) -> LinksClient { + LinksClient::connect(self.address, protocol).unwrap() + } +} + +impl Drop for RunningServer { + fn drop(&mut self) { + self.shutdown.shutdown(); + if let Some(thread) = self.thread.take() { + thread.join().unwrap(); + } + } +} + +pub fn protocols() -> Vec> { + let mut protocols: Vec> = vec![Box::new(TextLinoProtocol::new())]; + for external_references in [false, true] { + for sequences in [false, true] { + for progressive_widths in [false, true] { + protocols.push(Box::new(BinaryLinoProtocol::with_options( + BinaryLinoOptions { + external_references, + sequences, + progressive_widths, + }, + ))); + } + } + } + protocols +} diff --git a/rust/tests/protocol_tcp_tests.rs b/rust/tests/protocol_tcp_tests.rs index 048c66b..778ff47 100644 --- a/rust/tests/protocol_tcp_tests.rs +++ b/rust/tests/protocol_tcp_tests.rs @@ -1,78 +1,16 @@ //! LiNo substitution operations over TCP (issue #105). +mod common; + +use common::{protocols, RunningServer}; use link_cli::protocol::{ - AcceptedProtocols, BinaryLinoOptions, BinaryLinoProtocol, LinksClient, LinksServer, - LinoProtocol, ProtocolError, ServerOptions, TextLinoProtocol, + AcceptedProtocols, BinaryLinoOptions, BinaryLinoProtocol, LinksClient, ProtocolError, + ServerOptions, TextLinoProtocol, }; -use link_cli::{LinkStorage, NamedTypesDecorator}; use std::io::{BufRead, BufReader, Read, Write}; -use std::net::{SocketAddr, TcpStream}; -use std::thread::{self, JoinHandle}; +use std::net::TcpStream; +use std::thread; use std::time::Duration; -use tempfile::NamedTempFile; - -struct RunningServer { - address: SocketAddr, - shutdown: link_cli::protocol::ShutdownHandle, - thread: Option>, - _database: NamedTempFile, -} - -impl RunningServer { - fn start(options: ServerOptions, named: bool) -> Self { - let database = NamedTempFile::new().unwrap(); - let path = database.path().to_path_buf(); - let server = LinksServer::bind("127.0.0.1:0", options).unwrap(); - let address = server.local_addr().unwrap(); - let shutdown = server.shutdown_handle().unwrap(); - let thread = thread::spawn(move || { - if named { - let mut storage = NamedTypesDecorator::new(&path, false).unwrap(); - server.serve(&mut storage).unwrap(); - } else { - let mut storage = LinkStorage::new(&path, false).unwrap(); - server.serve(&mut storage).unwrap(); - } - }); - Self { - address, - shutdown, - thread: Some(thread), - _database: database, - } - } - - fn client(&self, protocol: impl LinoProtocol + 'static) -> LinksClient { - LinksClient::connect(self.address, protocol).unwrap() - } -} - -impl Drop for RunningServer { - fn drop(&mut self) { - self.shutdown.shutdown(); - if let Some(thread) = self.thread.take() { - thread.join().unwrap(); - } - } -} - -fn protocols() -> Vec> { - let mut protocols: Vec> = vec![Box::new(TextLinoProtocol::new())]; - for external_references in [false, true] { - for sequences in [false, true] { - for progressive_widths in [false, true] { - protocols.push(Box::new(BinaryLinoProtocol::with_options( - BinaryLinoOptions { - external_references, - sequences, - progressive_widths, - }, - ))); - } - } - } - protocols -} #[test] fn crud_works_over_every_protocol() { From ff6da7ef0aebd5cb9b6d981ad21d66fa72cc8be0 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 23:49:48 +0000 Subject: [PATCH 16/44] fix(rust): report every change an update makes, matching C# merges NamedTypeLinks gains update_observed next to delete_observed; every layer (LinkStorage, NamedTypesDecorator, transactions, version control, persistent transformations, RemoteLinks) forwards the observer, so an update that merges into an existing doublet reports the deletion of the updated link, exactly as AdvancedMixedQueryProcessor does in C#. The server's update operation returns every change, and RemoteLinks is tested against a local store over every protocol. --- rust/src/link_storage.rs | 20 +- rust/src/named_type_links.rs | 103 +++--- rust/src/persistent_transformations.rs | 17 +- rust/src/protocol/links_operations.rs | 15 +- rust/src/protocol/remote_links.rs | 37 +- rust/src/query_processor.rs | 28 +- rust/src/query_processor/mutations.rs | 85 +++-- rust/src/transactions/mod.rs | 20 +- rust/src/version_control/mod.rs | 16 +- rust/tests/common/mod.rs | 11 +- .../query_processor_csharp_parity_tests.rs | 16 + rust/tests/remote_links_tests.rs | 326 ++++++++++++++++++ 12 files changed, 566 insertions(+), 128 deletions(-) create mode 100644 rust/tests/remote_links_tests.rs diff --git a/rust/src/link_storage.rs b/rust/src/link_storage.rs index 0930007..6f86892 100644 --- a/rust/src/link_storage.rs +++ b/rust/src/link_storage.rs @@ -482,6 +482,12 @@ impl LinkStorage { /// ``` /// /// `observer` is that handler. A change with a null `after` is a deletion. + /// + /// The upstream resolver also reports the duplicate a link is merged into + /// as an unchanged `(before, before)` pair — a documented deviation from + /// C#, whose `LinksUniquenessResolver` reports nothing for it. That pair + /// is not a change, so it is not passed on, and `--changes` lists the same + /// records in both languages. pub fn update_observed( &mut self, id: u32, @@ -495,9 +501,17 @@ impl LinkStorage { .ok_or_else(|| LinkError::not_found(id))?; let mut resolved = (&mut *self).with_automatic_uniqueness_and_usages_resolution(); resolved - .update_by_with([id], [id, source, target], &mut |before, after| { - observe(observer, before, after) - }) + .update_by_with( + [id], + [id, source, target], + &mut |before: doublets::Link, after| { + let merged_into = before == after && before.index != id; + if !merged_into { + observer(Link::from(before), Link::from(after)); + } + doublets::data::Flow::Continue + }, + ) .map_err(LinkError::from)?; Ok(before) } diff --git a/rust/src/named_type_links.rs b/rust/src/named_type_links.rs index e68f70d..9bbce72 100644 --- a/rust/src/named_type_links.rs +++ b/rust/src/named_type_links.rs @@ -31,28 +31,41 @@ pub trait NamedTypeLinks { } fn get_link(&mut self, id: u32) -> Option; fn exists(&mut self, id: u32) -> bool; - fn update(&mut self, id: u32, source: u32, target: u32) -> Result; - fn delete(&mut self, id: u32) -> Result; + fn update(&mut self, id: u32, source: u32, target: u32) -> Result { + self.update_observed(id, source, target, &mut |_, _| {}) + } + /// [`Self::update`], reporting every change the update caused. + /// + /// An update that would duplicate an existing doublet is resolved into a + /// merge: the updated link is deleted and its usages are moved to the + /// duplicate, so one call can make several changes. The C# CLI sees them + /// because `AdvancedMixedQueryProcessor` hands a handler to the store: + /// + /// ```csharp + /// links.Update(before, after, (beforeState, afterState) => + /// options.ChangesHandler?.Invoke(beforeState, afterState) ?? links.Constants.Continue); + /// ``` + fn update_observed( + &mut self, + id: u32, + source: u32, + target: u32, + observer: ChangeObserver<'_>, + ) -> Result; + fn delete(&mut self, id: u32) -> Result { + self.delete_observed(id, &mut |_, _| {}) + } /// [`Self::delete`], reporting every change the deletion caused. /// /// Deleting a link cascades into every link that still used it, and each of - /// those removals is a change in its own right. The C# CLI sees them - /// because `AdvancedMixedQueryProcessor.RemoveLinks` hands a handler to the - /// store: + /// those removals is a change in its own right, reported the way + /// `AdvancedMixedQueryProcessor.RemoveLinks` reports them in C#: /// /// ```csharp /// links.Delete(link, (before, after) => /// options.ChangesHandler?.Invoke(before, after) ?? links.Constants.Continue); /// ``` - /// - /// The default implementation reports only the link that was asked for, - /// which is correct for stores that cannot cascade; every decorator over a - /// cascading store overrides it. - fn delete_observed(&mut self, id: u32, observer: ChangeObserver<'_>) -> Result { - let before = self.delete(id)?; - observer(before, Link::null()); - Ok(before) - } + fn delete_observed(&mut self, id: u32, observer: ChangeObserver<'_>) -> Result; fn all_links(&mut self) -> Vec; fn search(&mut self, source: u32, target: u32) -> Option; fn get_or_create(&mut self, source: u32, target: u32) -> u32; @@ -181,12 +194,14 @@ impl NamedTypeLinks for LinkStorage { LinkStorage::exists(self, id) } - fn update(&mut self, id: u32, source: u32, target: u32) -> Result { - LinkStorage::update(self, id, source, target) - } - - fn delete(&mut self, id: u32) -> Result { - LinkStorage::delete(self, id) + fn update_observed( + &mut self, + id: u32, + source: u32, + target: u32, + observer: ChangeObserver<'_>, + ) -> Result { + LinkStorage::update_observed(self, id, source, target, observer) } fn delete_observed(&mut self, id: u32, observer: ChangeObserver<'_>) -> Result { @@ -249,12 +264,14 @@ impl NamedTypeLinks for NamedTypesDecorator { NamedTypesDecorator::exists(self, id) } - fn update(&mut self, id: u32, source: u32, target: u32) -> Result { - NamedTypesDecorator::update(self, id, source, target) - } - - fn delete(&mut self, id: u32) -> Result { - NamedTypesDecorator::delete(self, id) + fn update_observed( + &mut self, + id: u32, + source: u32, + target: u32, + observer: ChangeObserver<'_>, + ) -> Result { + NamedTypesDecorator::update_observed(self, id, source, target, observer) } fn delete_observed(&mut self, id: u32, observer: ChangeObserver<'_>) -> Result { @@ -313,15 +330,15 @@ impl NamedTypeLinks for crate::transactions::TransactionsDecorator { crate::transactions::TransactionsDecorator::exists(self, id) } - fn update(&mut self, id: u32, source: u32, target: u32) -> Result { - Ok(crate::transactions::TransactionsDecorator::update( - self, id, source, target, - )?) - } - - fn delete(&mut self, id: u32) -> Result { - Ok(crate::transactions::TransactionsDecorator::delete( - self, id, + fn update_observed( + &mut self, + id: u32, + source: u32, + target: u32, + observer: ChangeObserver<'_>, + ) -> Result { + Ok(crate::transactions::TransactionsDecorator::update_observed( + self, id, source, target, observer, )?) } @@ -383,12 +400,16 @@ impl NamedTypeLinks for crate::version_control::VersionControlDecorator { crate::version_control::VersionControlDecorator::exists(self, id) } - fn update(&mut self, id: u32, source: u32, target: u32) -> Result { - crate::version_control::VersionControlDecorator::update(self, id, source, target) - } - - fn delete(&mut self, id: u32) -> Result { - crate::version_control::VersionControlDecorator::delete(self, id) + fn update_observed( + &mut self, + id: u32, + source: u32, + target: u32, + observer: ChangeObserver<'_>, + ) -> Result { + crate::version_control::VersionControlDecorator::update_observed( + self, id, source, target, observer, + ) } fn delete_observed(&mut self, id: u32, observer: ChangeObserver<'_>) -> Result { diff --git a/rust/src/persistent_transformations.rs b/rust/src/persistent_transformations.rs index 24ffff7..2a258d9 100644 --- a/rust/src/persistent_transformations.rs +++ b/rust/src/persistent_transformations.rs @@ -599,15 +599,14 @@ impl NamedTypeLinks for PersistentTransformationDecorator self.links.exists(id) } - fn update(&mut self, id: u32, source: u32, target: u32) -> Result { - let link = self.links.update(id, source, target)?; - self.apply_triggers_after_operation()?; - self.take_pending_error()?; - Ok(link) - } - - fn delete(&mut self, id: u32) -> Result { - let link = self.links.delete(id)?; + fn update_observed( + &mut self, + id: u32, + source: u32, + target: u32, + observer: ChangeObserver<'_>, + ) -> Result { + let link = self.links.update_observed(id, source, target, observer)?; self.apply_triggers_after_operation()?; self.take_pending_error()?; Ok(link) diff --git a/rust/src/protocol/links_operations.rs b/rust/src/protocol/links_operations.rs index 0c2c825..117adae 100644 --- a/rust/src/protocol/links_operations.rs +++ b/rust/src/protocol/links_operations.rs @@ -12,7 +12,7 @@ //! | `(count: (index source target))` | `(count: N)` | //! | `(each: (index source target))` | one `(index: source target)` per match | //! | `(create: (source target))` | `() ((index: source target))` | -//! | `(update: (index source target))`| `((index: s t)) ((index: source target))` | +//! | `(update: (index source target))`| `((index: s t)) ((index: source target))` per change | //! | `(delete: index)` | `((index: source target)) ()` per removed link | //! | `(get-name: link)` | `(name: 'text')`, or nothing | //! | `(set-name: (link 'text'))` | `(link: N)` | @@ -46,7 +46,8 @@ pub enum LinksOperation { Each(Vec), /// Creates a link. Create { source: u32, target: u32 }, - /// Points an existing link at a new source and target. + /// Points an existing link at a new source and target; an update that + /// would duplicate an existing link merges into it instead. Update { index: u32, source: u32, @@ -162,12 +163,12 @@ impl LinksOperation { source, target, } => { - let before = storage.update(*index, *source, *target)?; - let after = storage - .get_link(*index) - .unwrap_or_else(|| Link::new(*index, *source, *target)); + let mut changes = Vec::new(); + storage.update_observed(*index, *source, *target, &mut |before, after| { + changes.push((before, after)) + })?; storage.save()?; - changes_document(&[(before, after)]) + changes_document(&changes) } Self::Delete(index) => { let mut changes = Vec::new(); diff --git a/rust/src/protocol/remote_links.rs b/rust/src/protocol/remote_links.rs index 7d25fc0..2ff93b3 100644 --- a/rust/src/protocol/remote_links.rs +++ b/rust/src/protocol/remote_links.rs @@ -99,7 +99,18 @@ impl RemoteLinks { .ok_or_else(|| ProtocolError::malformed("the create reply holds no created link")) } - fn removed_link(changes: &[Change], index: u32) -> ProtocolResult { + /// Runs an `update` or `delete` of `index`, reports its changes to + /// `observer` and returns the state `index` was in before. + fn observed( + &self, + operation: &LinksOperation, + index: u32, + observer: ChangeObserver<'_>, + ) -> ProtocolResult { + let changes = self.changes(operation)?; + for (before, after) in &changes { + observer(*before, *after); + } changes .iter() .find(|(before, _)| before.index == index) @@ -246,25 +257,23 @@ impl NamedTypeLinks for RemoteLinks { connected(self.link(id)).is_some() } - fn update(&mut self, id: u32, source: u32, target: u32) -> anyhow::Result { - let changes = self.changes(&LinksOperation::Update { + fn update_observed( + &mut self, + id: u32, + source: u32, + target: u32, + observer: ChangeObserver<'_>, + ) -> anyhow::Result { + let operation = LinksOperation::Update { index: id, source, target, - })?; - Ok(Self::removed_link(&changes, id)?) - } - - fn delete(&mut self, id: u32) -> anyhow::Result { - self.delete_observed(id, &mut |_, _| {}) + }; + Ok(self.observed(&operation, id, observer)?) } fn delete_observed(&mut self, id: u32, observer: ChangeObserver<'_>) -> anyhow::Result { - let changes = self.changes(&LinksOperation::Delete(id))?; - for (before, after) in &changes { - observer(*before, *after); - } - Ok(Self::removed_link(&changes, id)?) + Ok(self.observed(&LinksOperation::Delete(id), id, observer)?) } fn all_links(&mut self) -> Vec { diff --git a/rust/src/query_processor.rs b/rust/src/query_processor.rs index 98a7b69..3fe52ff 100644 --- a/rust/src/query_processor.rs +++ b/rust/src/query_processor.rs @@ -622,26 +622,28 @@ impl QueryProcessor { } } (None, Some(after)) => { - let (before, created) = self.create_or_update_resolved_link(storage, &after)?; - changes.push((before, Some(created))); + self.create_or_update_resolved_link(storage, &after, changes)?; } (Some(before), Some(after)) => { if before.index == after.index && storage.exists(before.index) { - let before_link = storage.get_link(before.index).unwrap(); - if before_link.source != after.source || before_link.target != after.target { - storage.update(before.index, after.source, after.target)?; + let stored = storage.get_link(before.index).unwrap(); + if stored.source != after.source || stored.target != after.target { + // A merge into a duplicate deletes `before.index`; + // `restore_unexpected_deletions` puts it back when the + // query asked for it, exactly as the C# processor does. + self.update_observed( + storage, + before.index, + after.source, + after.target, + changes, + )?; + } else { + changes.push((Some(stored), Some(stored))); } if let Some(name) = &after.name { storage.set_name(before.index, name)?; } - // The update can be resolved into a merge, which deletes - // `before.index`; report the state the query asked for and - // let `restore_unexpected_deletions` put the link back, - // exactly as the C# processor does. - let after_link = storage - .get_link(before.index) - .unwrap_or_else(|| Link::new(before.index, after.source, after.target)); - changes.push((Some(before_link), Some(after_link))); } else { self.apply_operation(storage, Some(before), None, changes)?; self.apply_operation(storage, None, Some(after), changes)?; diff --git a/rust/src/query_processor/mutations.rs b/rust/src/query_processor/mutations.rs index 01884d2..9bc749b 100644 --- a/rust/src/query_processor/mutations.rs +++ b/rust/src/query_processor/mutations.rs @@ -17,6 +17,14 @@ use crate::query_types::ResolvedLink; use super::QueryProcessor; +/// A store change as `--changes` reports it: a null link is no link at all. +fn change(before: Link, after: Link) -> (Option, Option) { + ( + (!before.is_null()).then_some(before), + (!after.is_null()).then_some(after), + ) +} + impl QueryProcessor { /// Deletes `id` and appends every resulting change to `changes`. /// @@ -35,15 +43,25 @@ impl QueryProcessor { id: u32, changes: &mut Vec<(Option, Option)>, ) -> Result { - let mut observed = Vec::new(); - let deleted = storage.delete_observed(id, &mut |before, after| { - observed.push(( - (!before.is_null()).then_some(before), - (!after.is_null()).then_some(after), - )); - })?; - changes.append(&mut observed); - Ok(deleted) + storage.delete_observed(id, &mut |before, after| changes.push(change(before, after))) + } + + /// Updates `id` and appends every resulting change to `changes`. + /// + /// An update that duplicates an existing doublet is merged into it, which + /// deletes `id`; that deletion is reported too, as C# reports it through + /// the handler it passes to `links.Update`. + pub(super) fn update_observed( + &self, + storage: &mut impl NamedTypeLinks, + id: u32, + source: u32, + target: u32, + changes: &mut Vec<(Option, Option)>, + ) -> Result { + storage.update_observed(id, source, target, &mut |before, after| { + changes.push(change(before, after)) + }) } /// Final state every planned operation asks for, keyed by link address @@ -108,22 +126,20 @@ impl QueryProcessor { self.trace_msg(&format!( "[RestoreUnexpectedLinkDeletions] Recreating link {index} => was unexpected deletion." )); - let (before, restored) = self.create_or_update_resolved_link(storage, intended)?; - changes.push((before, Some(restored))); + self.create_or_update_resolved_link(storage, intended, changes)?; } Ok(()) } - /// Creates or updates the link a resolved definition asks for and reports - /// the states a `--changes` listener would see. + /// Creates or updates the link a resolved definition asks for and appends + /// the changes a `--changes` listener would see. /// - /// The returned pair is `(before, after)`, where `before` is `None` only - /// when the link genuinely had to be allocated from nothing. Everything - /// else — an address that had to be filled in with - /// [`try_ensure_created`](NamedTypeLinks::try_ensure_created), a definition - /// that already matches its stored state, a duplicate of an existing - /// doublet — reports the state that was there before, mirroring - /// `CreateOrUpdateLink` in the C# processor: + /// Only a link that genuinely had to be allocated from nothing is reported + /// as a creation. An address filled in with + /// [`try_ensure_created`](NamedTypeLinks::try_ensure_created) reports the + /// update from its empty state, and a definition that already matches its + /// stored state — or a duplicate of an existing doublet — reports that + /// state unchanged, mirroring `CreateOrUpdateLink` in the C# processor: /// /// ```csharp /// if (existingDoublet.Source != linkDefinition.Source || existingDoublet.Target != linkDefinition.Target) @@ -141,8 +157,9 @@ impl QueryProcessor { &self, storage: &mut impl NamedTypeLinks, definition: &ResolvedLink, - ) -> Result<(Option, Link)> { - let (before, id) = if Self::is_normal_index(definition.index) { + changes: &mut Vec<(Option, Option)>, + ) -> Result { + let id = if Self::is_normal_index(definition.index) { storage.try_ensure_created(definition.index)?; let existing = storage .get_link(definition.index) @@ -154,14 +171,15 @@ impl QueryProcessor { "[CreateOrUpdateLink] Updating link {}: {}->{source}, {}->{target}.", definition.index, existing.source, existing.target )); - storage.update(definition.index, source, target)?; + self.update_observed(storage, definition.index, source, target, changes)?; } else { self.trace_msg(&format!( "[CreateOrUpdateLink] Link {} is already S={source}, T={target} => no change.", definition.index )); + changes.push((Some(existing), Some(existing))); } - (Some(existing), definition.index) + definition.index } else if let Some(existing_id) = Self::search_unspecified(storage, definition.source, definition.target) { @@ -171,24 +189,23 @@ impl QueryProcessor { let existing = storage .get_link(existing_id) .unwrap_or_else(|| Link::new(existing_id, definition.source, definition.target)); - (Some(existing), existing_id) + changes.push((Some(existing), Some(existing))); + existing_id } else { let source = Self::resolve_unspecified(definition.source, 0); let target = Self::resolve_unspecified(definition.target, 0); self.trace_msg(&format!( "[CreateOrUpdateLink] Creating new link => (S={source},T={target})." )); - (None, storage.create(source, target)) + let created = storage.create(source, target); + changes.push((None, storage.get_link(created))); + created }; if let Some(name) = &definition.name { storage.set_name(id, name)?; } - - let after = storage - .get_link(id) - .unwrap_or_else(|| Link::new(id, definition.source, definition.target)); - Ok((before, after)) + Ok(id) } /// Ensures a link is created from a LiNo pattern, recursing into its parts. @@ -300,11 +317,7 @@ impl QueryProcessor { "[EnsureLinkCreated] Updating link {index} => {}->{source}, {}->{target}.", stored.source, stored.target )); - storage.update(index, source, target)?; - let after = storage - .get_link(index) - .unwrap_or_else(|| Link::new(index, source, target)); - changes.push((Some(stored), Some(after))); + self.update_observed(storage, index, source, target, changes)?; } else { self.trace_msg(&format!( "[EnsureLinkCreated] Link {index} is already correct => no-op." diff --git a/rust/src/transactions/mod.rs b/rust/src/transactions/mod.rs index 07b658d..c6acae6 100644 --- a/rust/src/transactions/mod.rs +++ b/rust/src/transactions/mod.rs @@ -322,8 +322,25 @@ where } pub fn update(&mut self, id: T, source: T, target: T) -> Result, LinkError> { + self.update_observed(id, source, target, &mut |_, _| {}) + } + + /// [`Self::update`], reporting every change the underlying store made. + /// + /// An update that would duplicate an existing doublet is resolved into a + /// merge that deletes the updated link, so one call can report several + /// changes, exactly like C#'s `links.Update(restriction, substitution, handler)`. + pub fn update_observed( + &mut self, + id: T, + source: T, + target: T, + observer: &mut dyn FnMut(GenericLink, GenericLink), + ) -> Result, LinkError> { if self.replaying { - return self.inner.update_link(id, source, target); + return self + .inner + .update_link_observed(id, source, target, observer); } let before = self.snapshot(id); let owns = self.ensure_open_transaction(); @@ -331,6 +348,7 @@ where let outcome = self .inner .update_link_observed(id, source, target, &mut |before, after| { + observer(before, after); record_observed(&mut observed, before, after) }); let prev = match outcome { diff --git a/rust/src/version_control/mod.rs b/rust/src/version_control/mod.rs index 26fd9df..ffdae79 100644 --- a/rust/src/version_control/mod.rs +++ b/rust/src/version_control/mod.rs @@ -202,8 +202,22 @@ impl VersionControlDecorator { } pub fn update(&mut self, id: u32, source: u32, target: u32) -> Result { + self.update_observed(id, source, target, &mut |_, _| {}) + } + + /// [`Self::update`], reporting every change the decorator stack made — + /// the deletion of a link merged into its duplicate included. + pub fn update_observed( + &mut self, + id: u32, + source: u32, + target: u32, + observer: ChangeObserver<'_>, + ) -> Result { let before_seq = self.transactions.last_logged_sequence(); - let result = self.transactions.update(id, source, target)?; + let result = self + .transactions + .update_observed(id, source, target, observer)?; if self.active_transaction.is_none() { let branch = self.current_branch.clone(); self.attribute_new_transitions_for_branch(before_seq, &branch)?; diff --git a/rust/tests/common/mod.rs b/rust/tests/common/mod.rs index 14ac958..9989f30 100644 --- a/rust/tests/common/mod.rs +++ b/rust/tests/common/mod.rs @@ -47,10 +47,9 @@ impl RunningServer { pub fn client(&self, protocol: impl LinoProtocol + 'static) -> LinksClient { LinksClient::connect(self.address, protocol).unwrap() } -} -impl Drop for RunningServer { - fn drop(&mut self) { + /// Stops the server and waits for it to finish. + pub fn stop(&mut self) { self.shutdown.shutdown(); if let Some(thread) = self.thread.take() { thread.join().unwrap(); @@ -58,6 +57,12 @@ impl Drop for RunningServer { } } +impl Drop for RunningServer { + fn drop(&mut self) { + self.stop(); + } +} + pub fn protocols() -> Vec> { let mut protocols: Vec> = vec![Box::new(TextLinoProtocol::new())]; for external_references in [false, true] { diff --git a/rust/tests/query_processor_csharp_parity_tests.rs b/rust/tests/query_processor_csharp_parity_tests.rs index cf47832..3e35a2e 100644 --- a/rust/tests/query_processor_csharp_parity_tests.rs +++ b/rust/tests/query_processor_csharp_parity_tests.rs @@ -346,6 +346,22 @@ fn test_update_into_existing_pair_merges_matches_csharp() -> Result<()> { }) } +/// The merge is reported the way C# reports it: the updated link is gone and +/// the link it merged into is not reported at all. +#[test] +fn test_update_into_existing_pair_reports_merge_matches_csharp() -> Result<()> { + with_storage(|storage, processor| { + processor.process_query(storage, "() ((1 1))")?; + processor.process_query(storage, "() ((2 2))")?; + + let changes = processor.process_query(storage, "((2: 2 2)) ((2: 1 1))")?; + + assert_eq!(changes, vec![(Some(Link::new(2, 2, 2)), None)]); + assert_eq!(sorted_links(storage), vec![Link::new(1, 1, 1)]); + Ok(()) + }) +} + /// Deleting a named link cascades through the links that use it, and the names /// of the survivors are untouched. #[test] diff --git a/rust/tests/remote_links_tests.rs b/rust/tests/remote_links_tests.rs new file mode 100644 index 0000000..cfd24a4 --- /dev/null +++ b/rust/tests/remote_links_tests.rs @@ -0,0 +1,326 @@ +//! [`RemoteLinks`] is a drop-in replacement for a local store: every protocol +//! gives the same results as the store it talks to. + +mod common; + +use common::{protocols, RunningServer}; +use doublets::data::Flow; +use doublets::{Doublets, Error, Link as DoubletsLink, Links}; +use link_cli::protocol::{ + links_operations::matches, parse_document, LinksOperation, LinoProtocol, RemoteLinks, + ServerOptions, TextLinoProtocol, +}; +use link_cli::{Link, NamedTypeLinks, NamedTypesDecorator, QueryProcessor}; +use std::panic::{self, AssertUnwindSafe}; +use tempfile::NamedTempFile; + +/// Changes as the query processor reports them. +type Changes = Vec<(Option, Option)>; + +/// The README walkthrough plus the cases that cascade: a merge into an +/// existing doublet and the deletion of a link that others use. +const QUERIES: &[&str] = &[ + "() ((1 1))", + "() ((2 2))", + "((1: 1 1)) ((1: 1 2))", + "((1: 1 2)) ((1: 2 1))", + "((($i: $s $t)) (($i: $t $s)))", + "((2: 2 2)) ((2: 1 1))", + "() ((father: father father))", + "() ((mother: mother mother))", + "() ((child: father mother))", + "((child: father mother)) ((child: father mother))", + "((child: father mother)) ((child: mother father))", + "((father: father father)) ()", + "((1 *)) ()", +]; + +fn local_store() -> (NamedTypesDecorator, NamedTempFile, NamedTempFile) { + let database = NamedTempFile::new().unwrap(); + let names = NamedTempFile::new().unwrap(); + let store = NamedTypesDecorator::with_names_database_path(database.path(), names.path(), false) + .unwrap(); + (store, database, names) +} + +fn connect(server: &RunningServer, protocol: impl LinoProtocol + 'static) -> RemoteLinks { + RemoteLinks::new(server.client(protocol)) +} + +fn run_queries(store: &mut impl NamedTypeLinks) -> Vec { + let processor = QueryProcessor::new(false).with_auto_create_missing_references(true); + QUERIES + .iter() + .map(|query| processor.process_query(store, query).unwrap()) + .collect() +} + +#[test] +fn the_query_processor_gives_the_same_results_on_a_remote_store() { + let (mut local, _database, _names) = local_store(); + let expected_changes = run_queries(&mut local); + let expected_lines = local.lino_lines().unwrap(); + + for protocol in protocols() { + let server = RunningServer::start(ServerOptions::default(), true); + let mut remote = connect(&server, protocol); + + assert_eq!(run_queries(&mut remote), expected_changes); + assert_eq!(remote.lino_lines().unwrap(), expected_lines); + } +} + +#[test] +fn every_named_types_links_call_matches_a_local_store() { + for protocol in protocols() { + let (mut local, _database, _names) = local_store(); + let server = RunningServer::start(ServerOptions::default(), true); + let mut remote = connect(&server, protocol); + + for store in [&mut local as &mut dyn NamedTypeLinksCalls, &mut remote] { + store.exercise(); + } + assert_eq!(remote.all_links(), local.all_links()); + assert_eq!(remote.lino_lines().unwrap(), local.lino_lines().unwrap()); + } +} + +/// Runs every [`NamedTypeLinks`] call once and checks what a local store +/// returns, so the remote store is held to exactly the same answers. +trait NamedTypeLinksCalls { + fn exercise(&mut self); +} + +impl NamedTypeLinksCalls for S { + fn exercise(&mut self) { + assert_eq!(self.create(0, 0), 1); + assert_eq!(self.get_or_create(1, 1), 2); + assert_eq!(self.get_or_create(1, 1), 2); + assert_eq!(self.search(1, 1), Some(2)); + assert_eq!(self.search(2, 2), None); + assert_eq!(self.get_link(2), Some(Link::new(2, 1, 1))); + assert_eq!(self.get_link(9), None); + assert!(self.exists(1)); + assert!(!self.exists(9)); + + assert_eq!(self.ensure_created(5), 5); + assert_eq!(self.ensure_created(5), 5); + assert!(!self.exists(3) && !self.exists(4)); + assert_eq!(self.try_ensure_created(0).ok(), None); + + assert_eq!(self.update(5, 2, 2).unwrap(), Link::new(5, 0, 0)); + assert!(self.update(9, 1, 1).is_err()); + + let mut changes = Vec::new(); + let merged = self + .update_observed(5, 1, 1, &mut |before, after| changes.push((before, after))) + .unwrap(); + assert_eq!(merged, Link::new(5, 2, 2)); + assert_eq!( + changes, + vec![ + (Link::new(5, 2, 2), Link::new(5, 0, 0)), + (Link::new(5, 0, 0), Link::null()) + ] + ); + assert!(!self.exists(5)); + + self.set_name(2, "pair").unwrap(); + assert_eq!(self.get_name(2).unwrap().as_deref(), Some("pair")); + assert_eq!(self.get_by_name("pair").unwrap(), Some(2)); + assert_eq!(self.get_by_name("none").unwrap(), None); + assert_eq!( + self.format_lino(&Link::new(2, 1, 1)).unwrap(), + "(pair: 1 1)" + ); + self.remove_name(2).unwrap(); + assert_eq!(self.get_name(2).unwrap(), None); + + let named = self.get_or_create_named("leaf").unwrap(); + assert_eq!(self.get_link(named), Some(Link::new(named, named, named))); + + let mut deleted = Vec::new(); + let removed = self + .delete_observed(1, &mut |before, after| deleted.push((before, after))) + .unwrap(); + assert_eq!(removed, Link::new(1, 0, 0)); + assert!(deleted.contains(&(Link::new(1, 0, 0), Link::null()))); + assert!(deleted.contains(&(Link::new(2, 1, 1), Link::null()))); + assert!(self.delete(1).is_err()); + self.save().unwrap(); + } +} + +#[test] +fn the_raw_links_interface_works_over_every_protocol() { + for protocol in protocols() { + let server = RunningServer::start(ServerOptions::default(), false); + let mut remote = connect(&server, protocol); + let any = remote.constants().any; + + let mut created = Vec::new(); + let flow = Links::create_links(&mut remote, &[], &mut |before, after| { + created.push((before, after)); + Flow::Continue + }) + .unwrap(); + assert_eq!(flow, Flow::Continue); + assert_eq!( + created, + vec![(DoubletsLink::new(0, 0, 0), DoubletsLink::new(1, 0, 0))] + ); + Links::create_links(&mut remote, &[], &mut |_, _| Flow::Continue).unwrap(); + + Links::update_links(&mut remote, &[1], &[1, 1, 2], &mut |_, _| Flow::Continue).unwrap(); + Links::update_links(&mut remote, &[2], &[2, 2, 1], &mut |_, _| Flow::Continue).unwrap(); + assert_eq!( + Doublets::get_link(&remote, 1), + Some(DoubletsLink::new(1, 1, 2)) + ); + assert_eq!(Doublets::get_link(&remote, 3), None); + + assert_eq!(remote.count_links(&[]), 2); + assert_eq!(remote.count_links(&[any]), 2); + assert_eq!(remote.count_links(&[2]), 1); + assert_eq!(remote.count_links(&[any, 1]), 2); + assert_eq!(remote.count_links(&[any, 1, any]), 1); + assert_eq!(remote.count_links(&[any, any, 1]), 1); + assert_eq!(remote.count_links(&[any, 2, 2]), 0); + + let mut seen = Vec::new(); + let flow = remote.each_links(&[any, any, any], &mut |link| { + seen.push(link); + Flow::Break + }); + assert_eq!( + (flow, seen), + (Flow::Break, vec![DoubletsLink::new(1, 1, 2)]) + ); + + let mut deleted = Vec::new(); + let flow = Links::delete_links(&mut remote, &[1], &mut |before, after| { + deleted.push((before, after)); + Flow::Break + }) + .unwrap(); + assert_eq!(flow, Flow::Break); + assert_eq!(deleted.len(), 1); + assert_eq!(remote.count_links(&[]), 0); + } +} + +#[test] +fn server_failures_are_errors_not_panics() { + let server = RunningServer::start(ServerOptions::default(), false); + let mut remote = connect(&server, TextLinoProtocol::new()); + + let error = + Links::update_links(&mut remote, &[7], &[7, 1, 1], &mut |_, _| Flow::Continue).unwrap_err(); + assert!(matches!(error, Error::Other(_)), "{error:?}"); + assert!(Links::delete_links(&mut remote, &[7], &mut |_, _| Flow::Continue).is_err()); + assert!(NamedTypeLinks::update(&mut remote, 7, 1, 1).is_err()); + assert!(remote + .count_matching(&[Some(1), Some(2), Some(3), Some(4)]) + .is_err()); +} + +#[test] +fn a_lost_connection_panics_where_the_interface_cannot_fail() { + let mut server = RunningServer::start(ServerOptions::default(), false); + let remote = connect(&server, TextLinoProtocol::new()); + server.stop(); + + assert!(remote.count_matching(&[]).is_err()); + let panic = panic::catch_unwind(AssertUnwindSafe(|| remote.count_links(&[]))) + .expect_err("a lost connection cannot be counted"); + let message = panic.downcast_ref::().unwrap(); + assert!( + message.starts_with("remote links store failed"), + "{message}" + ); +} + +#[test] +fn operations_round_trip_through_their_documents() { + let operations = [ + LinksOperation::Count(vec![]), + LinksOperation::Count(vec![Some(1)]), + LinksOperation::Each(vec![None, Some(2)]), + LinksOperation::Each(vec![Some(1), None, Some(3)]), + LinksOperation::Create { + source: 1, + target: 2, + }, + LinksOperation::Update { + index: 3, + source: 4, + target: 5, + }, + LinksOperation::Delete(6), + LinksOperation::GetName(7), + LinksOperation::SetName(8, "a name".to_string()), + LinksOperation::GetByName("a name".to_string()), + LinksOperation::RemoveName(9), + ]; + for protocol in protocols() { + for operation in &operations { + let mut bytes = Vec::new(); + protocol + .write_document(&mut bytes, &operation.to_document()) + .unwrap(); + let document = protocol + .read_document(&mut bytes.as_slice()) + .unwrap() + .unwrap(); + assert_eq!( + LinksOperation::from_document(&document).unwrap().as_ref(), + Some(operation) + ); + } + } +} + +#[test] +fn only_operation_shaped_documents_are_operations() { + let parse = |text: &str| LinksOperation::from_document(&parse_document(text).unwrap()); + + for query in [ + "() ((1 1))", + "((1: 1 1)) ()", + "(unknown: 1)", + "(count: 1 2)", + ] { + assert_eq!(parse(query).unwrap(), None, "{query}"); + } + for malformed in [ + "(count: (1 2 3 4))", + "(each: (x))", + "(create: (1))", + "(update: (1 2))", + "(delete: x)", + "(set-name: (1))", + "(get-by-name: (a b))", + ] { + assert!(parse(malformed).is_err(), "{malformed}"); + } +} + +#[test] +fn restrictions_match_like_the_raw_links_interface() { + let link = Link::new(1, 2, 3); + let shapes: &[(&[Option], bool)] = &[ + (&[], true), + (&[None], true), + (&[Some(1)], true), + (&[Some(2)], false), + (&[None, Some(2)], true), + (&[None, Some(3)], true), + (&[None, Some(1)], false), + (&[Some(1), Some(2), Some(3)], true), + (&[None, Some(3), None], false), + (&[None, None, None, None], false), + ]; + for (restriction, expected) in shapes { + assert_eq!(matches(&link, restriction), *expected, "{restriction:?}"); + } +} From df54e92ea908a67fd199a93ee164e35edc707264 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 4 Oct 2026 23:56:49 +0000 Subject: [PATCH 17/44] fix: drop the unchanged report of a link whose change is reported A create followed by a match of the same link reported both '(1: 1 1) -> (1: 1 1)' and '() -> (1: 1 1)'. Both simplifiers now keep an unchanged pair only when no other change touches that link. --- .../ChangesSimplifier.cs | 8 +++++ .../ChangesSimplifier.cs | 31 +++++++++++++++++++ rust/src/changes_simplifier.rs | 22 +++++++------ rust/tests/changes_simplifier_tests.rs | 29 +++++++++++++++++ 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/ChangesSimplifier.cs b/csharp/Foundation.Data.Doublets.Cli.Library/ChangesSimplifier.cs index 9a26127..6dba1c3 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/ChangesSimplifier.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/ChangesSimplifier.cs @@ -47,6 +47,14 @@ public static class ChangesSimplifier } } + // A link that really changed is described by its change alone: reading + // it unchanged on the way (a create followed by a match) is not news. + var changedIndices = changedStates + .SelectMany(c => new[] { c.Before.Index, c.After.Index }) + .Where(index => index != 0) + .ToHashSet(); + unchangedStates.RemoveAll(c => changedIndices.Contains(c.Before.Index)); + // Gather all 'Before' links and all 'After' links from changed states var beforeLinks = new HashSet>(changedStates.Select(c => c.Before), LinkEqualityComparer.Instance); var afterLinks = new HashSet>(changedStates.Select(c => c.After), LinkEqualityComparer.Instance); diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/ChangesSimplifier.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/ChangesSimplifier.cs index 9dec3a6..9ac16d5 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/ChangesSimplifier.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/ChangesSimplifier.cs @@ -398,6 +398,37 @@ public void SimplifyChanges_Issue26_AlternativeScenario_NoSimplificationOccurs() // If the simplifier doesn't work, we'd see all 3 changes } + [Fact] + public void SimplifyChanges_CreateThenMatch_ReportsOnlyTheCreate() + { + var nothing = new Link(index: 0, source: 0, target: 0); + var created = new Link(index: 1, source: 1, target: 1); + var changes = new List<(Link Before, Link After)> { (nothing, created), (created, created) }; + + AssertChangeSetEqual(new() { (nothing, created) }, SimplifyChanges(changes).ToList()); + } + + [Fact] + public void SimplifyChanges_MatchThenUpdate_ReportsOnlyTheUpdate() + { + var before = new Link(index: 1, source: 1, target: 1); + var after = new Link(index: 1, source: 2, target: 2); + var changes = new List<(Link Before, Link After)> { (before, before), (before, after) }; + + AssertChangeSetEqual(new() { (before, after) }, SimplifyChanges(changes).ToList()); + } + + [Fact] + public void SimplifyChanges_KeepsUnchangedLinksThatDidNotChangeElsewhere() + { + var nothing = new Link(index: 0, source: 0, target: 0); + var unchanged = new Link(index: 2, source: 2, target: 2); + var created = new Link(index: 1, source: 1, target: 1); + var changes = new List<(Link Before, Link After)> { (unchanged, unchanged), (nothing, created) }; + + AssertChangeSetEqual(new() { (nothing, created), (unchanged, unchanged) }, SimplifyChanges(changes).ToList()); + } + private static void AssertChangeSetEqual( List<(Link Before, Link After)> expectedSimplifiedChanges, List<(Link Before, Link After)> simplifiedChanges diff --git a/rust/src/changes_simplifier.rs b/rust/src/changes_simplifier.rs index 10c7e24..2dd9637 100644 --- a/rust/src/changes_simplifier.rs +++ b/rust/src/changes_simplifier.rs @@ -21,16 +21,20 @@ pub fn simplify_changes(changes: Vec<(Link, Link)>) -> Vec<(Link, Link)> { let changes = remove_duplicate_before_states(changes); // First, handle unchanged states directly - let mut unchanged_states = Vec::new(); - let mut changed_states = Vec::new(); + let (unchanged_states, changed_states): (Vec<_>, Vec<_>) = + changes.iter().partition(|(before, after)| before == after); - for (before, after) in changes.iter() { - if before == after { - unchanged_states.push((*before, *after)); - } else { - changed_states.push((*before, *after)); - } - } + // A link that really changed is described by its change alone: reading + // it unchanged on the way (a create followed by a match) is not news. + let changed_indices: HashSet = changed_states + .iter() + .flat_map(|(before, after)| [before.index, after.index]) + .filter(|&index| index != 0) + .collect(); + let unchanged_states: Vec<(Link, Link)> = unchanged_states + .into_iter() + .filter(|(link, _)| !changed_indices.contains(&link.index)) + .collect(); // Gather all 'Before' links and all 'After' links from changed states. // diff --git a/rust/tests/changes_simplifier_tests.rs b/rust/tests/changes_simplifier_tests.rs index 86de9de..1ae8b2b 100644 --- a/rust/tests/changes_simplifier_tests.rs +++ b/rust/tests/changes_simplifier_tests.rs @@ -148,3 +148,32 @@ fn test_simplify_keeps_unchanged_states() { assert!(result.contains(&(Link::new(1, 1, 2), Link::new(1, 2, 1)))); assert!(result.contains(&(Link::new(2, 2, 2), Link::new(2, 2, 2)))); } + +#[test] +fn test_simplify_create_then_match_reports_only_the_create() { + let created = Link::new(1, 1, 1); + let changes = vec![(Link::null(), created), (created, created)]; + + assert_eq!(simplify_changes(changes), vec![(Link::null(), created)]); +} + +#[test] +fn test_simplify_match_then_update_reports_only_the_update() { + let before = Link::new(1, 1, 1); + let after = Link::new(1, 2, 2); + let changes = vec![(before, before), (before, after)]; + + assert_eq!(simplify_changes(changes), vec![(before, after)]); +} + +#[test] +fn test_simplify_keeps_unchanged_links_that_did_not_change_elsewhere() { + let unchanged = Link::new(2, 2, 2); + let created = Link::new(1, 1, 1); + let changes = vec![(unchanged, unchanged), (Link::null(), created)]; + + assert_eq!( + simplify_changes(changes), + vec![(Link::null(), created), (unchanged, unchanged)] + ); +} From 50f36f05aa780e33f2bf48d3c1d621226d3e9fe9 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 00:03:41 +0000 Subject: [PATCH 18/44] fix: simplify changes to the net change of every link address The simplifier walked a graph of link states in which the null link was a single node, so every deletion chained into every creation: deleting link 3 and creating link 4 was reported as '(3: 3 3) -> (4: 4 4)', and C# reported both '((1: 1 1)) ((1: 3 3))' and '() ((1: 3 3))' when '((1: 1 1)) ((3 3))' deleted and recreated address 1. Both ports now reduce the raw steps of each address to (first before, last after), drop links created and deleted within the query, and keep ordering by the after state. The C# tests that were commented out describe exactly this behaviour and are enabled again; the debug-only test now asserts its result. --- .../ChangesSimplifier.cs | 218 ++------ .../ChangesSimplifier.cs | 507 +++++++++--------- rust/src/changes_simplifier.rs | 219 ++------ rust/tests/changes_simplifier_tests.rs | 45 ++ 4 files changed, 379 insertions(+), 610 deletions(-) diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/ChangesSimplifier.cs b/csharp/Foundation.Data.Doublets.Cli.Library/ChangesSimplifier.cs index 6dba1c3..737e906 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/ChangesSimplifier.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/ChangesSimplifier.cs @@ -5,208 +5,52 @@ namespace Foundation.Data.Doublets.Cli public static class ChangesSimplifier { /// - /// Simplifies a list of changes by identifying chains of transformations. - /// If multiple final states are reachable from the same initial state, returns multiple simplified changes. - /// If a scenario arises where no initial or final states can be identified (no-ops), returns the original transitions as-is. + /// Reduces the raw (before, after) steps a store reports to one change per link address: + /// from its state before the query to its state after it. /// - /// List of tuples representing changes (before, after). - /// - /// Simplified list of changes from initial states to final states, - /// or original transitions if no change is detected. - /// + /// + /// The store reports every step it takes — a link is emptied to (i: 0 0) before it is deleted + /// and created as (i: 0 0) before it is set — and the null link (0: 0 0) stands for "no link", + /// before a creation and after a deletion. The steps of one address form a chain, so its net + /// change is the before of its first step and the after of its last: + /// a link created and deleted again within the query is not reported, and a link that ends + /// where it started is reported unchanged, like a link the query only matched. + /// The changes are ordered by their after state, so deletions come first; changes with equal + /// after states keep the order of their first step. + /// public static IEnumerable<(Link Before, Link After)> SimplifyChanges( IEnumerable<(Link Before, Link After)> changes ) { - if (changes == null) throw new ArgumentNullException(nameof(changes)); + ArgumentNullException.ThrowIfNull(changes); - var changesList = changes.ToList(); - if (changesList.Count == 0) + var netChanges = new List<(Link Before, Link After)>(); + var positionOfAddress = new Dictionary(); + foreach (var (before, after) in changes) { - // No changes at all, return empty - return Enumerable.Empty<(Link, Link)>(); - } - - // **FIX for Issue #26**: Remove duplicate before states by keeping the last occurrence - // This handles cases where the same link is reported with multiple different transformations - changesList = RemoveDuplicateBeforeStates(changesList); - - // First, handle unchanged states directly - var unchangedStates = new List<(Link Before, Link After)>(); - var changedStates = new List<(Link Before, Link After)>(); - - foreach (var change in changesList) - { - if (LinkEqualityComparer.Instance.Equals(change.Before, change.After)) - { - unchangedStates.Add(change); - } - else + var address = before.Index != 0 ? before.Index : after.Index; + if (address == 0) { - changedStates.Add(change); + continue; } - } - - // A link that really changed is described by its change alone: reading - // it unchanged on the way (a create followed by a match) is not news. - var changedIndices = changedStates - .SelectMany(c => new[] { c.Before.Index, c.After.Index }) - .Where(index => index != 0) - .ToHashSet(); - unchangedStates.RemoveAll(c => changedIndices.Contains(c.Before.Index)); - - // Gather all 'Before' links and all 'After' links from changed states - var beforeLinks = new HashSet>(changedStates.Select(c => c.Before), LinkEqualityComparer.Instance); - var afterLinks = new HashSet>(changedStates.Select(c => c.After), LinkEqualityComparer.Instance); - - // Identify initial states: appear as Before but never as After - var initialStates = beforeLinks.Where(b => !afterLinks.Contains(b)).ToList(); - - // Identify final states: appear as After but never as Before - var finalStates = afterLinks.Where(a => !beforeLinks.Contains(a)) - .ToHashSet(LinkEqualityComparer.Instance); - - // Build adjacency (Before -> possible list of After links) - var adjacency = new Dictionary, List>>(LinkEqualityComparer.Instance); - foreach (var (before, after) in changedStates) - { - if (!adjacency.TryGetValue(before, out var list)) + if (positionOfAddress.TryGetValue(address, out var position)) { - list = new List>(); - adjacency[before] = list; - } - list.Add(after); - } - - // If we have no identified initial states, treat it as a no-op scenario: - // just return original transitions. - if (initialStates.Count == 0) - { - return changesList; - } - - var results = new List<(Link Before, Link After)>(); - - // Add unchanged states first - results.AddRange(unchangedStates); - - // Traverse each initial state with DFS - foreach (var initial in initialStates.Distinct(LinkEqualityComparer.Instance)) - { - var stack = new Stack>(); - stack.Push(initial); - - var visited = new HashSet>(LinkEqualityComparer.Instance); - - while (stack.Count > 0) - { - var current = stack.Pop(); - // Skip if already visited - if (!visited.Add(current)) - { - continue; - } - - bool hasNext = adjacency.TryGetValue(current, out var nextLinks); - bool isFinalOrDeadEnd = finalStates.Contains(current) || !hasNext || nextLinks!.Count == 0; - - // If final or no further transitions, record (initial -> current) - if (isFinalOrDeadEnd) - { - results.Add((initial, current)); - } - - // Otherwise push neighbors - if (hasNext) - { - foreach (var next in nextLinks!) - { - stack.Push(next); - } - } - } - } - - // ***** IMPORTANT: Sort the final results so that - // items appear in ascending order by their After link. - // This ensures tests that expect a specific order pass reliably. - return results - .OrderBy(r => r.After.Index) - .ThenBy(r => r.After.Source) - .ThenBy(r => r.After.Target); - } - - /// - /// Removes problematic duplicate before states that lead to simplification issues. - /// This fixes Issue #26 where multiple transformations from the same before state - /// to conflicting after states (including null states) would cause the simplifier to fail. - /// - /// The key insight: If we have multiple transitions from the same before state, - /// and one of them is to a "null" state (0: 0 0), we should prefer the non-null transition - /// as it represents the actual final transformation. - /// - /// The list of changes that may contain problematic duplicate before states - /// A list with problematic duplicates resolved - private static List<(Link Before, Link After)> RemoveDuplicateBeforeStates( - List<(Link Before, Link After)> changes) - { - // Group changes by their before state - var groupedChanges = changes.GroupBy(c => c.Before, LinkEqualityComparer.Instance); - - var result = new List<(Link Before, Link After)>(); - - foreach (var group in groupedChanges) - { - var changesForThisBefore = group.ToList(); - - if (changesForThisBefore.Count == 1) - { - // No duplicates, keep as is - result.AddRange(changesForThisBefore); + netChanges[position] = (netChanges[position].Before, after); } else { - // Multiple changes from the same before state - // Check if any of them is to a null state (0: 0 0) - var nullTransition = changesForThisBefore.FirstOrDefault(c => - c.After.Index == 0 && c.After.Source == 0 && c.After.Target == 0); - var nonNullTransitions = changesForThisBefore.Where(c => - !(c.After.Index == 0 && c.After.Source == 0 && c.After.Target == 0)).ToList(); - - if (nullTransition != default && nonNullTransitions.Count > 0) - { - // Issue #26 scenario: We have both null and non-null transitions - // Prefer the non-null transitions as they represent the actual final states - result.AddRange(nonNullTransitions); - } - else - { - // No null transitions involved, this is a legitimate multiple-branch scenario - // Keep all transitions - result.AddRange(changesForThisBefore); - } + positionOfAddress[address] = netChanges.Count; + netChanges.Add((before, after)); } } - - return result; + return netChanges + .Where(change => !(IsNull(change.Before) && IsNull(change.After))) + .OrderBy(change => change.After.Index) + .ThenBy(change => change.After.Source) + .ThenBy(change => change.After.Target) + .ToList(); } - /// - /// An equality comparer for Link{uint} that checks Index/Source/Target. - /// - private class LinkEqualityComparer : IEqualityComparer> - { - public static readonly LinkEqualityComparer Instance = new LinkEqualityComparer(); - - public bool Equals(Link x, Link y) - => x.Index == y.Index && x.Source == y.Source && x.Target == y.Target; - - public int GetHashCode(Link obj) - => HashCode.Combine(obj.Index, obj.Source, obj.Target); - } + private static bool IsNull(Link link) => link.Index == 0 && link.Source == 0 && link.Target == 0; } - - - - -} \ No newline at end of file +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/ChangesSimplifier.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/ChangesSimplifier.cs index 9ac16d5..867a0f5 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/ChangesSimplifier.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/ChangesSimplifier.cs @@ -360,42 +360,23 @@ public void SimplifyChanges_Issue26_UpdateOperationSimplification() } [Fact] - public void SimplifyChanges_Issue26_AlternativeScenario_NoSimplificationOccurs() + public void SimplifyChanges_Issue26_AlternativeScenario_LastAfterOfAnAddressWins() { - // Arrange - This tests a different scenario that might represent the actual issue - // Maybe the problem is that the changes are NOT being chained correctly - // Let's simulate what might happen if the simplifier doesn't work correctly + // Two reports start from the same state of link 1: a deletion and the update that follows it. var changes = new List<(Link Before, Link After)> - { - // Let's say we get these individual changes that don't form a proper chain - (new Link(index: 1, source: 1, target: 2), new Link(index: 0, source: 0, target: 0)), // delete - (new Link(index: 1, source: 1, target: 2), new Link(index: 1, source: 2, target: 1)), // direct update (this might be what's reported) - (new Link(index: 2, source: 2, target: 1), new Link(index: 2, source: 1, target: 2)), // direct update - }; - - // Act - var simplifiedChanges = SimplifyChanges(changes).ToList(); - - // Debug output - Console.WriteLine("=== Debug: Alternative Scenario ==="); - Console.WriteLine("Input changes:"); - for (int i = 0; i < changes.Count; i++) { - var (b, a) = changes[i]; - Console.WriteLine($" {i + 1}. ({b.Index}: {b.Source} {b.Target}) -> ({a.Index}: {a.Source} {a.Target})"); - } + (new Link(index: 1, source: 1, target: 2), new Link(index: 0, source: 0, target: 0)), + (new Link(index: 1, source: 1, target: 2), new Link(index: 1, source: 2, target: 1)), + (new Link(index: 2, source: 2, target: 1), new Link(index: 2, source: 1, target: 2)), + }; - Console.WriteLine("Actual simplified changes:"); - for (int i = 0; i < simplifiedChanges.Count; i++) + var expectedSimplifiedChanges = new List<(Link Before, Link After)> { - var (b, a) = simplifiedChanges[i]; - Console.WriteLine($" {i + 1}. ({b.Index}: {b.Source} {b.Target}) -> ({a.Index}: {a.Source} {a.Target})"); - } - Console.WriteLine($"Count: {simplifiedChanges.Count}"); - Console.WriteLine("=== End Debug ==="); + (new Link(index: 1, source: 1, target: 2), new Link(index: 1, source: 2, target: 1)), + (new Link(index: 2, source: 2, target: 1), new Link(index: 2, source: 1, target: 2)), + }; - // The issue might be that we get 3 changes instead of 2 - // If the simplifier doesn't work, we'd see all 3 changes + AssertChangeSetEqual(expectedSimplifiedChanges, SimplifyChanges(changes).ToList()); } [Fact] @@ -429,6 +410,56 @@ public void SimplifyChanges_KeepsUnchangedLinksThatDidNotChangeElsewhere() AssertChangeSetEqual(new() { (nothing, created), (unchanged, unchanged) }, SimplifyChanges(changes).ToList()); } + [Fact] + public void SimplifyChanges_RecreatingAnAddress_ReportsOneChange() + { + // The raw steps reported for `((1: 1 1)) ((3 3))`. + var nothing = new Link(index: 0, source: 0, target: 0); + var emptied = new Link(index: 1, source: 0, target: 0); + var changes = new List<(Link Before, Link After)> + { + (new Link(index: 1, source: 1, target: 1), emptied), + (emptied, nothing), + (nothing, emptied), + (emptied, new Link(index: 1, source: 3, target: 3)), + }; + + AssertChangeSetEqual( + new() { (new Link(index: 1, source: 1, target: 1), new Link(index: 1, source: 3, target: 3)) }, + SimplifyChanges(changes).ToList()); + } + + [Fact] + public void SimplifyChanges_DoesNotChainADeletionIntoAnotherCreation() + { + var nothing = new Link(index: 0, source: 0, target: 0); + var deleted = new Link(index: 3, source: 3, target: 3); + var created = new Link(index: 4, source: 4, target: 4); + var changes = new List<(Link Before, Link After)> { (deleted, nothing), (nothing, created) }; + + AssertChangeSetEqual(new() { (deleted, nothing), (nothing, created) }, SimplifyChanges(changes).ToList()); + } + + [Fact] + public void SimplifyChanges_DropsALinkCreatedAndDeletedAgain() + { + var nothing = new Link(index: 0, source: 0, target: 0); + var temporary = new Link(index: 3, source: 0, target: 0); + var changes = new List<(Link Before, Link After)> { (nothing, temporary), (temporary, nothing) }; + + Assert.Empty(SimplifyChanges(changes)); + } + + [Fact] + public void SimplifyChanges_ReportsALinkChangedBackAsUnchanged() + { + var original = new Link(index: 1, source: 1, target: 2); + var swapped = new Link(index: 1, source: 2, target: 1); + var changes = new List<(Link Before, Link After)> { (original, swapped), (swapped, original) }; + + AssertChangeSetEqual(new() { (original, original) }, SimplifyChanges(changes).ToList()); + } + private static void AssertChangeSetEqual( List<(Link Before, Link After)> expectedSimplifiedChanges, List<(Link Before, Link After)> simplifiedChanges @@ -450,214 +481,212 @@ private static void AssertChangeSetEqual( } } - // [Fact] - // public void SimplifyChanges_NoChanges_ReturnsEmpty() - // { - // // Arrange - // var changes = new List<(Link Before, Link After)>(); - - // // Act - // var simplified = ChangesSimplifier.SimplifyChanges(changes); - - // // Assert - // Assert.Empty(simplified); - // } - - // [Fact] - // public void SimplifyChanges_SingleChange_ReturnsSameChange() - // { - // // Arrange - // var changes = new List<(Link Before, Link After)> - // { - // (new Link(1, 2, 3), new Link(1, 4, 5)) - // }; - - // // Act - // var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); - - // // Assert - // Assert.Single(simplified); - // Assert.Equal(1u, simplified[0].Before.Index); - // Assert.Equal(0u, simplified[0].Before.Source); // Assuming default values for other fields - // Assert.Equal(0u, simplified[0].Before.Target); - // Assert.Equal(new Link(1, 4, 5), simplified[0].After); - // } - - // [Fact] - // public void SimplifyChanges_MultipleNonOverlappingChanges_ReturnsAllChanges() - // { - // // Arrange - // var changes = new List<(Link Before, Link After)> - // { - // (new Link(1, 2, 3), new Link(1, 4, 5)), - // (new Link(2, 3, 4), new Link(2, 5, 6)), - // (new Link(3, 4, 5), new Link(3, 6, 7)) - // }; - - // // Act - // var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); - - // // Assert - // Assert.Equal(3, simplified.Count); - - // Assert.Contains(simplified, c => c.Before.Index == 1 && c.After.Index == 1 && c.After.Source == 4 && c.After.Target == 5); - // Assert.Contains(simplified, c => c.Before.Index == 2 && c.After.Index == 2 && c.After.Source == 5 && c.After.Target == 6); - // Assert.Contains(simplified, c => c.Before.Index == 3 && c.After.Index == 3 && c.After.Source == 6 && c.After.Target == 7); - // } - - // [Fact] - // public void SimplifyChanges_MultipleOverlappingChanges_ReturnsInitialToFinal() - // { - // // Arrange - // var changes = new List<(Link Before, Link After)> - // { - // (new Link(1, 2, 3), new Link(1, 4, 5)), - // (new Link(1, 4, 5), new Link(1, 6, 7)), - // (new Link(2, 3, 4), new Link(2, 5, 6)), - // (new Link(2, 5, 6), new Link(2, 0, 0)), - // (new Link(3, 4, 5), new Link(3, 6, 7)) - // }; - - // // Act - // var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); - - // // Assert - // Assert.Equal(3, simplified.Count); - - // // Link 1: from (1,2,3) to (1,6,7) - // var link1 = simplified.FirstOrDefault(c => c.Before.Index == 1); - // Assert.Equal(new Link(1, 2, 3), link1.Before); - // Assert.Equal(new Link(1, 6, 7), link1.After); - - // // Link 2: from (2,3,4) to (2,0,0) - // var link2 = simplified.FirstOrDefault(c => c.Before.Index == 2); - // Assert.Equal(new Link(2, 3, 4), link2.Before); - // Assert.Equal(new Link(2, 0, 0), link2.After); - - // // Link 3: from (3,4,5) to (3,6,7) - // var link3 = simplified.FirstOrDefault(c => c.Before.Index == 3); - // Assert.Equal(new Link(3, 4, 5), link3.Before); - // Assert.Equal(new Link(3, 6, 7), link3.After); - // } - - // [Fact] - // public void SimplifyChanges_ComplexScenario_MixedChanges() - // { - // // Arrange - // var changes = new List<(Link Before, Link After)> - // { - // // Link 1: Multiple changes - // (new Link(1, 2, 3), new Link(1, 4, 5)), - // (new Link(1, 4, 5), new Link(1, 6, 7)), - // (new Link(1, 6, 7), new Link(1, 8, 9)), - - // // Link 2: Single change - // (new Link(2, 3, 4), new Link(2, 5, 6)), - - // // Link 3: Multiple changes - // (new Link(3, 4, 5), new Link(3, 6, 7)), - // (new Link(3, 6, 7), new Link(3, 0, 0)), - - // // Link 4: No changes - // }; - - // // Act - // var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); - - // // Assert - // Assert.Equal(3, simplified.Count); - - // // Link 1: from (1,2,3) to (1,8,9) - // var link1 = simplified.FirstOrDefault(c => c.Before.Index == 1); - // Assert.Equal(new Link(1, 2, 3), link1.Before); - // Assert.Equal(new Link(1, 8, 9), link1.After); - - // // Link 2: from (2,3,4) to (2,5,6) - // var link2 = simplified.FirstOrDefault(c => c.Before.Index == 2); - // Assert.Equal(new Link(2, 3, 4), link2.Before); - // Assert.Equal(new Link(2, 5, 6), link2.After); - - // // Link 3: from (3,4,5) to (3,0,0) - // var link3 = simplified.FirstOrDefault(c => c.Before.Index == 3); - // Assert.Equal(new Link(3, 4, 5), link3.Before); - // Assert.Equal(new Link(3, 0, 0), link3.After); - // } - - // [Fact] - // public void SimplifyChanges_SameAfterMultipleChanges_ReturnsLastChange() - // { - // // Arrange - // var changes = new List<(Link Before, Link After)> - // { - // (new Link(1, 2, 3), new Link(1, 4, 5)), - // (new Link(1, 4, 5), new Link(1, 6, 7)), - // (new Link(1, 6, 7), new Link(1, 4, 5)) // Reverting back - // }; - - // // Act - // var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); - - // // Assert - // Assert.Single(simplified); - - // // Link 1: from (1,2,3) to (1,4,5) - // var link1 = simplified.First(); - // Assert.Equal(new Link(1, 2, 3), link1.Before); - // Assert.Equal(new Link(1, 4, 5), link1.After); - // } - - // [Fact] - // public void SimplifyChanges_DuplicateChanges_IgnoresDuplicates() - // { - // // Arrange - // var changes = new List<(Link Before, Link After)> - // { - // (new Link(1, 2, 3), new Link(1, 4, 5)), - // (new Link(1, 2, 3), new Link(1, 4, 5)), // Duplicate - // (new Link(2, 3, 4), new Link(2, 5, 6)), - // (new Link(2, 3, 4), new Link(2, 5, 6)) // Duplicate - // }; - - // // Act - // var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); - - // // Assert - // Assert.Equal(2, simplified.Count); - - // Assert.Contains(simplified, c => c.Before.Index == 1 && c.After.Index == 1 && c.After.Source == 4 && c.After.Target == 5); - // Assert.Contains(simplified, c => c.Before.Index == 2 && c.After.Index == 2 && c.After.Source == 5 && c.After.Target == 6); - // } - - // [Fact] - // public void SimplifyChanges_NullChanges_ThrowsException() - // { - // // Arrange - // List<(Link Before, Link After)> changes = null; - - // // Act & Assert - // Assert.Throws(() => ChangesSimplifier.SimplifyChanges(changes).ToList()); - // } - - // [Fact] - // public void SimplifyChanges_ChangesWithSameBeforeDifferentAfter_LastAfterIsRetained() - // { - // // Arrange - // var changes = new List<(Link Before, Link After)> - // { - // (new Link(1, 2, 3), new Link(1, 4, 5)), - // (new Link(1, 2, 3), new Link(1, 6, 7)), - // (new Link(1, 2, 3), new Link(1, 8, 9)) - // }; - - // // Act - // var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); - - // // Assert - // Assert.Single(simplified); - - // var link1 = simplified.First(); - // Assert.Equal(new Link(1, 2, 3), link1.Before); - // Assert.Equal(new Link(1, 8, 9), link1.After); // Last change is retained - // } + [Fact] + public void SimplifyChanges_NoChanges_ReturnsEmpty() + { + // Arrange + var changes = new List<(Link Before, Link After)>(); + + // Act + var simplified = ChangesSimplifier.SimplifyChanges(changes); + + // Assert + Assert.Empty(simplified); + } + + [Fact] + public void SimplifyChanges_SingleChange_ReturnsSameChange() + { + // Arrange + var changes = new List<(Link Before, Link After)> + { + (new Link(1, 2, 3), new Link(1, 4, 5)) + }; + + // Act + var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); + + // Assert + Assert.Single(simplified); + Assert.Equal(new Link(1, 2, 3), simplified[0].Before); + Assert.Equal(new Link(1, 4, 5), simplified[0].After); + } + + [Fact] + public void SimplifyChanges_MultipleNonOverlappingChanges_ReturnsAllChanges() + { + // Arrange + var changes = new List<(Link Before, Link After)> + { + (new Link(1, 2, 3), new Link(1, 4, 5)), + (new Link(2, 3, 4), new Link(2, 5, 6)), + (new Link(3, 4, 5), new Link(3, 6, 7)) + }; + + // Act + var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); + + // Assert + Assert.Equal(3, simplified.Count); + + Assert.Contains(simplified, c => c.Before.Index == 1 && c.After.Index == 1 && c.After.Source == 4 && c.After.Target == 5); + Assert.Contains(simplified, c => c.Before.Index == 2 && c.After.Index == 2 && c.After.Source == 5 && c.After.Target == 6); + Assert.Contains(simplified, c => c.Before.Index == 3 && c.After.Index == 3 && c.After.Source == 6 && c.After.Target == 7); + } + + [Fact] + public void SimplifyChanges_MultipleOverlappingChanges_ReturnsInitialToFinal() + { + // Arrange + var changes = new List<(Link Before, Link After)> + { + (new Link(1, 2, 3), new Link(1, 4, 5)), + (new Link(1, 4, 5), new Link(1, 6, 7)), + (new Link(2, 3, 4), new Link(2, 5, 6)), + (new Link(2, 5, 6), new Link(2, 0, 0)), + (new Link(3, 4, 5), new Link(3, 6, 7)) + }; + + // Act + var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); + + // Assert + Assert.Equal(3, simplified.Count); + + // Link 1: from (1,2,3) to (1,6,7) + var link1 = simplified.FirstOrDefault(c => c.Before.Index == 1); + Assert.Equal(new Link(1, 2, 3), link1.Before); + Assert.Equal(new Link(1, 6, 7), link1.After); + + // Link 2: from (2,3,4) to (2,0,0) + var link2 = simplified.FirstOrDefault(c => c.Before.Index == 2); + Assert.Equal(new Link(2, 3, 4), link2.Before); + Assert.Equal(new Link(2, 0, 0), link2.After); + + // Link 3: from (3,4,5) to (3,6,7) + var link3 = simplified.FirstOrDefault(c => c.Before.Index == 3); + Assert.Equal(new Link(3, 4, 5), link3.Before); + Assert.Equal(new Link(3, 6, 7), link3.After); + } + + [Fact] + public void SimplifyChanges_ComplexScenario_MixedChanges() + { + // Arrange + var changes = new List<(Link Before, Link After)> + { + // Link 1: Multiple changes + (new Link(1, 2, 3), new Link(1, 4, 5)), + (new Link(1, 4, 5), new Link(1, 6, 7)), + (new Link(1, 6, 7), new Link(1, 8, 9)), + + // Link 2: Single change + (new Link(2, 3, 4), new Link(2, 5, 6)), + + // Link 3: Multiple changes + (new Link(3, 4, 5), new Link(3, 6, 7)), + (new Link(3, 6, 7), new Link(3, 0, 0)), + + // Link 4: No changes + }; + + // Act + var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); + + // Assert + Assert.Equal(3, simplified.Count); + + // Link 1: from (1,2,3) to (1,8,9) + var link1 = simplified.FirstOrDefault(c => c.Before.Index == 1); + Assert.Equal(new Link(1, 2, 3), link1.Before); + Assert.Equal(new Link(1, 8, 9), link1.After); + + // Link 2: from (2,3,4) to (2,5,6) + var link2 = simplified.FirstOrDefault(c => c.Before.Index == 2); + Assert.Equal(new Link(2, 3, 4), link2.Before); + Assert.Equal(new Link(2, 5, 6), link2.After); + + // Link 3: from (3,4,5) to (3,0,0) + var link3 = simplified.FirstOrDefault(c => c.Before.Index == 3); + Assert.Equal(new Link(3, 4, 5), link3.Before); + Assert.Equal(new Link(3, 0, 0), link3.After); + } + + [Fact] + public void SimplifyChanges_SameAfterMultipleChanges_ReturnsLastChange() + { + // Arrange + var changes = new List<(Link Before, Link After)> + { + (new Link(1, 2, 3), new Link(1, 4, 5)), + (new Link(1, 4, 5), new Link(1, 6, 7)), + (new Link(1, 6, 7), new Link(1, 4, 5)) // Reverting back + }; + + // Act + var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); + + // Assert + Assert.Single(simplified); + + // Link 1: from (1,2,3) to (1,4,5) + var link1 = simplified.First(); + Assert.Equal(new Link(1, 2, 3), link1.Before); + Assert.Equal(new Link(1, 4, 5), link1.After); + } + + [Fact] + public void SimplifyChanges_DuplicateChanges_IgnoresDuplicates() + { + // Arrange + var changes = new List<(Link Before, Link After)> + { + (new Link(1, 2, 3), new Link(1, 4, 5)), + (new Link(1, 2, 3), new Link(1, 4, 5)), // Duplicate + (new Link(2, 3, 4), new Link(2, 5, 6)), + (new Link(2, 3, 4), new Link(2, 5, 6)) // Duplicate + }; + + // Act + var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); + + // Assert + Assert.Equal(2, simplified.Count); + + Assert.Contains(simplified, c => c.Before.Index == 1 && c.After.Index == 1 && c.After.Source == 4 && c.After.Target == 5); + Assert.Contains(simplified, c => c.Before.Index == 2 && c.After.Index == 2 && c.After.Source == 5 && c.After.Target == 6); + } + + [Fact] + public void SimplifyChanges_NullChanges_ThrowsException() + { + // Arrange + List<(Link Before, Link After)> changes = null!; + + // Act & Assert + Assert.Throws(() => ChangesSimplifier.SimplifyChanges(changes).ToList()); + } + + [Fact] + public void SimplifyChanges_ChangesWithSameBeforeDifferentAfter_LastAfterIsRetained() + { + // Arrange + var changes = new List<(Link Before, Link After)> + { + (new Link(1, 2, 3), new Link(1, 4, 5)), + (new Link(1, 2, 3), new Link(1, 6, 7)), + (new Link(1, 2, 3), new Link(1, 8, 9)) + }; + + // Act + var simplified = ChangesSimplifier.SimplifyChanges(changes).ToList(); + + // Assert + Assert.Single(simplified); + + var link1 = simplified.First(); + Assert.Equal(new Link(1, 2, 3), link1.Before); + Assert.Equal(new Link(1, 8, 9), link1.After); // Last change is retained + } } } \ No newline at end of file diff --git a/rust/src/changes_simplifier.rs b/rust/src/changes_simplifier.rs index 2dd9637..467c97e 100644 --- a/rust/src/changes_simplifier.rs +++ b/rust/src/changes_simplifier.rs @@ -1,195 +1,46 @@ -//! ChangesSimplifier - Simplifies a list of changes +//! Simplifies the raw changes of one query to the net change of every link. //! -//! This module provides functionality to simplify a list of changes by -//! identifying chains of transformations. -//! Corresponds to ChangesSimplifier.cs in C# +//! Corresponds to `ChangesSimplifier.cs` in C#. use crate::link::Link; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; -/// Simplifies a list of changes by identifying chains of transformations. +/// Reduces the raw `(before, after)` steps a store reports to one change per +/// link address: from its state before the query to its state after it. /// -/// If multiple final states are reachable from the same initial state, returns multiple simplified changes. -/// If a scenario arises where no initial or final states can be identified (no-ops), returns the original transitions as-is. -pub fn simplify_changes(changes: Vec<(Link, Link)>) -> Vec<(Link, Link)> { - if changes.is_empty() { - return vec![]; - } - - // **FIX for Issue #26**: Remove duplicate before states by keeping the non-null transitions - // This handles cases where the same link is reported with multiple different transformations - let changes = remove_duplicate_before_states(changes); - - // First, handle unchanged states directly - let (unchanged_states, changed_states): (Vec<_>, Vec<_>) = - changes.iter().partition(|(before, after)| before == after); - - // A link that really changed is described by its change alone: reading - // it unchanged on the way (a create followed by a match) is not news. - let changed_indices: HashSet = changed_states - .iter() - .flat_map(|(before, after)| [before.index, after.index]) - .filter(|&index| index != 0) - .collect(); - let unchanged_states: Vec<(Link, Link)> = unchanged_states - .into_iter() - .filter(|(link, _)| !changed_indices.contains(&link.index)) - .collect(); - - // Gather all 'Before' links and all 'After' links from changed states. - // - // C# builds these with `new HashSet>(...)`, which enumerates in - // insertion order; a Rust `HashSet` enumerates in an order derived from a - // per-process random seed. Since the order of `initialStates` decides the - // order of the simplified results whenever the final sort ties, that - // difference would make `--changes` non-reproducible. Keeping the first - // occurrences in a `Vec` (and using the set only for lookups) restores the - // C# ordering. - let before_links = distinct(changed_states.iter().map(|(b, _)| *b)); - let after_links = distinct(changed_states.iter().map(|(_, a)| *a)); - let before_set: HashSet = before_links.iter().copied().collect(); - let after_set: HashSet = after_links.iter().copied().collect(); - - // Identify initial states: appear as Before but never as After - let initial_states: Vec = before_links - .iter() - .filter(|b| !after_set.contains(b)) - .copied() - .collect(); - - // Identify final states: appear as After but never as Before - let final_states: HashSet = after_links - .iter() - .filter(|a| !before_set.contains(a)) - .copied() - .collect(); - - // Build adjacency (Before -> possible list of After links) - let mut adjacency: HashMap> = HashMap::new(); - for (before, after) in changed_states.iter() { - adjacency.entry(*before).or_default().push(*after); - } - - // If we have no identified initial states, treat it as a no-op scenario: - // just return original transitions. - if initial_states.is_empty() { - return changes; - } - - let mut results = Vec::new(); - - // Add unchanged states first - results.extend(unchanged_states); - - // Traverse each initial state with DFS - for initial in distinct(initial_states.iter().copied()) { - let initial = &initial; - let mut stack = vec![*initial]; - let mut visited: HashSet = HashSet::new(); - - while let Some(current) = stack.pop() { - // Skip if already visited - if !visited.insert(current) { - continue; - } - - let has_next = adjacency.contains_key(¤t); - let next_links = adjacency.get(¤t); - let is_final_or_dead_end = final_states.contains(¤t) - || !has_next - || next_links.is_none_or(|v| v.is_empty()); - - // If final or no further transitions, record (initial -> current) - if is_final_or_dead_end { - results.push((*initial, current)); - } - - // Otherwise push neighbors - if let Some(next_links) = next_links { - for next in next_links { - stack.push(*next); - } - } - } - } - - // Sort the final results so that items appear in ascending order by their After link. - // This ensures tests that expect a specific order pass reliably. - results.sort_by(|a, b| { - a.1.index - .cmp(&b.1.index) - .then_with(|| a.1.source.cmp(&b.1.source)) - .then_with(|| a.1.target.cmp(&b.1.target)) - }); - - results -} - -/// Removes problematic duplicate before states that lead to simplification issues. -/// This fixes Issue #26 where multiple transformations from the same before state -/// to conflicting after states (including null states) would cause the simplifier to fail. +/// The store reports every step it takes — a link is emptied to `(i: 0 0)` +/// before it is deleted and created as `(i: 0 0)` before it is set — and the +/// null link `(0: 0 0)` stands for "no link", before a creation and after a +/// deletion. The steps of one address form a chain, so its net change is the +/// `before` of its first step and the `after` of its last: /// -/// The key insight: If we have multiple transitions from the same before state, -/// and one of them is to a "null" state (0: 0 0), we should prefer the non-null transition -/// as it represents the actual final transformation. -fn remove_duplicate_before_states(changes: Vec<(Link, Link)>) -> Vec<(Link, Link)> { - // Group changes by their before state, keeping the groups in the order - // their first member appeared. C# uses `GroupBy`, which is documented to - // preserve that order; iterating a `HashMap` instead would shuffle the - // reported changes differently on every process start. - let mut order: Vec = Vec::new(); - let mut grouped: HashMap> = HashMap::new(); - for change in changes { - let group = grouped.entry(change.0).or_insert_with(|| { - order.push(change.0); - Vec::new() - }); - group.push(change); - } - - let mut result = Vec::new(); - - for before in &order { - let changes_for_this_before = &grouped[before]; - if changes_for_this_before.len() == 1 { - // No duplicates, keep as is - result.extend(changes_for_this_before.iter().copied()); +/// - a link created and deleted again within the query is not reported; +/// - a link that ends where it started is reported unchanged, like a link the +/// query only matched. +/// +/// The changes are ordered by their `after` state, so deletions come first; +/// changes with equal `after` states keep the order of their first step. +pub fn simplify_changes(changes: Vec<(Link, Link)>) -> Vec<(Link, Link)> { + let mut net_changes: Vec<(Link, Link)> = Vec::new(); + let mut position_of_address: HashMap = HashMap::new(); + for (before, after) in changes { + let address = if before.index != 0 { + before.index } else { - // Multiple changes from the same before state - // Check if any of them is to a null state (0: 0 0) - let null_link = Link::new(0, 0, 0); - let has_null_transition = changes_for_this_before - .iter() - .any(|(_, after)| *after == null_link); - let non_null_transitions: Vec<_> = changes_for_this_before - .iter() - .filter(|(_, after)| *after != null_link) - .cloned() - .collect(); - - if has_null_transition && !non_null_transitions.is_empty() { - // Issue #26 scenario: We have both null and non-null transitions - // Prefer the non-null transitions as they represent the actual final states - result.extend(non_null_transitions); - } else { - // No null transitions involved, this is a legitimate multiple-branch scenario - // Keep all transitions - result.extend(changes_for_this_before.iter().copied()); + after.index + }; + if address == 0 { + continue; + } + match position_of_address.get(&address) { + Some(&position) => net_changes[position].1 = after, + None => { + position_of_address.insert(address, net_changes.len()); + net_changes.push((before, after)); } } } - - result -} - -/// The distinct elements of `items`, in the order they first appear. -/// -/// Stands in for enumerating a C# `HashSet`, which yields its elements in -/// insertion order. -fn distinct(items: impl IntoIterator) -> Vec { - let mut seen = HashSet::new(); - items - .into_iter() - .filter(|item| seen.insert(*item)) - .collect() + net_changes.retain(|(before, after)| !(before.is_null() && after.is_null())); + net_changes.sort_by_key(|(_, after)| (after.index, after.source, after.target)); + net_changes } diff --git a/rust/tests/changes_simplifier_tests.rs b/rust/tests/changes_simplifier_tests.rs index 1ae8b2b..164cd52 100644 --- a/rust/tests/changes_simplifier_tests.rs +++ b/rust/tests/changes_simplifier_tests.rs @@ -177,3 +177,48 @@ fn test_simplify_keeps_unchanged_links_that_did_not_change_elsewhere() { vec![(Link::null(), created), (unchanged, unchanged)] ); } + +#[test] +fn test_simplify_recreating_an_address_reports_one_change() { + // The raw steps C# reports for `((1: 1 1)) ((3 3))`. + let changes = vec![ + (Link::new(1, 1, 1), Link::new(1, 0, 0)), + (Link::new(1, 0, 0), Link::null()), + (Link::null(), Link::new(1, 0, 0)), + (Link::new(1, 0, 0), Link::new(1, 3, 3)), + ]; + + assert_eq!( + simplify_changes(changes), + vec![(Link::new(1, 1, 1), Link::new(1, 3, 3))] + ); +} + +#[test] +fn test_simplify_does_not_chain_a_deletion_into_another_creation() { + let deleted = Link::new(3, 3, 3); + let created = Link::new(4, 4, 4); + let changes = vec![(deleted, Link::null()), (Link::null(), created)]; + + assert_eq!( + simplify_changes(changes), + vec![(deleted, Link::null()), (Link::null(), created)] + ); +} + +#[test] +fn test_simplify_drops_a_link_created_and_deleted_again() { + let temporary = Link::new(3, 0, 0); + let changes = vec![(Link::null(), temporary), (temporary, Link::null())]; + + assert_eq!(simplify_changes(changes), vec![]); +} + +#[test] +fn test_simplify_reports_a_link_changed_back_as_unchanged() { + let original = Link::new(1, 1, 2); + let swapped = Link::new(1, 2, 1); + let changes = vec![(original, swapped), (swapped, original)]; + + assert_eq!(simplify_changes(changes), vec![(original, original)]); +} From d4cec6768437e6821e07a7ebc3327181968d040c Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 00:34:35 +0000 Subject: [PATCH 19/44] fix: report every link a query creates and predict new addresses from the store Named point links and leaves created on the way were not reported, or were reported as an empty link that was filled in later. Every creation is now reported once as the link it ends up being, in both ports. Creating under an existing name redefines that link, and a new name for an existing doublet names it instead of copying it. Reference validation assumed that a new link gets the lowest free address. Both stores reuse the address freed last first, so a reference to a freed address the query never refills was accepted and then silently dropped, and C# EnsureCreated threw on a freed address above its target. The validator now asks the store for the next addresses by creating and deleting them in reverse, and repeats the plan until auto-created references settle. --- README.md | 4 +- .../.changeset/issue-105-creation-reports.md | 10 + .../AdvancedMixedQueryProcessor.Mutations.cs | 115 ++++++---- .../AdvancedMixedQueryProcessor.References.cs | 156 +++++-------- .../AdvancedMixedQueryProcessor.cs | 5 +- .../LinksExtensions.cs | 17 +- ...vancedMixedQueryProcessor.ChangeReports.cs | 191 +++++++++++++++ .../AdvancedMixedQueryProcessor.More.cs | 34 +++ ...61005_090000_issue_105_creation_reports.md | 9 + rust/src/lib.rs | 2 +- rust/src/link_reference_validator.rs | 217 +++++++----------- rust/src/query_processor.rs | 101 ++++---- rust/src/query_processor/mutations.rs | 91 +++++--- rust/tests/protocol_tcp_tests.rs | 2 +- .../query_processor_change_report_tests.rs | 167 ++++++++++++++ rust/tests/query_processor_tests.rs | 56 ++++- 16 files changed, 809 insertions(+), 368 deletions(-) create mode 100644 csharp/.changeset/issue-105-creation-reports.md create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.ChangeReports.cs create mode 100644 rust/changelog.d/20261005_090000_issue_105_creation_reports.md create mode 100644 rust/tests/query_processor_change_report_tests.rs diff --git a/README.md b/README.md index 41cbaaf..b568479 100644 --- a/README.md +++ b/README.md @@ -214,8 +214,8 @@ clink --db family.links --auto-create-missing-references '() ((child: father mot ``` → ``` -((father: 0 0)) ((father: father father)) -((mother: 0 0)) ((mother: mother mother)) +() ((father: father father)) +() ((mother: mother mother)) () ((child: father mother)) (father: father father) (mother: mother mother) diff --git a/csharp/.changeset/issue-105-creation-reports.md b/csharp/.changeset/issue-105-creation-reports.md new file mode 100644 index 0000000..48c9c2b --- /dev/null +++ b/csharp/.changeset/issue-105-creation-reports.md @@ -0,0 +1,10 @@ +--- +'Foundation.Data.Doublets.Cli': patch +--- + +Fixed the `--changes` report of creations: + +- Every link a query creates is reported, named point links and leaves created on the way included. Each one is reported once, as `() ((name: name name))`, not as an empty link filled in later. +- Creating a link under an existing name redefines that link in place, and a new name for an existing doublet names it instead of copying it. +- Reference validation predicts the addresses new links get by asking the store. Stores reuse the address freed last first, so the lowest free address is not always next. A reference to a freed address the query does not refill is now reported missing instead of being dropped. +- `EnsureCreated` no longer throws when the store hands out a freed address above the target before reaching it. diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.Mutations.cs b/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.Mutations.cs index 48719a0..7af1f1c 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.Mutations.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.Mutations.cs @@ -42,11 +42,7 @@ private static void CreateOrUpdateLink(INamedTypesLinks links, DoubletLink if (linkDefinition.Index != nullConstant) { // update existing link - if (!links.Exists(linkDefinition.Index)) - { - TraceIfEnabled(options, $"[CreateOrUpdateLink] Link #{linkDefinition.Index} doesn't exist => ensuring creation."); - LinksExtensions.EnsureCreated(links, linkDefinition.Index); - } + EnsureAddress(links, linkDefinition.Index, options); var existingLinkRecord = links.GetLink(linkDefinition.Index); var existingDoublet = new DoubletLink(existingLinkRecord); @@ -54,7 +50,6 @@ private static void CreateOrUpdateLink(INamedTypesLinks links, DoubletLink { TraceIfEnabled(options, $"[CreateOrUpdateLink] Updating link #{linkDefinition.Index}: {existingDoublet.Source}->{linkDefinition.Source}, {existingDoublet.Target}->{linkDefinition.Target}."); - LinksExtensions.EnsureCreated(links, linkDefinition.Index); links.Update( new DoubletLink(linkDefinition.Index, anyConstant, anyConstant), linkDefinition, @@ -261,11 +256,7 @@ private static uint EnsureLinkCreated(INamedTypesLinks links, DoubletLink else { // We have an index => ensure created or updated - if (!links.Exists(link.Index)) - { - TraceIfEnabled(options, $"[EnsureLinkCreated] Link #{link.Index} doesn't exist => ensuring creation."); - LinksExtensions.EnsureCreated(links, link.Index); - } + EnsureAddress(links, link.Index, options); var stored = links.GetLink(link.Index); var storedD = new DoubletLink(stored); if (storedD.Source != link.Source || storedD.Target != link.Target) @@ -303,30 +294,65 @@ private static void TraceIfEnabled(Options options, string message) } } - // Consolidates getting or creating a named link (leaf) without setting its relationships - private static uint EnsureNamedLeafLink(INamedTypesLinks links, string name, Options options) + /// + /// Fills in when the store lacks it and reports the empty + /// (index: 0 0) link that creates. + /// + private static void EnsureAddress(INamedTypesLinks links, uint index, Options options) + { + if (links.Exists(index)) + { + return; + } + TraceIfEnabled(options, $"[EnsureAddress] Link #{index} doesn't exist => ensuring creation."); + LinksExtensions.EnsureCreated(links, index); + ReportCreatedEmpty(links, index, options); + } + + private static void ReportCreatedEmpty(INamedTypesLinks links, uint index, Options options) + { + var nothing = links.Constants.Null; + options.ChangesHandler?.Invoke(new DoubletLink(nothing, nothing, nothing), new DoubletLink(index, nothing, nothing)); + } + + /// + /// The link called , created and reported as the point link + /// (name: name name) when there is none yet. + /// + private static uint EnsureNamedPointLink(INamedTypesLinks links, string name, Options options) { var existing = links.GetByName(name); - if (existing != links.Constants.Null) return existing; - var newId = links.CreateAndUpdate(links.Constants.Null, links.Constants.Null); - TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] Created named leaf '{name}' => ID={newId}"); - links.SetName(newId, name); - return newId; + if (existing != links.Constants.Null) + { + return existing; + } + var created = links.Create(); + ReportCreatedEmpty(links, created, options); + links.SetName(created, name); + ApplyCompositeUpdate(links, created, created, created, options); + TraceIfEnabled(options, $"[EnsureNamedPointLink] Created point link '{name}' => ID={created}"); + return created; } - // Applies a single structural update to an existing link: sets its source and target + /// + /// Points the existing link at and + /// ; a link that already does is reported unchanged and not rewritten. + /// private static void ApplyCompositeUpdate(INamedTypesLinks links, uint id, uint source, uint target, Options options) { - var restriction = new DoubletLink(id, links.Constants.Null, links.Constants.Null); - var substitution = new DoubletLink(id, source, target); - TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] Updating link ID={id} => Source={source}, Target={target}"); - links.Update(restriction, substitution, (before, after) => + var stored = new DoubletLink(links.GetLink(id)); + if (stored.Source == source && stored.Target == target) { - TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] Update handler: before={before}, after={after}"); - return links.Constants.Continue; - }); + options.ChangesHandler?.Invoke(stored, stored); + return; + } + TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] Updating link ID={id} => Source={source}, Target={target}"); + links.Update(new DoubletLink(id, links.Constants.Any, links.Constants.Any), new DoubletLink(id, source, target), ReportChanges(links, options)); } + private static WriteHandler ReportChanges(INamedTypesLinks links, Options options) => + (before, after) => options.ChangesHandler?.Invoke(before, after) ?? links.Constants.Continue; + /// /// Detects a two-child composite pattern where at least one child matches the composite identifier. /// @@ -366,7 +392,7 @@ private static CompositeCase ClassifyCompositeCase(string name, LinoLink left, L private static uint HandleStringComposite(string name, LinoLink left, LinoLink right, INamedTypesLinks links, Options options) { - var id = EnsureNamedLeafLink(links, name, options); + var id = EnsureNamedPointLink(links, name, options); var caseType = ClassifyCompositeCase(name, left, right); switch (caseType) { @@ -418,25 +444,7 @@ private static uint ResolveLeaf(LinoLink pattern, INamedTypesLinks links, TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] Leaf parse => returning {parsedNumber}."); return parsedNumber; } - var existingId = links.GetByName(pattern.Id); - if (existingId != links.Constants.Null) - { - TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] Found existing named leaf '{pattern.Id}' => ID={existingId}"); - return existingId; - } - var newLeafId = links.CreateAndUpdate(links.Constants.Null, links.Constants.Null); - TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] SetName({newLeafId}, '{pattern.Id}')"); - links.SetName(newLeafId, pattern.Id); - var restriction = new DoubletLink(newLeafId, links.Constants.Null, links.Constants.Null); - var substitution = new DoubletLink(newLeafId, newLeafId, newLeafId); - TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] Updating link {newLeafId} to be self-referential"); - links.Update(restriction, substitution, (beforeState, afterState) => - { - TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] Update handler: before={beforeState}, after={afterState}"); - return links.Constants.Continue; - }); - TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] Created new self-referential named leaf '{pattern.Id}' => ID={newLeafId}"); - return newLeafId; + return EnsureNamedPointLink(links, pattern.Id, options); } /// @@ -467,14 +475,23 @@ private static uint CreateCompositeLink( } } } + var isName = !string.IsNullOrEmpty(literalIdentifier) && !IsNumericOrStar(literalIdentifier) && !literalIdentifier.StartsWith("$"); + if (isName) + { + // A name is an address too: the link it already names is updated in place. + var named = links.GetByName(literalIdentifier!); + if (named != links.Constants.Null) + { + compositeIndex = named; + } + } // Build the composite link structure and ensure it exists var compositeLinkDefinition = new DoubletLink(compositeIndex, sourceLinkId, targetLinkId); var compositeLinkId = EnsureLinkCreated(links, compositeLinkDefinition, options); TraceIfEnabled(options, $"[EnsureNestedLinkCreatedRecursively] Created or ensured composite link => Index={compositeIndex}, Source={sourceLinkId}, Target={targetLinkId} => Actual ID={compositeLinkId}"); - // Assign the name for non-numeric identifiers - if (!string.IsNullOrEmpty(literalIdentifier) && !IsNumericOrStar(literalIdentifier) && !literalIdentifier.StartsWith("$")) + if (isName) { - links.SetName(compositeLinkId, literalIdentifier); + links.SetName(compositeLinkId, literalIdentifier!); } return compositeLinkId; } diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.References.cs b/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.References.cs index 7b5a408..80b8bae 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.References.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.References.cs @@ -19,16 +19,15 @@ private static void ValidateLinksExistOrWillBeCreated( { TraceIfEnabled(options, "[ValidateLinksExistOrWillBeCreated] Starting validation"); - var plan = BuildLinkReferencePlan(links, substitutionPatterns); - - TraceIfEnabled(options, $"[ValidateLinksExistOrWillBeCreated] Numeric links to be created: {string.Join(", ", plan.NumericIdsToBeCreated.OrderBy(id => id))}"); - TraceIfEnabled(options, $"[ValidateLinksExistOrWillBeCreated] Named links to be created: {string.Join(", ", plan.NamesToBeCreated.OrderBy(name => name, StringComparer.Ordinal))}"); - - CollectMissingReferences(restrictionPatterns, links, plan, false, "restriction", options); - CollectMissingReferences(substitutionPatterns, links, plan, true, "substitution", options); - - if (plan.MissingReferences.Count > 0) + while (true) { + var plan = PlanReferences(links, restrictionPatterns, substitutionPatterns, options); + if (plan.MissingReferences.Count == 0) + { + TraceIfEnabled(options, "[ValidateLinksExistOrWillBeCreated] Validation completed"); + return; + } + if (!options.AutoCreateMissingReferences) { var missing = plan.MissingReferences[0]; @@ -39,10 +38,26 @@ private static void ValidateLinksExistOrWillBeCreated( ); } + // Creating a reference frees addresses on the way, which changes the ones the query's + // new links get, so the plan is made again until every reference is accounted for. AutoCreateMissingReferences(links, plan, options); } + } + + private static LinkReferencePlan PlanReferences( + INamedTypesLinks links, + IList restrictionPatterns, + IList substitutionPatterns, + Options options) + { + var plan = BuildLinkReferencePlan(links, substitutionPatterns); - TraceIfEnabled(options, "[ValidateLinksExistOrWillBeCreated] Validation completed"); + TraceIfEnabled(options, $"[ValidateLinksExistOrWillBeCreated] Numeric links to be created: {string.Join(", ", plan.NumericIdsToBeCreated.OrderBy(id => id))}"); + TraceIfEnabled(options, $"[ValidateLinksExistOrWillBeCreated] Named links to be created: {string.Join(", ", plan.NamesToBeCreated.OrderBy(name => name, StringComparer.Ordinal))}"); + + CollectMissingReferences(restrictionPatterns, links, plan, false, "restriction", options); + CollectMissingReferences(substitutionPatterns, links, plan, true, "substitution", options); + return plan; } private sealed class LinkReferencePlan @@ -73,100 +88,66 @@ private sealed class MissingLinkReference private static LinkReferencePlan BuildLinkReferencePlan(INamedTypesLinks links, IList substitutionPatterns) { var plan = new LinkReferencePlan(); - var reservedNumericIds = new HashSet(); - + var anonymousLinks = 0; foreach (var pattern in substitutionPatterns) { - CollectExplicitDefinitions(pattern, plan, reservedNumericIds); + CollectDefinitions(pattern, plan, ref anonymousLinks); } - - foreach (var pattern in substitutionPatterns) - { - CollectImplicitDefinitions(pattern, links, plan, reservedNumericIds); - } - - foreach (var pattern in substitutionPatterns) - { - CollectCompositePairs(pattern, plan); - } - + plan.NumericIdsToBeCreated.UnionWith(NextCreatedAddresses(links, anonymousLinks)); return plan; } - private static void CollectExplicitDefinitions(LinoLink pattern, LinkReferencePlan plan, HashSet reservedNumericIds) + /// + /// Collects the ids and names the substitution defines, the (source, target) pairs it writes + /// under them, and how many links it writes without an id. + /// + private static void CollectDefinitions(LinoLink pattern, LinkReferencePlan plan, ref int anonymousLinks) { - if (IsComposite(pattern) && TryGetConcreteIdentifier(pattern.Id, out var identifier)) - { - if (uint.TryParse(identifier, out var linkId)) - { - plan.NumericIdsToBeCreated.Add(linkId); - reservedNumericIds.Add(linkId); - } - else - { - plan.NamesToBeCreated.Add(identifier); - } - } - if (pattern.Values != null) { foreach (var subPattern in pattern.Values) { - CollectExplicitDefinitions(subPattern, plan, reservedNumericIds); + CollectDefinitions(subPattern, plan, ref anonymousLinks); } } - } - - private static void CollectCompositePairs(LinoLink pattern, LinkReferencePlan plan) - { - if (IsComposite(pattern) && - TryGetConcreteIdentifier(pattern.Id, out var _ignoredIdentifier) && - pattern.Values != null && - TryGetConcreteNumericIdentifier(pattern.Values[0].Id, out var source) && - TryGetConcreteNumericIdentifier(pattern.Values[1].Id, out var target)) + if (!IsComposite(pattern)) { - plan.CompositePairsToBeCreated.Add((source, target)); + return; } - - if (pattern.Values != null) + if (!TryGetConcreteIdentifier(pattern.Id, out var identifier)) { - foreach (var subPattern in pattern.Values) - { - CollectCompositePairs(subPattern, plan); - } + anonymousLinks++; + return; } - } - - private static void CollectImplicitDefinitions( - LinoLink pattern, - INamedTypesLinks links, - LinkReferencePlan plan, - HashSet reservedNumericIds) - { - if (pattern.Values != null) + if (uint.TryParse(identifier, out var linkId)) { - foreach (var subPattern in pattern.Values) - { - CollectImplicitDefinitions(subPattern, links, plan, reservedNumericIds); - } + plan.NumericIdsToBeCreated.Add(linkId); } - - if (IsComposite(pattern) && !TryGetConcreteIdentifier(pattern.Id, out var _ignoredIdentifier)) + else { - var nextId = GetNextAvailableLinkId(links, reservedNumericIds); - reservedNumericIds.Add(nextId); - plan.NumericIdsToBeCreated.Add(nextId); + plan.NamesToBeCreated.Add(identifier); + } + if (TryGetConcreteNumericIdentifier(pattern.Values![0].Id, out var source) && + TryGetConcreteNumericIdentifier(pattern.Values[1].Id, out var target)) + { + plan.CompositePairsToBeCreated.Add((source, target)); } } - private static uint GetNextAvailableLinkId(INamedTypesLinks links, HashSet reservedNumericIds) + /// The addresses the next links created in get. + /// + /// A store reuses the address freed last first, so the lowest free one is not necessarily next. + /// Creating the links and deleting them again in reverse leaves the store exactly as it was, + /// and is the one way to ask any store, a remote one included. + /// + private static List NextCreatedAddresses(INamedTypesLinks links, int count) { - uint nextId = 1; - while (links.Exists(nextId) || reservedNumericIds.Contains(nextId)) + var addresses = Enumerable.Range(0, count).Select(_ => links.Create()).ToList(); + for (var i = addresses.Count - 1; i >= 0; i--) { - nextId++; + links.Delete(addresses[i]); } - return nextId; + return addresses; } private static void CollectMissingReferences( @@ -258,7 +239,7 @@ private static void AutoCreateMissingReferences( } TraceIfEnabled(options, $"[ValidateLinksExistOrWillBeCreated] Auto-creating missing numeric reference {linkId}."); - LinksExtensions.EnsureCreated(links, linkId); + EnsureAddress(links, linkId, options); if (plan.CompositePairsToBeCreated.Contains((linkId, linkId))) { TraceIfEnabled(options, $"[ValidateLinksExistOrWillBeCreated] Link {linkId} exists as a placeholder because ({linkId}, {linkId}) is defined by the substitution."); @@ -284,23 +265,6 @@ private static void AutoCreateMissingReferences( } } - private static void EnsureNamedPointLink(INamedTypesLinks links, string name, Options options) - { - if (links.GetByName(name) != links.Constants.Null) - { - return; - } - - var newId = links.CreateAndUpdate(links.Constants.Null, links.Constants.Null); - links.SetName(newId, name); - links.Update( - new DoubletLink(newId, links.Constants.Null, links.Constants.Null), - new DoubletLink(newId, newId, newId), - (beforeState, afterState) => - options.ChangesHandler?.Invoke(beforeState, afterState) ?? links.Constants.Continue - ); - } - private static bool IsComposite(LinoLink pattern) => pattern.Values?.Count == 2; private static bool TryGetConcreteIdentifier(string? id, out string identifier) diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.cs b/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.cs index 27105fc..fe5f52e 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/AdvancedMixedQueryProcessor.cs @@ -452,10 +452,7 @@ private static void ApplyAllPlannedOperations( if (before.Index == after.Index) { TraceIfEnabled(options, $"[ApplyAllPlannedOperations] Updating link in-place => ID={before.Index}"); - if (!links.Exists(after.Index)) - { - LinksExtensions.EnsureCreated(links, after.Index); - } + EnsureAddress(links, after.Index, options); links.Update(before, after, (beforeState, afterState) => options.ChangesHandler?.Invoke(beforeState, afterState) ?? links.Constants.Continue); } diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/LinksExtensions.cs b/csharp/Foundation.Data.Doublets.Cli.Library/LinksExtensions.cs index 0622e64..a3377a4 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/LinksExtensions.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/LinksExtensions.cs @@ -37,7 +37,9 @@ public static void EnsureCreated(this ILinks links, throw new InvalidOperationException($"Link creation returned address {createdLink} more than once before reaching target {max}."); } - if (Comparer.Default.Compare(createdLink, max) > 0) + // A store hands out the address freed last first, which can lie beyond the target. An + // address beyond every link the store held is appended instead, and so is every one after it. + if (Comparer.Default.Compare(createdLink, max) > 0 && !IsBelowAStoredLink(links, createdLink, seenCreatedLinks)) { throw new InvalidOperationException($"Link creation produced address {createdLink} beyond requested target {max}."); } @@ -56,6 +58,19 @@ public static void EnsureCreated(this ILinks links, } } + private static bool IsBelowAStoredLink(ILinks links, TLinkAddress address, ISet createdLinks) where TLinkAddress : IUnsignedNumber + { + var constants = links.Constants; + var found = false; + links.Each([constants.Any], link => + { + var index = link![constants.IndexPart]; + found = Comparer.Default.Compare(index, address) > 0 && !createdLinks.Contains(index); + return found ? constants.Break : constants.Continue; + }); + return found; + } + private static void EnsureSupportedInternalReference(ILinks links, TLinkAddress address) where TLinkAddress : IUnsignedNumber { if (!links.Constants.IsInternalReference(address)) diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.ChangeReports.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.ChangeReports.cs new file mode 100644 index 0000000..d54ace8 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.ChangeReports.cs @@ -0,0 +1,191 @@ +// A query reports every link it creates, names or redefines, and refers only +// to links that exist once it is done. +using DoubletLink = Platform.Data.Doublets.Link; + +using static Foundation.Data.Doublets.Cli.AdvancedMixedQueryProcessor; +namespace Foundation.Data.Doublets.Cli.Tests.Tests +{ + public partial class AdvancedMixedQueryProcessor + { + private static readonly DoubletLink NoLink = new(0, 0, 0); + + private static List<(DoubletLink Before, DoubletLink After)> ReportedChanges(NamedTypesDecorator links, string query) + { + var changes = new List<(DoubletLink Before, DoubletLink After)>(); + ProcessQuery(links, new Options + { + Query = query, + ChangesHandler = (before, after) => + { + changes.Add((new DoubletLink(before), new DoubletLink(after))); + return links.Constants.Continue; + } + }); + return Sorted(Foundation.Data.Doublets.Cli.ChangesSimplifier.SimplifyChanges(changes)); + } + + private static List<(DoubletLink Before, DoubletLink After)> Sorted(IEnumerable<(DoubletLink Before, DoubletLink After)> changes) => + changes.OrderBy(change => change.Before.Index).ThenBy(change => change.After.Index).ToList(); + + private static (DoubletLink Before, DoubletLink After) Created(uint index, uint source, uint target) => + (NoLink, new DoubletLink(index, source, target)); + + private static uint Named(NamedTypesDecorator links, string name) + { + var index = links.GetByName(name); + Assert.NotEqual(links.Constants.Null, index); + return index; + } + + [Fact] + public void NumericPointLinkIsReportedCreated() + { + RunTestWithLinks(links => + { + Assert.Equal([Created(2, 2, 2)], ReportedChanges(links, "() ((2: 2 2))")); + }); + } + + [Fact] + public void NamedPointLinkIsReportedCreated() + { + RunTestWithLinks(links => + { + var changes = ReportedChanges(links, "() ((a: a a))"); + var a = Named(links, "a"); + Assert.Equal([Created(a, a, a)], changes); + }); + } + + [Fact] + public void NamedLeafCreatedOnTheWayIsReportedCreated() + { + RunTestWithLinks(links => + { + var changes = ReportedChanges(links, "() ((c: c d))"); + var (c, d) = (Named(links, "c"), Named(links, "d")); + Assert.Equal(Sorted([Created(c, c, d), Created(d, d, d)]), changes); + }); + } + + [Fact] + public void EveryNamedLeafOfACompositeIsReportedCreated() + { + RunTestWithLinks(links => + { + var changes = ReportedChanges(links, "() ((child: father mother))"); + var (child, father, mother) = (Named(links, "child"), Named(links, "father"), Named(links, "mother")); + Assert.Equal(Sorted([Created(child, father, mother), Created(father, father, father), Created(mother, mother, mother)]), changes); + Assert.Equal(3, GetAllLinks(links).Count); + }); + } + + [Fact] + public void CreatingAnExistingNameRedefinesThatLink() + { + RunTestWithLinks(links => + { + ProcessQuery(links, "() ((child: father mother))"); + var (child, father, mother) = (Named(links, "child"), Named(links, "father"), Named(links, "mother")); + + Assert.Equal( + [(new DoubletLink(child, father, mother), new DoubletLink(child, mother, father))], + ReportedChanges(links, "() ((child: mother father))")); + Assert.Equal(3, GetAllLinks(links).Count); + }); + } + + [Fact] + public void NewNameForAnExistingDoubletNamesItInsteadOfCopyingIt() + { + RunTestWithLinks(links => + { + ProcessQuery(links, "() ((a: a a))"); + var a = Named(links, "a"); + var point = new DoubletLink(a, a, a); + + Assert.Equal([(point, point)], ReportedChanges(links, "() ((b: a a))")); + Assert.Equal(a, Named(links, "b")); + Assert.Equal([point], GetAllLinks(links)); + }); + } + + [Fact] + public void NumericReferenceIsReportedCreatedAsThePointLink() + { + RunTestWithLinks(links => + { + Assert.Equal([Created(2, 2, 2), Created(3, 3, 2)], ReportedChanges(links, "() ((3: 3 2))")); + }); + } + + /// + /// Doublets are unique, so the point link (2: 2 2) and the defined (3: 2 2) cannot both exist: + /// the reference is left empty. + /// + [Fact] + public void ReferenceToThePairALinkDefinesIsReportedCreatedEmpty() + { + RunTestWithLinks(links => + { + Assert.Equal([Created(2, 0, 0), Created(3, 2, 2)], ReportedChanges(links, "() ((3: 2 2))")); + Assert.Equal([new DoubletLink(2, 0, 0), new DoubletLink(3, 2, 2)], GetAllLinks(links)); + }); + } + + /// Links 1 to 4, with 2 and then 3 deleted: the store hands out 3, the address freed last, before 2. + private static void FreeAddresses2And3(NamedTypesDecorator links) + { + ProcessQueryStrict(links, "() ((1 1) (2 2) (3 3) (4 4))"); + ProcessQueryStrict(links, "((2: 2 2)) ()"); + ProcessQueryStrict(links, "((3: 3 3)) ()"); + } + + [Fact] + public void ReferenceToAFreedAddressTheNewLinkDoesNotGetFails() + { + RunTestWithLinks(links => + { + FreeAddresses2And3(links); + + var exception = Assert.Throws(() => ProcessQueryStrict(links, "() ((2 2))")); + + Assert.Contains("'2'", exception.Message); + Assert.Equal(2, GetAllLinks(links).Count); + }); + } + + [Fact] + public void AutoCreateAReferenceToAFreedAddressTheNewLinkDoesNotGet() + { + RunTestWithLinks(links => + { + FreeAddresses2And3(links); + + ProcessQuery(links, "() ((2 2))"); + + var allLinks = GetAllLinks(links); + AssertLinkExists(allLinks, 2, 2, 2); + Assert.Equal(3, allLinks.Count); + }); + } + + [Fact] + public void AutoCreateAReferenceTheNewLinkWouldHaveGotBefore() + { + RunTestWithLinks(links => + { + ProcessQueryStrict(links, "() ((1 1) (2 2) (3 3) (4 4))"); + + // Creating 7 frees 5 and 6 on the way, so (5 7) gets 6, not 5. + ProcessQuery(links, "() ((5 7))"); + + var allLinks = GetAllLinks(links); + AssertLinkExists(allLinks, 5, 5, 5); + AssertLinkExists(allLinks, 6, 5, 7); + AssertLinkExists(allLinks, 7, 7, 7); + Assert.Equal(7, allLinks.Count); + }); + } + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.More.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.More.cs index 5d12fde..7a0de3a 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.More.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.More.cs @@ -885,6 +885,40 @@ public void EnsureCreated_WithSpecialAnyReference_ShouldThrowControlledException }); } + [Fact] + public void EnsureCreated_ReachesAnAddressBelowOneFreedAfterIt() + { + RunTestWithLinks(links => + { + ProcessQueryStrict(links, "() ((1 1) (2 2) (3 3) (4 4))"); + ProcessQueryStrict(links, "((2: 2 2)) ()"); + ProcessQueryStrict(links, "((3: 3 3)) ()"); + + LinksExtensions.EnsureCreated(links, 2u); + + Assert.True(links.Exists(2u)); + Assert.False(links.Exists(3u)); + Assert.Equal(3u, links.Create()); + }); + } + + [Fact] + public void EnsureCreated_WithACreatorThatAppendsPastTheTarget_ShouldThrowControlledException() + { + RunTestWithLinks(links => + { + uint SkipOne() + { + links.Create(); + return links.Create(); + } + + var exception = Assert.Throws(() => LinksExtensions.EnsureCreated(links, SkipOne, 1u)); + + Assert.Contains("beyond requested target 1", exception.Message); + }); + } + // Helper methods /// diff --git a/rust/changelog.d/20261005_090000_issue_105_creation_reports.md b/rust/changelog.d/20261005_090000_issue_105_creation_reports.md new file mode 100644 index 0000000..800ea9a --- /dev/null +++ b/rust/changelog.d/20261005_090000_issue_105_creation_reports.md @@ -0,0 +1,9 @@ +--- +bump: patch +--- + +Fixed the `--changes` report of creations: + +- Every link a query creates is reported, named point links and leaves created on the way included. Each one is reported once, as `() ((name: name name))`, not as an empty link filled in later. +- Creating a link under an existing name redefines that link in place, and a new name for an existing doublet names it instead of copying it. +- Reference validation predicts the addresses new links get by asking the store. Stores reuse the address freed last first, so the lowest free address is not always next. A reference to a freed address the query does not refill is now reported missing instead of being dropped. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 60ba2bf..6382c07 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -90,7 +90,7 @@ pub use protocol::{ BinaryLinoOptions, BinaryLinoProtocol, LinksClient, LinksServer, LinoProtocol, TextLinoProtocol, }; pub use query_options::QueryOptions; -pub use query_processor::QueryProcessor; +pub use query_processor::{Changes, QueryProcessor}; pub use storage::{ decorators, lock_file_path, DoubletsStorage, FileLock, FileMappedUnitStore, FileMappedValue, LinksStorage, LinksStorageRef, LockMode, PersistentFileMapped, ResolvedFileMappedUnitStore, diff --git a/rust/src/link_reference_validator.rs b/rust/src/link_reference_validator.rs index 3317b53..f36a0e3 100644 --- a/rust/src/link_reference_validator.rs +++ b/rust/src/link_reference_validator.rs @@ -12,6 +12,7 @@ use crate::error::LinkError; use crate::link::Link; use crate::lino_link::LinoLink; use crate::named_type_links::NamedTypeLinks; +use crate::query_processor::Changes; pub struct LinkReferenceValidator { trace: bool, @@ -70,10 +71,41 @@ impl LinkReferenceValidator { storage: &mut impl NamedTypeLinks, restriction_patterns: &[LinoLink], substitution_patterns: &[LinoLink], - ) -> Result> { + ) -> Result { self.trace_msg("[ValidateLinksExistOrWillBeCreated] Starting validation"); - let mut plan = self.build_link_reference_plan(storage, substitution_patterns); + let mut created = Vec::new(); + loop { + let plan = + self.plan_references(storage, restriction_patterns, substitution_patterns)?; + if plan.missing_references.is_empty() { + self.trace_msg("[ValidateLinksExistOrWillBeCreated] Validation completed"); + return Ok(created); + } + + if !self.auto_create_missing_references { + let missing = &plan.missing_references[0]; + return Err(LinkError::QueryError(format!( + "Invalid reference to non-existent link '{}' in {} pattern. Link '{}' does not exist and will not be created by this operation. Use --auto-create-missing-references to create missing references as point links.", + missing.identifier, missing.pattern_type, missing.identifier + )) + .into()); + } + + // Creating a reference frees addresses on the way, which changes + // the ones the query's new links get, so the plan is made again + // until every reference is accounted for. + created.extend(self.auto_create_missing_references(storage, &plan)?); + } + } + + fn plan_references( + &self, + storage: &mut impl NamedTypeLinks, + restriction_patterns: &[LinoLink], + substitution_patterns: &[LinoLink], + ) -> Result { + let mut plan = self.build_link_reference_plan(storage, substitution_patterns)?; self.trace_msg(&format!( "[ValidateLinksExistOrWillBeCreated] Numeric links to be created: {:?}", plan.numeric_ids_to_be_created @@ -97,130 +129,69 @@ impl LinkReferenceValidator { true, "substitution", )?; - - if plan.missing_references.is_empty() { - self.trace_msg("[ValidateLinksExistOrWillBeCreated] Validation completed"); - return Ok(vec![]); - } - - if !self.auto_create_missing_references { - let missing = &plan.missing_references[0]; - return Err(LinkError::QueryError(format!( - "Invalid reference to non-existent link '{}' in {} pattern. Link '{}' does not exist and will not be created by this operation. Use --auto-create-missing-references to create missing references as point links.", - missing.identifier, missing.pattern_type, missing.identifier - )) - .into()); - } - - let created = self.auto_create_missing_references(storage, &plan)?; - self.trace_msg("[ValidateLinksExistOrWillBeCreated] Validation completed"); - Ok(created) + Ok(plan) } fn build_link_reference_plan( &self, storage: &mut impl NamedTypeLinks, substitution_patterns: &[LinoLink], - ) -> LinkReferencePlan { + ) -> Result { let mut plan = LinkReferencePlan::default(); - let mut reserved_numeric_ids = HashSet::new(); - - for pattern in substitution_patterns { - self.collect_explicit_definitions(pattern, &mut plan, &mut reserved_numeric_ids); - } - + let mut anonymous_links = 0; for pattern in substitution_patterns { - self.collect_implicit_definitions( - storage, - pattern, - &mut plan, - &mut reserved_numeric_ids, - ); + Self::collect_definitions(pattern, &mut plan, &mut anonymous_links); } - - for pattern in substitution_patterns { - Self::collect_composite_pairs(pattern, &mut plan); - } - - plan + plan.numeric_ids_to_be_created + .extend(Self::next_created_addresses(storage, anonymous_links)?); + Ok(plan) } - fn collect_explicit_definitions( - &self, + /// Collects the ids and names the substitution defines, the `(source, + /// target)` pairs it writes under them, and how many links it writes + /// without an id. + fn collect_definitions( pattern: &LinoLink, plan: &mut LinkReferencePlan, - reserved_numeric_ids: &mut HashSet, + anonymous_links: &mut usize, ) { - if Self::is_composite_lino(pattern) { - if let Some(identifier) = Self::concrete_identifier(pattern.id.as_deref()) { - if let Ok(link_id) = identifier.parse::() { - plan.numeric_ids_to_be_created.insert(link_id); - reserved_numeric_ids.insert(link_id); - } else { - plan.names_to_be_created.insert(identifier); - } - } - } - if let Some(values) = &pattern.values { for sub_pattern in values { - self.collect_explicit_definitions(sub_pattern, plan, reserved_numeric_ids); - } - } - } - - fn collect_composite_pairs(pattern: &LinoLink, plan: &mut LinkReferencePlan) { - if Self::is_composite_lino(pattern) - && Self::concrete_identifier(pattern.id.as_deref()).is_some() - { - if let Some(values) = &pattern.values { - if let (Some(source), Some(target)) = ( - Self::concrete_numeric_identifier(values[0].id.as_deref()), - Self::concrete_numeric_identifier(values[1].id.as_deref()), - ) { - plan.composite_pairs_to_be_created.insert((source, target)); - } + Self::collect_definitions(sub_pattern, plan, anonymous_links); } } - - if let Some(values) = &pattern.values { - for sub_pattern in values { - Self::collect_composite_pairs(sub_pattern, plan); - } - } - } - - fn collect_implicit_definitions( - &self, - storage: &mut impl NamedTypeLinks, - pattern: &LinoLink, - plan: &mut LinkReferencePlan, - reserved_numeric_ids: &mut HashSet, - ) { - if let Some(values) = &pattern.values { - for sub_pattern in values { - self.collect_implicit_definitions(storage, sub_pattern, plan, reserved_numeric_ids); - } + if !Self::is_composite_lino(pattern) { + return; } - - if Self::is_composite_lino(pattern) - && Self::concrete_identifier(pattern.id.as_deref()).is_none() - { - let next_id = Self::next_available_link_id(storage, reserved_numeric_ids); - reserved_numeric_ids.insert(next_id); - plan.numeric_ids_to_be_created.insert(next_id); + let Some(identifier) = Self::concrete_identifier(pattern.id.as_deref()) else { + *anonymous_links += 1; + return; + }; + match identifier.parse::() { + Ok(link_id) => plan.numeric_ids_to_be_created.insert(link_id), + Err(_) => plan.names_to_be_created.insert(identifier), + }; + let values = pattern.values.as_deref().unwrap_or_default(); + if let (Some(source), Some(target)) = ( + Self::concrete_numeric_identifier(values[0].id.as_deref()), + Self::concrete_numeric_identifier(values[1].id.as_deref()), + ) { + plan.composite_pairs_to_be_created.insert((source, target)); } } - fn next_available_link_id( - storage: &mut impl NamedTypeLinks, - reserved_numeric_ids: &HashSet, - ) -> u32 { - let mut next_id = 1; - while storage.exists(next_id) || reserved_numeric_ids.contains(&next_id) { - next_id += 1; + /// The addresses the next `count` links created in `storage` get. + /// + /// A store reuses the address freed last first, so the lowest free one is + /// not necessarily next. Creating the links and deleting them again in + /// reverse leaves the store exactly as it was, and is the one way to ask + /// any store, a remote one included. + fn next_created_addresses(storage: &mut impl NamedTypeLinks, count: usize) -> Result> { + let addresses: Vec = (0..count).map(|_| storage.create(0, 0)).collect(); + for &address in addresses.iter().rev() { + storage.delete(address)?; } - next_id + Ok(addresses) } fn collect_missing_references( @@ -314,30 +285,19 @@ impl LinkReferenceValidator { Ok(()) } - /// Creates every missing reference and reports the `(before, after)` state - /// of each one. - /// - /// The before state is the placeholder the reference is turned into a point - /// link *from*, never `null`: both branches of the C# original create the - /// link silently — `EnsureCreated`, and `CreateAndUpdate(Null, Null)` with - /// no handler — and only pass the changes handler to the `Update` that - /// makes it a point link: + /// Creates every missing reference as a point link and reports each + /// creation the way it happens: a numeric reference is first filled in as + /// the empty `(id: 0 0)` and then pointed at itself, a named one is created + /// as `(name: name name)` directly. The simplified `--changes` report of + /// both is the creation of the point link, `() ((2: 2 2))`. /// - /// ```csharp - /// links.Update( - /// new DoubletLink(linkId, links.Constants.Null, links.Constants.Null), - /// new DoubletLink(linkId, linkId, linkId), - /// (beforeState, afterState) => - /// options.ChangesHandler?.Invoke(beforeState, afterState) ?? links.Constants.Continue - /// ); - /// ``` - /// - /// So `--changes` shows `((2: 0 0)) ((2: 2 2))`, not `() ((2: 2 2))`. + /// A numeric reference the substitution defines as `(id: id id)` itself + /// is only filled in: the query writes the point link. fn auto_create_missing_references( &self, storage: &mut impl NamedTypeLinks, plan: &LinkReferencePlan, - ) -> Result> { + ) -> Result { let missing_references = &plan.missing_references; let mut created = Vec::new(); let mut numeric_references = missing_references @@ -356,6 +316,8 @@ impl LinkReferenceValidator { "[ValidateLinksExistOrWillBeCreated] Auto-creating missing numeric reference {link_id}." )); storage.try_ensure_created(link_id)?; + let placeholder = Link::new(link_id, 0, 0); + created.push((None, Some(placeholder))); if plan .composite_pairs_to_be_created .contains(&(link_id, link_id)) @@ -365,13 +327,8 @@ impl LinkReferenceValidator { )); continue; } - let before = storage - .get_link(link_id) - .unwrap_or_else(|| Link::new(link_id, 0, 0)); storage.update(link_id, link_id, link_id)?; - if let Some(after) = storage.get_link(link_id) { - created.push((before, after)); - } + created.push((Some(placeholder), storage.get_link(link_id))); } let mut named_references = missing_references @@ -391,9 +348,7 @@ impl LinkReferenceValidator { "[ValidateLinksExistOrWillBeCreated] Auto-creating missing named reference '{name}' as point link." )); let link_id = storage.get_or_create_named(&name)?; - if let Some(after) = storage.get_link(link_id) { - created.push((Link::new(link_id, 0, 0), after)); - } + created.push((None, storage.get_link(link_id))); } Ok(created) diff --git a/rust/src/query_processor.rs b/rust/src/query_processor.rs index 3fe52ff..9e9f589 100644 --- a/rust/src/query_processor.rs +++ b/rust/src/query_processor.rs @@ -20,6 +20,10 @@ mod matching; // Write-side operations live in a submodule; see query_processor/mutations.rs. mod mutations; +/// The `(before, after)` states a query reports, where `None` is no link: +/// `(None, Some(link))` is a creation and `(Some(link), None)` a deletion. +pub type Changes = Vec<(Option, Option)>; + /// QueryProcessor handles LiNo query parsing and execution /// Corresponds to AdvancedMixedQueryProcessor in C# pub struct QueryProcessor { @@ -50,22 +54,14 @@ impl QueryProcessor { /// collects the raw handler calls and runs `SimplifyChanges` over them once /// in `Program.cs` regardless of which branch of the processor produced /// them. - pub fn process_query( - &self, - storage: &mut impl NamedTypeLinks, - query: &str, - ) -> Result, Option)>> { + pub fn process_query(&self, storage: &mut impl NamedTypeLinks, query: &str) -> Result { let changes = self.process_query_raw(storage, query)?; Ok(self.simplify_changes_list(&changes)) } /// The processor proper: applies `query` and reports the raw /// `(before, after)` states, in the order they happened. - fn process_query_raw( - &self, - storage: &mut impl NamedTypeLinks, - query: &str, - ) -> Result, Option)>> { + fn process_query_raw(&self, storage: &mut impl NamedTypeLinks, query: &str) -> Result { self.trace_msg(&format!("[ProcessQuery] Query: \"{}\"", query)); let query = query.trim(); @@ -125,11 +121,11 @@ impl QueryProcessor { "[ProcessQuery] No restriction, but substitution is non-empty => creation scenario.", ); if let Some(values) = &substitution_link.values { - changes_list.extend( - self.validate_links_exist_or_will_be_created(storage, &[], values)? - .into_iter() - .map(|(before, after)| (Some(before), Some(after))), - ); + changes_list.extend(self.validate_links_exist_or_will_be_created( + storage, + &[], + values, + )?); for link_to_create in values { let created_id = @@ -150,11 +146,11 @@ impl QueryProcessor { "[ProcessQuery] Restriction non-empty, substitution empty => deletion scenario.", ); let restriction_values = restriction_link.values.as_deref().unwrap_or(&[]); - changes_list.extend( - self.validate_links_exist_or_will_be_created(storage, restriction_values, &[])? - .into_iter() - .map(|(before, after)| (Some(before), Some(after))), - ); + changes_list.extend(self.validate_links_exist_or_will_be_created( + storage, + restriction_values, + &[], + )?); let restriction_patterns = self.patterns_from_lino(restriction_link); let mut links_to_delete = Vec::new(); @@ -183,15 +179,11 @@ impl QueryProcessor { let substitution_patterns = self.patterns_from_lino(substitution_link); let restriction_values = restriction_link.values.as_deref().unwrap_or(&[]); let substitution_values = substitution_link.values.as_deref().unwrap_or(&[]); - changes_list.extend( - self.validate_links_exist_or_will_be_created( - storage, - restriction_values, - substitution_values, - )? - .into_iter() - .map(|(before, after)| (Some(before), Some(after))), - ); + changes_list.extend(self.validate_links_exist_or_will_be_created( + storage, + restriction_values, + substitution_values, + )?); let solutions = self.find_all_solutions(storage, &restriction_patterns)?; if solutions.is_empty() { @@ -230,10 +222,20 @@ impl QueryProcessor { let mut all_planned_operations = Vec::new(); for solution in &solutions { - let restriction_links = - self.resolve_patterns(storage, &restriction_patterns, solution, false)?; - let substitution_links = - self.resolve_patterns(storage, &substitution_patterns, solution, true)?; + let restriction_links = self.resolve_patterns( + storage, + &restriction_patterns, + solution, + false, + &mut changes_list, + )?; + let substitution_links = self.resolve_patterns( + storage, + &substitution_patterns, + solution, + true, + &mut changes_list, + )?; all_planned_operations .extend(self.determine_operations(&restriction_links, &substitution_links)); } @@ -256,7 +258,7 @@ impl QueryProcessor { storage: &mut impl NamedTypeLinks, restriction_patterns: &[LinoLink], substitution_patterns: &[LinoLink], - ) -> Result> { + ) -> Result { LinkReferenceValidator::new(self.trace, self.auto_create_missing_references) .validate_links_exist_or_will_be_created( storage, @@ -422,6 +424,7 @@ impl QueryProcessor { patterns: &[Pattern], solution: &HashMap, is_substitution: bool, + changes: &mut Changes, ) -> Result> { let mut working_solution = solution.clone(); let mut visited_indexes = HashSet::new(); @@ -433,6 +436,7 @@ impl QueryProcessor { &mut working_solution, is_substitution, &mut visited_indexes, + changes, )?); } Ok(resolved) @@ -445,6 +449,7 @@ impl QueryProcessor { solution: &mut HashMap, is_substitution: bool, visited_indexes: &mut HashSet, + changes: &mut Changes, ) -> Result { if pattern.is_leaf() { let index = self.resolve_identifier( @@ -453,6 +458,7 @@ impl QueryProcessor { solution, if is_substitution { 0 } else { u32::MAX }, is_substitution, + changes, )?; return Ok(ResolvedLink::new(index, u32::MAX, u32::MAX, None)); } @@ -464,6 +470,7 @@ impl QueryProcessor { solution, is_substitution, visited_indexes, + changes, )? .index; let mut target = self @@ -473,11 +480,18 @@ impl QueryProcessor { solution, is_substitution, visited_indexes, + changes, )? .index; let default_index = if is_substitution { 0 } else { u32::MAX }; - let mut index = - self.resolve_identifier(storage, &pattern.index, solution, default_index, false)?; + let mut index = self.resolve_identifier( + storage, + &pattern.index, + solution, + default_index, + false, + changes, + )?; let mut name = None; if is_substitution @@ -515,6 +529,7 @@ impl QueryProcessor { solution: &HashMap, default_value: u32, create_named_leaf: bool, + changes: &mut Changes, ) -> Result { if identifier.is_empty() { return Ok(default_value); @@ -531,13 +546,10 @@ impl QueryProcessor { if let Ok(parsed) = identifier.parse::() { return Ok(parsed); } - if let Some(named_id) = storage.get_by_name(identifier)? { - return Ok(named_id); - } if create_named_leaf { - return storage.get_or_create_named(identifier); + return self.ensure_named_point_link(storage, identifier, changes); } - Ok(default_value) + Ok(storage.get_by_name(identifier)?.unwrap_or(default_value)) } fn determine_operations( @@ -608,7 +620,7 @@ impl QueryProcessor { storage: &mut impl NamedTypeLinks, before: Option, after: Option, - changes: &mut Vec<(Option, Option)>, + changes: &mut Changes, ) -> Result<()> { match (before, after) { (Some(before), None) => { @@ -752,10 +764,7 @@ impl QueryProcessor { /// around the simplifier (as this used to) both reported creations and /// deletions in a different order than C# and hid them from the chain /// collapsing that is the whole point of the pass. - fn simplify_changes_list( - &self, - changes: &[(Option, Option)], - ) -> Vec<(Option, Option)> { + fn simplify_changes_list(&self, changes: &[(Option, Option)]) -> Changes { let to_simplify: Vec<(Link, Link)> = changes .iter() .map(|(before, after)| { diff --git a/rust/src/query_processor/mutations.rs b/rust/src/query_processor/mutations.rs index 9bc749b..1e44c5a 100644 --- a/rust/src/query_processor/mutations.rs +++ b/rust/src/query_processor/mutations.rs @@ -15,7 +15,7 @@ use crate::lino_link::LinoLink; use crate::named_type_links::NamedTypeLinks; use crate::query_types::ResolvedLink; -use super::QueryProcessor; +use super::{Changes, QueryProcessor}; /// A store change as `--changes` reports it: a null link is no link at all. fn change(before: Link, after: Link) -> (Option, Option) { @@ -41,7 +41,7 @@ impl QueryProcessor { &self, storage: &mut impl NamedTypeLinks, id: u32, - changes: &mut Vec<(Option, Option)>, + changes: &mut Changes, ) -> Result { storage.delete_observed(id, &mut |before, after| changes.push(change(before, after))) } @@ -57,7 +57,7 @@ impl QueryProcessor { id: u32, source: u32, target: u32, - changes: &mut Vec<(Option, Option)>, + changes: &mut Changes, ) -> Result { storage.update_observed(id, source, target, &mut |before, after| { changes.push(change(before, after)) @@ -111,7 +111,7 @@ impl QueryProcessor { &self, storage: &mut impl NamedTypeLinks, intended_final_states: &[(u32, Option)], - changes: &mut Vec<(Option, Option)>, + changes: &mut Changes, ) -> Result<()> { for (index, intended) in intended_final_states { let Some(intended) = intended else { @@ -134,12 +134,10 @@ impl QueryProcessor { /// Creates or updates the link a resolved definition asks for and appends /// the changes a `--changes` listener would see. /// - /// Only a link that genuinely had to be allocated from nothing is reported - /// as a creation. An address filled in with - /// [`try_ensure_created`](NamedTypeLinks::try_ensure_created) reports the - /// update from its empty state, and a definition that already matches its - /// stored state — or a duplicate of an existing doublet — reports that - /// state unchanged, mirroring `CreateOrUpdateLink` in the C# processor: + /// An address the store lacks is reported as created empty and then + /// updated, and a definition that already matches its stored state — or a + /// duplicate of an existing doublet — reports that state unchanged, + /// mirroring `CreateOrUpdateLink` in the C# processor: /// /// ```csharp /// if (existingDoublet.Source != linkDefinition.Source || existingDoublet.Target != linkDefinition.Target) @@ -157,10 +155,10 @@ impl QueryProcessor { &self, storage: &mut impl NamedTypeLinks, definition: &ResolvedLink, - changes: &mut Vec<(Option, Option)>, + changes: &mut Changes, ) -> Result { let id = if Self::is_normal_index(definition.index) { - storage.try_ensure_created(definition.index)?; + self.ensure_address(storage, definition.index, changes)?; let existing = storage .get_link(definition.index) .unwrap_or_else(|| Link::new(definition.index, 0, 0)); @@ -214,13 +212,13 @@ impl QueryProcessor { /// doublet it touches — including the nested ones — appends its /// `(before, after)` states to `changes`, exactly as the C# version reports /// them through `options.ChangesHandler`, so `--changes` lists the same - /// records in both languages. Leaves report nothing: C#'s `ResolveLeaf` - /// passes the update handler that ignores the changes handler. + /// records in both languages. A named leaf created on the way is reported + /// as created too. pub(super) fn ensure_link_created( &self, storage: &mut impl NamedTypeLinks, lino_link: &LinoLink, - changes: &mut Vec<(Option, Option)>, + changes: &mut Changes, ) -> Result { // Handle leaf nodes (names or numbers) if !lino_link.has_values() { @@ -234,8 +232,7 @@ impl QueryProcessor { return Ok(num); } - // It's a name - get or create - return storage.get_or_create_named(id); + return self.ensure_named_point_link(storage, id, changes); } return Ok(0); } @@ -255,23 +252,14 @@ impl QueryProcessor { self.ensure_indexed_link(storage, num, source_id, target_id, changes)? } else if id == "*" || Self::is_variable(id) { self.ensure_doublet(storage, source_id, target_id, changes) + } else if let Some(named) = storage.get_by_name(id)? { + self.ensure_indexed_link(storage, named, source_id, target_id, changes)? } else { - // Named link: this repository resolves the address through - // the name, where C# resolves it through `(source, target)` - // and names the result afterwards. The reported states are - // the same either way. - let existing = storage.get_by_name(id)?; - if let Some(id_num) = existing { - self.ensure_indexed_link(storage, id_num, source_id, target_id, changes)? - } else { - let new_id = storage.create( - Self::resolve_unspecified(source_id, 0), - Self::resolve_unspecified(target_id, 0), - ); - changes.push((None, storage.get_link(new_id))); - storage.set_name(new_id, id)?; - new_id - } + // A doublet is unique, so a new name for an existing + // `(source, target)` names that link instead of copying it. + let named = self.ensure_doublet(storage, source_id, target_id, changes); + storage.set_name(named, id)?; + named } } else { // Anonymous link @@ -304,9 +292,9 @@ impl QueryProcessor { index: u32, source: u32, target: u32, - changes: &mut Vec<(Option, Option)>, + changes: &mut Changes, ) -> Result { - storage.try_ensure_created(index)?; + self.ensure_address(storage, index, changes)?; let stored = storage .get_link(index) .unwrap_or_else(|| Link::new(index, 0, 0)); @@ -335,7 +323,7 @@ impl QueryProcessor { storage: &mut impl NamedTypeLinks, source: u32, target: u32, - changes: &mut Vec<(Option, Option)>, + changes: &mut Changes, ) -> u32 { if let Some(existing_id) = Self::search_unspecified(storage, source, target) { self.trace_msg(&format!( @@ -357,4 +345,35 @@ impl QueryProcessor { created } } + + /// Fills `index` in when the store lacks it, reporting the empty + /// `(index: 0 0)` link that creates. + fn ensure_address( + &self, + storage: &mut impl NamedTypeLinks, + index: u32, + changes: &mut Changes, + ) -> Result<()> { + if !storage.exists(index) { + storage.try_ensure_created(index)?; + changes.push((None, Some(Link::new(index, 0, 0)))); + } + Ok(()) + } + + /// The link called `name`, created and reported as the point link + /// `(name: name name)` when there is none yet. + pub(super) fn ensure_named_point_link( + &self, + storage: &mut impl NamedTypeLinks, + name: &str, + changes: &mut Changes, + ) -> Result { + if let Some(named) = storage.get_by_name(name)? { + return Ok(named); + } + let created = storage.get_or_create_named(name)?; + changes.push((None, storage.get_link(created))); + Ok(created) + } } diff --git a/rust/tests/protocol_tcp_tests.rs b/rust/tests/protocol_tcp_tests.rs index 778ff47..e8da460 100644 --- a/rust/tests/protocol_tcp_tests.rs +++ b/rust/tests/protocol_tcp_tests.rs @@ -157,7 +157,7 @@ fn concurrent_clients_share_one_store() { let name = format!("w{worker}i{item}"); let reply = client.query_text(&format!("() (({name}: {name} {name}))")); let link = format!("({name}: {name} {name})"); - assert_eq!(reply.unwrap(), format!("({link}) ({link})")); + assert_eq!(reply.unwrap(), format!("() ({link})")); } }) }) diff --git a/rust/tests/query_processor_change_report_tests.rs b/rust/tests/query_processor_change_report_tests.rs new file mode 100644 index 0000000..8bd1ceb --- /dev/null +++ b/rust/tests/query_processor_change_report_tests.rs @@ -0,0 +1,167 @@ +//! A query reports every link it creates, names or redefines, so the +//! `--changes` output tells the whole truth about what the store holds now. + +use anyhow::Result; +use link_cli::{Changes, Link, NamedTypeLinks, NamedTypesDecorator, QueryProcessor}; +use tempfile::NamedTempFile; + +struct Store { + links: NamedTypesDecorator, + processor: QueryProcessor, + _files: (NamedTempFile, NamedTempFile), +} + +impl Store { + fn new() -> Result { + let database = NamedTempFile::new()?; + let names = NamedTempFile::new()?; + let links = + NamedTypesDecorator::with_names_database_path(database.path(), names.path(), false)?; + let processor = QueryProcessor::new(false).with_auto_create_missing_references(true); + Ok(Self { + links, + processor, + _files: (database, names), + }) + } + + fn query(&mut self, query: &str) -> Result { + Ok(sorted( + self.processor.process_query(&mut self.links, query)?, + )) + } + + fn named(&mut self, name: &str) -> Result { + Ok(self + .links + .get_by_name(name)? + .unwrap_or_else(|| panic!("{name} should exist"))) + } + + fn point(&mut self, name: &str) -> Result { + let index = self.named(name)?; + Ok(Link::new(index, index, index)) + } +} + +fn sorted(mut changes: Changes) -> Changes { + changes.sort_by_key(|(before, after)| (before.map(|l| l.index), after.map(|l| l.index))); + changes +} + +fn created(link: Link) -> (Option, Option) { + (None, Some(link)) +} + +#[test] +fn a_numeric_point_link_is_reported_created() -> Result<()> { + let mut store = Store::new()?; + assert_eq!( + store.query("() ((2: 2 2))")?, + vec![created(Link::new(2, 2, 2))] + ); + Ok(()) +} + +#[test] +fn a_named_point_link_is_reported_created() -> Result<()> { + let mut store = Store::new()?; + let changes = store.query("() ((a: a a))")?; + assert_eq!(changes, vec![created(store.point("a")?)]); + Ok(()) +} + +#[test] +fn a_named_leaf_created_on_the_way_is_reported_created() -> Result<()> { + let mut store = Store::new()?; + let changes = store.query("() ((c: c d))")?; + let (c, d) = (store.named("c")?, store.named("d")?); + assert_eq!( + changes, + sorted(vec![ + created(Link::new(c, c, d)), + created(Link::new(d, d, d)) + ]) + ); + Ok(()) +} + +#[test] +fn every_named_leaf_of_a_composite_is_reported_created() -> Result<()> { + let mut store = Store::new()?; + let changes = store.query("() ((child: father mother))")?; + let (child, father, mother) = ( + store.named("child")?, + store.named("father")?, + store.named("mother")?, + ); + assert_eq!( + changes, + sorted(vec![ + created(Link::new(child, father, mother)), + created(Link::new(father, father, father)), + created(Link::new(mother, mother, mother)), + ]) + ); + assert_eq!(store.links.all_links().len(), 3); + Ok(()) +} + +#[test] +fn creating_an_existing_name_redefines_that_link() -> Result<()> { + let mut store = Store::new()?; + store.query("() ((child: father mother))")?; + let (child, father, mother) = ( + store.named("child")?, + store.named("father")?, + store.named("mother")?, + ); + + assert_eq!( + store.query("() ((child: mother father))")?, + vec![( + Some(Link::new(child, father, mother)), + Some(Link::new(child, mother, father)) + )] + ); + assert_eq!(store.links.all_links().len(), 3); + Ok(()) +} + +#[test] +fn a_new_name_for_an_existing_doublet_names_it_instead_of_copying_it() -> Result<()> { + let mut store = Store::new()?; + store.query("() ((a: a a))")?; + let a = store.point("a")?; + + assert_eq!(store.query("() ((b: a a))")?, vec![(Some(a), Some(a))]); + assert_eq!(store.named("b")?, a.index); + assert_eq!(store.links.all_links(), vec![a]); + Ok(()) +} + +#[test] +fn a_numeric_reference_is_reported_created_as_the_point_link() -> Result<()> { + let mut store = Store::new()?; + assert_eq!( + store.query("() ((3: 3 2))")?, + vec![created(Link::new(2, 2, 2)), created(Link::new(3, 3, 2))] + ); + Ok(()) +} + +/// Doublets are unique, so the point link `(2: 2 2)` and the defined +/// `(3: 2 2)` cannot both exist: the reference is left empty. +#[test] +fn a_reference_to_the_pair_a_link_defines_is_reported_created_empty() -> Result<()> { + let mut store = Store::new()?; + assert_eq!( + store.query("() ((3: 2 2))")?, + vec![created(Link::new(2, 0, 0)), created(Link::new(3, 2, 2))] + ); + assert_eq!( + store.links.all_links(), + vec![Link::new(2, 0, 0), Link::new(3, 2, 2)] + ); + Ok(()) +} diff --git a/rust/tests/query_processor_tests.rs b/rust/tests/query_processor_tests.rs index ce834d4..cb258bd 100644 --- a/rust/tests/query_processor_tests.rs +++ b/rust/tests/query_processor_tests.rs @@ -1,7 +1,7 @@ //! Tests for the QueryProcessor module use anyhow::Result; -use link_cli::{LinkStorage, QueryProcessor}; +use link_cli::{Link, LinkStorage, QueryProcessor}; use tempfile::NamedTempFile; fn auto_processor() -> QueryProcessor { @@ -123,6 +123,60 @@ fn test_auto_create_missing_numeric_reference_fills_existing_gap() -> Result<()> Ok(()) } +/// Links 1 to 4, with 2 and then 3 deleted: the store hands out 3, the +/// address freed last, before 2. +fn store_with_2_and_3_freed(storage: &mut LinkStorage) -> Result<()> { + let processor = QueryProcessor::new(false); + processor.process_query(storage, "() ((1 1) (2 2) (3 3) (4 4))")?; + processor.process_query(storage, "((2: 2 2)) ()")?; + processor.process_query(storage, "((3: 3 3)) ()")?; + Ok(()) +} + +#[test] +fn test_reference_to_a_freed_address_the_new_link_does_not_get_fails() -> Result<()> { + let temp_file = NamedTempFile::new()?; + let mut storage = LinkStorage::new(temp_file.path().to_str().unwrap(), false)?; + store_with_2_and_3_freed(&mut storage)?; + + let error = QueryProcessor::new(false) + .process_query(&mut storage, "() ((2 2))") + .expect_err("the new link gets 3, so nothing creates 2"); + + assert!(error.to_string().contains("'2'"), "{error}"); + assert_eq!(storage.all().len(), 2); + Ok(()) +} + +#[test] +fn test_auto_create_a_reference_to_a_freed_address_the_new_link_does_not_get() -> Result<()> { + let temp_file = NamedTempFile::new()?; + let mut storage = LinkStorage::new(temp_file.path().to_str().unwrap(), false)?; + store_with_2_and_3_freed(&mut storage)?; + + auto_processor().process_query(&mut storage, "() ((2 2))")?; + + assert_eq!(storage.get(2), Some(&Link::new(2, 2, 2))); + assert_eq!(storage.all().len(), 3); + Ok(()) +} + +#[test] +fn test_auto_create_a_reference_the_new_link_would_have_got_before() -> Result<()> { + let temp_file = NamedTempFile::new()?; + let mut storage = LinkStorage::new(temp_file.path().to_str().unwrap(), false)?; + QueryProcessor::new(false).process_query(&mut storage, "() ((1 1) (2 2) (3 3) (4 4))")?; + + // Creating 7 frees 5 and 6 on the way, so `(5 7)` gets 6, not 5. + auto_processor().process_query(&mut storage, "() ((5 7))")?; + + assert_eq!(storage.get(5), Some(&Link::new(5, 5, 5))); + assert_eq!(storage.get(6), Some(&Link::new(6, 5, 7))); + assert_eq!(storage.get(7), Some(&Link::new(7, 7, 7))); + assert_eq!(storage.all().len(), 7); + Ok(()) +} + #[test] fn test_missing_named_reference_fails_without_auto_create() -> Result<()> { let temp_file = NamedTempFile::new()?; From 1f6fc901f3ca1c65f6afe27e6368790944f41ab7 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 00:39:17 +0000 Subject: [PATCH 20/44] fix(csharp): report a query clink cannot run as one error line, not a stack trace A missing reference or a malformed query ended clink with an unhandled exception and its stack trace. Both are now printed as 'Error: Query error:' or 'Error: Parse error:' like the Rust clink, with the stack trace kept for --trace. Tests in both ports pin the output, and the CLI tests share one helper that runs the built clink. --- .../.changeset/issue-105-creation-reports.md | 2 + .../CliErrorReportingTests.cs | 59 ++++++++++ .../CliExportIntegrationTests.cs | 107 +++--------------- .../Clink.cs | 49 ++++++++ .../Protocol/CliTcpIntegrationTests.cs | 38 +------ .../Foundation.Data.Doublets.Cli/Program.cs | 16 ++- rust/tests/cli_error_reporting_tests.rs | 65 +++++++++++ 7 files changed, 212 insertions(+), 124 deletions(-) create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/CliErrorReportingTests.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/Clink.cs create mode 100644 rust/tests/cli_error_reporting_tests.rs diff --git a/csharp/.changeset/issue-105-creation-reports.md b/csharp/.changeset/issue-105-creation-reports.md index 48c9c2b..d57141b 100644 --- a/csharp/.changeset/issue-105-creation-reports.md +++ b/csharp/.changeset/issue-105-creation-reports.md @@ -8,3 +8,5 @@ Fixed the `--changes` report of creations: - Creating a link under an existing name redefines that link in place, and a new name for an existing doublet names it instead of copying it. - Reference validation predicts the addresses new links get by asking the store. Stores reuse the address freed last first, so the lowest free address is not always next. A reference to a freed address the query does not refill is now reported missing instead of being dropped. - `EnsureCreated` no longer throws when the store hands out a freed address above the target before reaching it. + +A query that refers to a missing link or does not parse is reported in one `Error: Query error: ...` or `Error: Parse error: ...` line on stderr, as the Rust clink reports it, instead of an unhandled exception with its stack trace. `--trace` still prints the stack trace. diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/CliErrorReportingTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/CliErrorReportingTests.cs new file mode 100644 index 0000000..b7a1192 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/CliErrorReportingTests.cs @@ -0,0 +1,59 @@ +namespace Foundation.Data.Doublets.Cli.Tests; + +/// A query clink cannot run is reported in one line on stderr, the way the Rust clink reports it. +public sealed class CliErrorReportingTests : IDisposable +{ + private readonly string _directory = Directory.CreateTempSubdirectory("clink-errors-").FullName; + + public void Dispose() => Directory.Delete(_directory, recursive: true); + + private Task RunQueryAsync(params string[] arguments) => + Clink.RunAsync(new[] { "--db", Path.Combine(_directory, "db.links") }.Concat(arguments).ToArray()); + + [Fact] + public async Task AMissingReferenceIsAQueryErrorWithoutAStackTrace() + { + var result = await RunQueryAsync("() ((2 2))"); + + Assert.Equal(1, result.ExitCode); + Assert.Equal( + "Error: Query error: Invalid reference to non-existent link '2' in substitution pattern. " + + "Link '2' does not exist and will not be created by this operation. " + + "Use --auto-create-missing-references to create missing references as point links.\n", + result.Stderr); + Assert.Equal("", result.Stdout); + } + + [Fact] + public async Task AMalformedQueryIsAParseErrorPointingAtTheProblem() + { + var result = await RunQueryAsync("((("); + + Assert.Equal(1, result.ExitCode); + Assert.StartsWith("Error: Parse error: Syntax error at line 1, column 4", result.Stderr, StringComparison.Ordinal); + Assert.Contains("1 | (((\n | ^", result.Stderr, StringComparison.Ordinal); + Assert.DoesNotContain(" at ", result.Stderr, StringComparison.Ordinal); + } + + [Fact] + public async Task AFailedQueryLeavesTheStoreUnchanged() + { + Clink.AssertSucceeded(await RunQueryAsync("() ((1 1))")); + + Assert.Equal(1, (await RunQueryAsync("() ((1 3))")).ExitCode); + + var after = await RunQueryAsync("--after"); + Clink.AssertSucceeded(after); + Assert.Equal("(1: 1 1)\n", after.Stdout); + } + + [Fact] + public async Task TraceKeepsTheStackTraceForDiagnosis() + { + var result = await RunQueryAsync("--trace", "() ((2 2))"); + + Assert.Equal(1, result.ExitCode); + Assert.Contains("Error: Query error: Invalid reference to non-existent link '2'", result.Stderr, StringComparison.Ordinal); + Assert.Contains(" at ", result.Stderr, StringComparison.Ordinal); + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/CliExportIntegrationTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/CliExportIntegrationTests.cs index 01e6347..7915a3a 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/CliExportIntegrationTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/CliExportIntegrationTests.cs @@ -1,5 +1,3 @@ -using System.Diagnostics; - namespace Foundation.Data.Doublets.Cli.Tests; public class CliExportIntegrationTests @@ -14,9 +12,9 @@ public async Task ExportAlias_WritesNumberedReferences() var dbPath = Path.Combine(tempDirectory, "numbered.links"); var outputPath = Path.Combine(tempDirectory, "numbered.lino"); - var result = await RunClinkAsync("--db", dbPath, "() ((1 1) (2 2))", "--export", outputPath); + var result = await Clink.RunAsync("--db", dbPath, "() ((1 1) (2 2))", "--export", outputPath); - AssertClinkSucceeded(result); + Clink.AssertSucceeded(result); Assert.Equal(new[] { "(1: 1 1)", "(2: 2 2)" }, File.ReadAllLines(outputPath)); } finally @@ -35,7 +33,7 @@ public async Task ExportAlias_WritesNamedReferences() var dbPath = Path.Combine(tempDirectory, "named.links"); var outputPath = Path.Combine(tempDirectory, "named.lino"); - var result = await RunClinkAsync( + var result = await Clink.RunAsync( "--db", dbPath, "--auto-create-missing-references", @@ -43,7 +41,7 @@ public async Task ExportAlias_WritesNamedReferences() "--export", outputPath); - AssertClinkSucceeded(result); + Clink.AssertSucceeded(result); Assert.Equal( new[] { "(father: father father)", "(mother: mother mother)", "(child: father mother)" }, File.ReadAllLines(outputPath)); @@ -63,15 +61,15 @@ public async Task StructureOption_RendersLeftBranchWithIndexes() { var dbPath = Path.Combine(tempDirectory, "structure.links"); - AssertClinkSucceeded(await RunClinkAsync("--db", dbPath, "() ((1: 1 1))")); - AssertClinkSucceeded(await RunClinkAsync("--db", dbPath, "() ((2: 1 2))")); - AssertClinkSucceeded(await RunClinkAsync("--db", dbPath, "() ((3: 2 1))")); - AssertClinkSucceeded(await RunClinkAsync("--db", dbPath, "() ((4: 3 2))")); + Clink.AssertSucceeded(await Clink.RunAsync("--db", dbPath, "() ((1: 1 1))")); + Clink.AssertSucceeded(await Clink.RunAsync("--db", dbPath, "() ((2: 1 2))")); + Clink.AssertSucceeded(await Clink.RunAsync("--db", dbPath, "() ((3: 2 1))")); + Clink.AssertSucceeded(await Clink.RunAsync("--db", dbPath, "() ((4: 3 2))")); - var result = await RunClinkAsync("--db", dbPath, "--structure", "4"); + var result = await Clink.RunAsync("--db", dbPath, "--structure", "4"); - AssertClinkSucceeded(result); - Assert.Equal("(4: (3: (2: (1: 1 1) 2) 1) 2)\n", NormalizeNewlines(result.Stdout)); + Clink.AssertSucceeded(result); + Assert.Equal("(4: (3: (2: (1: 1 1) 2) 1) 2)\n", result.Stdout); } finally { @@ -96,9 +94,9 @@ await File.WriteAllLinesAsync(inputPath, new[] "(3: 2 1)" }); - var result = await RunClinkAsync("--db", dbPath, "--import", inputPath, "--export", outputPath); + var result = await Clink.RunAsync("--db", dbPath, "--import", inputPath, "--export", outputPath); - AssertClinkSucceeded(result); + Clink.AssertSucceeded(result); Assert.Equal(new[] { "(1: 1 1)", @@ -123,7 +121,7 @@ public async Task AlwaysTriggerOption_StoresTriggerAndAppliesItOnLaterChange() var triggersPath = Path.Combine(tempDirectory, "triggers.links"); var outputPath = Path.Combine(tempDirectory, "triggered.lino"); - AssertClinkSucceeded(await RunClinkAsync( + Clink.AssertSucceeded(await Clink.RunAsync( "--db", dbPath, "--triggers-file", @@ -131,7 +129,7 @@ public async Task AlwaysTriggerOption_StoresTriggerAndAppliesItOnLaterChange() "--always", "(((1: 1 1)) ((1: 1 2)))")); - var result = await RunClinkAsync( + var result = await Clink.RunAsync( "--db", dbPath, "--triggers-file", @@ -141,7 +139,7 @@ public async Task AlwaysTriggerOption_StoresTriggerAndAppliesItOnLaterChange() "--export", outputPath); - AssertClinkSucceeded(result); + Clink.AssertSucceeded(result); Assert.Equal(new[] { "(1: 1 2)", "(2: 2 2)" }, File.ReadAllLines(outputPath)); } finally @@ -150,83 +148,10 @@ public async Task AlwaysTriggerOption_StoresTriggerAndAppliesItOnLaterChange() } } - private static async Task RunClinkAsync(params string[] clinkArguments) - { - var csharpDirectory = FindCsharpDirectory(); - var projectPath = Path.Combine(csharpDirectory, "Foundation.Data.Doublets.Cli", "Foundation.Data.Doublets.Cli.csproj"); - - using var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = "dotnet", - WorkingDirectory = csharpDirectory, - RedirectStandardOutput = true, - RedirectStandardError = true - } - }; - - process.StartInfo.ArgumentList.Add("run"); - process.StartInfo.ArgumentList.Add("--project"); - process.StartInfo.ArgumentList.Add(projectPath); - process.StartInfo.ArgumentList.Add("--"); - foreach (var argument in clinkArguments) - { - process.StartInfo.ArgumentList.Add(argument); - } - - process.Start(); - var stdout = process.StandardOutput.ReadToEndAsync(); - var stderr = process.StandardError.ReadToEndAsync(); - - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60)); - try - { - await process.WaitForExitAsync(cts.Token); - } - catch (OperationCanceledException) - { - process.Kill(entireProcessTree: true); - throw new TimeoutException("Timed out while running the C# clink integration test."); - } - - return new CommandResult(process.ExitCode, await stdout, await stderr); - } - - private static void AssertClinkSucceeded(CommandResult result) - { - Assert.True( - result.ExitCode == 0, - $"clink exited with {result.ExitCode}\nstdout:\n{result.Stdout}\nstderr:\n{result.Stderr}"); - } - - private static string NormalizeNewlines(string text) - { - return text.Replace("\r\n", "\n"); - } - - private static string FindCsharpDirectory() - { - var directory = new DirectoryInfo(AppContext.BaseDirectory); - while (directory is not null) - { - if (File.Exists(Path.Combine(directory.FullName, "Foundation.Data.Doublets.Cli.sln"))) - { - return directory.FullName; - } - - directory = directory.Parent; - } - - throw new DirectoryNotFoundException("Could not find the csharp directory containing Foundation.Data.Doublets.Cli.sln."); - } - private static string CreateTempDirectory() { var tempDirectory = Path.Combine(Path.GetTempPath(), $"clink-export-{Guid.NewGuid():N}"); Directory.CreateDirectory(tempDirectory); return tempDirectory; } - - private sealed record CommandResult(int ExitCode, string Stdout, string Stderr); } diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Clink.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Clink.cs new file mode 100644 index 0000000..a78f0cf --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Clink.cs @@ -0,0 +1,49 @@ +using System.Diagnostics; + +namespace Foundation.Data.Doublets.Cli.Tests; + +/// Runs the clink executable built next to the tests. +public static class Clink +{ + private static readonly string Executable = Path.Combine(AppContext.BaseDirectory, "clink.dll"); + + public sealed record Result(int ExitCode, string Stdout, string Stderr); + + public static ProcessStartInfo StartInfo(IEnumerable arguments) + { + var info = new ProcessStartInfo("dotnet") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + info.ArgumentList.Add(Executable); + foreach (var argument in arguments) + { + info.ArgumentList.Add(argument); + } + return info; + } + + public static async Task RunAsync(params string[] arguments) + { + using var process = Process.Start(StartInfo(arguments))!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + try + { + await process.WaitForExitAsync(timeout.Token); + } + catch (OperationCanceledException) + { + process.Kill(entireProcessTree: true); + throw new TimeoutException($"clink {string.Join(' ', arguments)} did not exit within a minute."); + } + return new Result(process.ExitCode, (await stdout).Replace("\r\n", "\n"), (await stderr).Replace("\r\n", "\n")); + } + + public static void AssertSucceeded(Result result) => + Assert.True( + result.ExitCode == 0, + $"clink exited with {result.ExitCode}\nstdout:\n{result.Stdout}\nstderr:\n{result.Stderr}"); +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs index 5cb68dd..e0fa9cc 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs @@ -6,36 +6,10 @@ namespace Foundation.Data.Doublets.Cli.Tests.Protocol; public sealed class CliTcpIntegrationTests : IDisposable { private const string Banner = "clink server listening on "; - private static readonly string Clink = Path.Combine(AppContext.BaseDirectory, "clink.dll"); private readonly string _directory = Directory.CreateTempSubdirectory("clink-tcp-").FullName; public void Dispose() => Directory.Delete(_directory, recursive: true); - private static ProcessStartInfo StartInfo(IEnumerable arguments) - { - var info = new ProcessStartInfo("dotnet") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - }; - info.ArgumentList.Add(Clink); - foreach (var argument in arguments) - { - info.ArgumentList.Add(argument); - } - return info; - } - - private static async Task<(int ExitCode, string Stdout, string Stderr)> RunAsync(params string[] arguments) - { - using var process = Process.Start(StartInfo(arguments))!; - var stdout = process.StandardOutput.ReadToEndAsync(); - var stderr = process.StandardError.ReadToEndAsync(); - using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(60)); - await process.WaitForExitAsync(timeout.Token); - return (process.ExitCode, (await stdout).Replace("\r\n", "\n"), await stderr); - } - private sealed class Server : IDisposable { private readonly Process _process; @@ -50,7 +24,7 @@ private Server(Process process, string address) public static async Task StartAsync(string database, params string[] extra) { - var process = Process.Start(StartInfo(new[] { "--db", database, "--serve", "127.0.0.1:0" }.Concat(extra)))!; + var process = Process.Start(Clink.StartInfo(new[] { "--db", database, "--serve", "127.0.0.1:0" }.Concat(extra)))!; using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(60)); var line = await process.StandardOutput.ReadLineAsync(timeout.Token); if (line is null || !line.StartsWith(Banner, StringComparison.Ordinal)) @@ -64,7 +38,7 @@ public static async Task StartAsync(string database, params string[] ext public async Task StdoutAsync(params string[] arguments) { - var (exitCode, stdout, stderr) = await RunAsync(new[] { "--connect", Address }.Concat(arguments).ToArray()); + var (exitCode, stdout, stderr) = await Clink.RunAsync(new[] { "--connect", Address }.Concat(arguments).ToArray()); Assert.True(exitCode == 0, $"{string.Join(' ', arguments)}: {stderr}"); return stdout; } @@ -95,7 +69,7 @@ public async Task ClientsQueryAServedDatabaseOverBothProtocols() Assert.Equal("((2: 2 2)) ()\n((1: 1 2)) ()\n", await server.StdoutAsync("--protocol", "binary", "((2: 2 2)) ()")); Assert.Equal("", await server.StdoutAsync("--protocol", "text")); - var failure = await RunAsync("--connect", server.Address, "((99: 1 1)) ()"); + var failure = await Clink.RunAsync("--connect", server.Address, "((99: 1 1)) ()"); Assert.NotEqual(0, failure.ExitCode); Assert.Contains("server error", failure.Stderr, StringComparison.Ordinal); } @@ -108,7 +82,7 @@ public async Task ServedChangesArePersisted() { await server.StdoutAsync("() ((child: father mother))"); } - var (exitCode, stdout, _) = await RunAsync("--db", database, "--after"); + var (exitCode, stdout, _) = await Clink.RunAsync("--db", database, "--after"); Assert.Equal(0, exitCode); Assert.Contains("(child: father mother)", stdout, StringComparison.Ordinal); } @@ -121,7 +95,7 @@ public async Task ServedChangesArePersisted() [InlineData("--serve", "127.0.0.1:0", "() ((1 1))")] public async Task InvalidCombinationsAreRejected(params string[] arguments) { - var (exitCode, _, stderr) = await RunAsync(new[] { "--db", Path.Combine(_directory, "x.links") }.Concat(arguments).ToArray()); + var (exitCode, _, stderr) = await Clink.RunAsync(new[] { "--db", Path.Combine(_directory, "x.links") }.Concat(arguments).ToArray()); Assert.NotEqual(0, exitCode); Assert.False(string.IsNullOrWhiteSpace(stderr)); } @@ -129,7 +103,7 @@ public async Task InvalidCombinationsAreRejected(params string[] arguments) [Fact] public async Task HelpListsTheTcpOptions() { - var (_, stdout, _) = await RunAsync("--help"); + var (_, stdout, _) = await Clink.RunAsync("--help"); foreach (var option in new[] { "--serve", "--connect", "--protocol", "--external-references", "--sequences", "--progressive-widths" }) { Assert.Contains(option, stdout, StringComparison.Ordinal); diff --git a/csharp/Foundation.Data.Doublets.Cli/Program.cs b/csharp/Foundation.Data.Doublets.Cli/Program.cs index 9d24dbc..632eeeb 100644 --- a/csharp/Foundation.Data.Doublets.Cli/Program.cs +++ b/csharp/Foundation.Data.Doublets.Cli/Program.cs @@ -1,6 +1,7 @@ using System.CommandLine; using Foundation.Data.Doublets.Cli; using Foundation.Data.Doublets.Cli.Protocol; +using Link.Foundation.Links.Notation; using Platform.Data; using Platform.Data.Doublets; @@ -672,7 +673,20 @@ int RunQueryPipeline() } }; - QueryProcessor.ProcessQuery(decoratedLinks, options); + try + { + QueryProcessor.ProcessQuery(decoratedLinks, options); + } + catch (Exception error) when (error is ParseException or InvalidOperationException) + { + var kind = error is ParseException ? "Parse error" : "Query error"; + Console.Error.WriteLine($"Error: {kind}: {error.Message}"); + if (trace) + { + Console.Error.WriteLine(error); + } + return 1; + } } if (changes && changesList.Any()) diff --git a/rust/tests/cli_error_reporting_tests.rs b/rust/tests/cli_error_reporting_tests.rs new file mode 100644 index 0000000..e08835f --- /dev/null +++ b/rust/tests/cli_error_reporting_tests.rs @@ -0,0 +1,65 @@ +//! A query clink cannot run is reported on stderr as one error, the way the +//! C# clink reports it. + +use std::path::Path; +use std::process::{Command, Output}; +use tempfile::tempdir; + +fn run_clink(db_path: &Path, arguments: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_clink")) + .arg("--db") + .arg(db_path) + .args(arguments) + .output() + .expect("clink runs") +} + +fn stdout(output: &Output) -> String { + String::from_utf8_lossy(&output.stdout).into_owned() +} + +fn stderr(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).into_owned() +} + +#[test] +fn a_missing_reference_is_a_query_error() { + let directory = tempdir().unwrap(); + let output = run_clink(&directory.path().join("db.links"), &["() ((2 2))"]); + + assert_eq!(output.status.code(), Some(1)); + assert_eq!( + stderr(&output), + "Error: Query error: Invalid reference to non-existent link '2' in substitution pattern. \ + Link '2' does not exist and will not be created by this operation. \ + Use --auto-create-missing-references to create missing references as point links.\n" + ); + assert_eq!(stdout(&output), ""); +} + +#[test] +fn a_malformed_query_is_a_parse_error_pointing_at_the_problem() { + let directory = tempdir().unwrap(); + let output = run_clink(&directory.path().join("db.links"), &["((("]); + + assert_eq!(output.status.code(), Some(1)); + let stderr = stderr(&output); + assert!( + stderr.starts_with("Error: Parse error: Syntax error at line 1, column 4"), + "{stderr}" + ); + assert!(stderr.contains("1 | (((\n | ^"), "{stderr}"); +} + +#[test] +fn a_failed_query_leaves_the_store_unchanged() { + let directory = tempdir().unwrap(); + let db_path = directory.path().join("db.links"); + assert!(run_clink(&db_path, &["() ((1 1))"]).status.success()); + + assert_eq!(run_clink(&db_path, &["() ((1 3))"]).status.code(), Some(1)); + + let after = run_clink(&db_path, &["--after"]); + assert!(after.status.success(), "{}", stderr(&after)); + assert_eq!(stdout(&after), "(1: 1 1)\n"); +} From 4e654f4d41954bc51cb57ee6a901a9de9c84027c Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 00:51:49 +0000 Subject: [PATCH 21/44] fix(csharp): re-point the usages of a merged link instead of blanking them An update into an existing doublet merges into it. Platform.Data.Doublets 0.18.1 MergeUsages builds new Link(a, b), which is (index: a, source: b, target: 0), so C# left (1: 1 0) where Rust re-points the usage to (1: 1 1) (linksplatform/Data.Doublets#515). Stores are now composed with DecorateWithAutomaticUniquenessAndUsagesRepointing(), whose LinksUniquenessAndUsagesRepointingResolver re-points both halves of each usage in one update. The issue-100 parity harness drops its last exemption: 41 scenarios agree. --- .../issue-105-merge-repoints-usages.md | 5 + ...ksUniquenessAndUsagesRepointingResolver.cs | 79 ++++++++++ .../NamedLinksDecorator.cs | 4 +- .../NamedTypesDecorator.cs | 4 +- .../SimpleLinksDecorator.cs | 4 +- .../AdvancedMixedQueryProcessor.Merges.cs | 51 +++++++ .../BasicQueryProcessor.cs | 2 +- .../ExtensibilityTests.cs | 1 + ...quenessAndUsagesRepointingResolverTests.cs | 135 ++++++++++++++++++ .../MixedQueryProcessor.cs | 2 +- .../NamedTypesDecoratorTests.cs | 2 +- .../UnicodeStringStorageTests.cs | 2 +- docs/case-studies/issue-100/README.md | 8 +- .../issue-100/evidence/cli-parity/run.sh | 33 +---- docs/case-studies/issue-105/README.md | 58 ++++++-- .../doublets_resolver_semantics_tests.rs | 10 +- .../query_processor_csharp_parity_tests.rs | 51 +++++++ 17 files changed, 396 insertions(+), 55 deletions(-) create mode 100644 csharp/.changeset/issue-105-merge-repoints-usages.md create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/LinksUniquenessAndUsagesRepointingResolver.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.Merges.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/LinksUniquenessAndUsagesRepointingResolverTests.cs diff --git a/csharp/.changeset/issue-105-merge-repoints-usages.md b/csharp/.changeset/issue-105-merge-repoints-usages.md new file mode 100644 index 0000000..b942feb --- /dev/null +++ b/csharp/.changeset/issue-105-merge-repoints-usages.md @@ -0,0 +1,5 @@ +--- +'Foundation.Data.Doublets.Cli': patch +--- + +An update into an existing doublet merges into it and now re-points every link that used the merged-away address at the surviving one, as the Rust port does. Before, the `MergeUsages` of Platform.Data.Doublets 0.18.1 ([Data.Doublets#515](https://github.com/linksplatform/Data.Doublets/issues/515)) blanked the half of the usage instead, so `() ((1 2) (2 1))` followed by `((1: 1 2)) ((1: 2 1))` left `(2: 2 0)` rather than `(2: 2 2)`. Stores are composed with the new `DecorateWithAutomaticUniquenessAndUsagesRepointing()`, whose top layer, `LinksUniquenessAndUsagesRepointingResolver`, replaces `LinksCascadeUniquenessAndUsagesResolver`. diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/LinksUniquenessAndUsagesRepointingResolver.cs b/csharp/Foundation.Data.Doublets.Cli.Library/LinksUniquenessAndUsagesRepointingResolver.cs new file mode 100644 index 0000000..a8855bf --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/LinksUniquenessAndUsagesRepointingResolver.cs @@ -0,0 +1,79 @@ +using System.Numerics; +using Platform.Delegates; +using Platform.Data; +using Platform.Data.Doublets; +using Platform.Data.Doublets.Decorators; + +namespace Foundation.Data.Doublets.Cli +{ + /// + /// Keeps every doublet unique: an update that would duplicate an existing link merges into it instead, + /// re-pointing every link that used the merged-away address at the surviving one. + /// + /// + /// Replaces , whose + /// MergeUsages in Platform.Data.Doublets 0.18.1 writes (index: a, source: b, target: 0) + /// where it means (source: a, target: b), so a usage loses its target instead of being re-pointed + /// (https://github.com/linksplatform/Data.Doublets/issues/515). + /// + public class LinksUniquenessAndUsagesRepointingResolver : LinksUniquenessResolver + where TLinkAddress : IUnsignedNumber + { + public LinksUniquenessAndUsagesRepointingResolver(ILinks links) : base(links) { } + + protected override TLinkAddress ResolveAddressChangeConflict(TLinkAddress oldLinkAddress, TLinkAddress newLinkAddress, WriteHandler? handler) + { + WriteHandlerState handlerState = new(_constants.Continue, _constants.Break, handler); + if (oldLinkAddress != newLinkAddress) + { + handlerState.Apply(RepointUsages(oldLinkAddress, newLinkAddress, handlerState.Handler)); + } + handlerState.Apply(base.ResolveAddressChangeConflict(oldLinkAddress, newLinkAddress, handlerState.Handler)); + return handlerState.Result; + } + + /// + /// Replaces with in every link that uses it, + /// both halves of a link in one update. + /// + public virtual TLinkAddress RepointUsages(TLinkAddress oldLinkAddress, TLinkAddress newLinkAddress, WriteHandler? handler) + { + WriteHandlerState handlerState = new(_constants.Continue, _constants.Break, handler); + var any = _constants.Any; + var usages = _facade.All(new Link(any, oldLinkAddress, any)) + .Concat(_facade.All(new Link(any, any, oldLinkAddress))) + .Select(usage => _facade.GetIndex(usage)) + .Where(usage => usage != oldLinkAddress) + .Distinct() + .ToList(); + foreach (var usage in usages) + { + // Re-pointing an earlier usage can merge this one away. + if (!_facade.Exists(usage)) + { + continue; + } + var link = new Link(_facade.GetLink(usage)); + var source = link.Source == oldLinkAddress ? newLinkAddress : link.Source; + var target = link.Target == oldLinkAddress ? newLinkAddress : link.Target; + handlerState.Apply(_facade.Update(new LinkAddress(usage), new Link(usage, source, target), handlerState.Handler)); + } + return handlerState.Result; + } + } + + public static class LinksUniquenessAndUsagesRepointingExtensions + { + /// + /// The decorators of DecorateWithAutomaticUniquenessAndUsagesResolution, with usages of a merged link + /// re-pointed by . + /// + public static ILinks DecorateWithAutomaticUniquenessAndUsagesRepointing(this ILinks links) + where TLinkAddress : IUnsignedNumber + { + links = new LinksCascadeUsagesResolver(links); + links = new NonNullContentsLinkDeletionResolver(links); + return new LinksUniquenessAndUsagesRepointingResolver(links); + } + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/NamedLinksDecorator.cs b/csharp/Foundation.Data.Doublets.Cli.Library/NamedLinksDecorator.cs index 61a978b..d604899 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/NamedLinksDecorator.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/NamedLinksDecorator.cs @@ -34,7 +34,7 @@ public class NamedLinksDecorator : LinksDecoratorBase MakeLinks(string databaseFilename) { var links = new UnitedMemoryLinks(databaseFilename); - return links.DecorateWithAutomaticUniquenessAndUsagesResolution(); + return links.DecorateWithAutomaticUniquenessAndUsagesRepointing(); } public static string MakeNamesDatabaseFilename(string databaseFilename) @@ -54,7 +54,7 @@ public NamedLinksDecorator(ILinks links, string namesDatabaseFilen var namesConstants = new LinksConstants(enableExternalReferencesSupport: true); var namesMemory = new FileMappedResizableDirectMemory(namesDatabaseFilename, UnitedMemoryLinks.DefaultLinksSizeStep); var namesLinks = new UnitedMemoryLinks(namesMemory, UnitedMemoryLinks.DefaultLinksSizeStep, namesConstants, IndexTreeType.Default); - var decoratedNamesLinks = namesLinks.DecorateWithAutomaticUniquenessAndUsagesResolution(); + var decoratedNamesLinks = namesLinks.DecorateWithAutomaticUniquenessAndUsagesRepointing(); _namedLinksFacade = decoratedNamesLinks; NamedLinks = new UnicodeStringStorage(decoratedNamesLinks).NamedLinks; NamedLinksDatabaseFileName = namesDatabaseFilename; diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/NamedTypesDecorator.cs b/csharp/Foundation.Data.Doublets.Cli.Library/NamedTypesDecorator.cs index 5c03369..8a4366e 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/NamedTypesDecorator.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/NamedTypesDecorator.cs @@ -35,7 +35,7 @@ public class NamedTypesDecorator : LinksDecoratorBase MakeLinks(string databaseFilename) { var links = new UnitedMemoryLinks(databaseFilename); - return links.DecorateWithAutomaticUniquenessAndUsagesResolution(); + return links.DecorateWithAutomaticUniquenessAndUsagesRepointing(); } public static string MakeNamesDatabaseFilename(string databaseFilename) @@ -61,7 +61,7 @@ public NamedTypesDecorator(PinnedTypesDecorator pinnedTypesDecorat var namesConstants = new LinksConstants(enableExternalReferencesSupport: true); var namesMemory = new FileMappedResizableDirectMemory(namesDatabaseFilename, UnitedMemoryLinks.DefaultLinksSizeStep); var namesLinks = new UnitedMemoryLinks(namesMemory, UnitedMemoryLinks.DefaultLinksSizeStep, namesConstants, IndexTreeType.Default); - var decoratedNamesLinks = namesLinks.DecorateWithAutomaticUniquenessAndUsagesResolution(); + var decoratedNamesLinks = namesLinks.DecorateWithAutomaticUniquenessAndUsagesRepointing(); _namedLinksFacade = decoratedNamesLinks; NamedLinks = new UnicodeStringStorage(decoratedNamesLinks).NamedLinks; NamedLinksDatabaseFileName = namesDatabaseFilename; diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/SimpleLinksDecorator.cs b/csharp/Foundation.Data.Doublets.Cli.Library/SimpleLinksDecorator.cs index a6b72e8..75d237b 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/SimpleLinksDecorator.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/SimpleLinksDecorator.cs @@ -32,7 +32,7 @@ public class SimpleLinksDecorator : LinksDecoratorBase MakeLinks(string databaseFilename) { var links = new UnitedMemoryLinks(databaseFilename); - return links.DecorateWithAutomaticUniquenessAndUsagesResolution(); + return links.DecorateWithAutomaticUniquenessAndUsagesRepointing(); } public static string MakeNamesDatabaseFilename(string databaseFilename) @@ -52,7 +52,7 @@ public SimpleLinksDecorator(ILinks links, string namesDatabaseFile var namesConstants = new LinksConstants(enableExternalReferencesSupport: true); var namesMemory = new FileMappedResizableDirectMemory(namesDatabaseFilename, UnitedMemoryLinks.DefaultLinksSizeStep); var namesLinks = new UnitedMemoryLinks(namesMemory, UnitedMemoryLinks.DefaultLinksSizeStep, namesConstants, IndexTreeType.Default); - var decoratedNamesLinks = namesLinks.DecorateWithAutomaticUniquenessAndUsagesResolution(); + var decoratedNamesLinks = namesLinks.DecorateWithAutomaticUniquenessAndUsagesRepointing(); _namedLinksFacade = decoratedNamesLinks; NamedLinks = new UnicodeStringStorage(decoratedNamesLinks).NamedLinks; NamedLinksDatabaseFileName = namesDatabaseFilename; diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.Merges.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.Merges.cs new file mode 100644 index 0000000..d5fe892 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.Merges.cs @@ -0,0 +1,51 @@ +// An update into an existing doublet merges into it and re-points the links that used the merged-away +// address, the way the Rust port does (https://github.com/linksplatform/Data.Doublets/issues/515). +using DoubletLink = Platform.Data.Doublets.Link; + +using static Foundation.Data.Doublets.Cli.AdvancedMixedQueryProcessor; +namespace Foundation.Data.Doublets.Cli.Tests.Tests +{ + public partial class AdvancedMixedQueryProcessor + { + [Fact] + public void UpdateIntoExistingPairRepointsUsagesTest() + { + RunTestWithLinks(links => + { + ProcessQuery(links, "() ((1 2) (2 1))"); + + var changes = ReportedChanges(links, "((1: 1 2)) ((1: 2 1))"); + + Assert.Equal( + new[] + { + (new DoubletLink(1, 1, 2), new DoubletLink(1, 2, 1)), + (new DoubletLink(2, 2, 1), new DoubletLink(2, 2, 2)), + }, + changes); + Assert.Equal(new[] { new DoubletLink(1, 2, 1), new DoubletLink(2, 2, 2) }, GetAllLinks(links)); + }); + } + + [Fact] + public void UpdateIntoAPairThatUsesItRepointsTheUsageTest() + { + RunTestWithLinks(links => + { + ProcessQuery(links, "() ((1 1) (2 2))"); + ProcessQuery(links, "((1: 1 1)) ((1: 1 2))"); + + var changes = ReportedChanges(links, "((2: 2 2)) ((2: 1 2))"); + + Assert.Equal( + new[] + { + (new DoubletLink(1, 1, 2), new DoubletLink(1, 1, 1)), + (new DoubletLink(2, 2, 2), new DoubletLink(2, 1, 2)), + }, + changes); + Assert.Equal(new[] { new DoubletLink(1, 1, 1), new DoubletLink(2, 1, 2) }, GetAllLinks(links)); + }); + } + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/BasicQueryProcessor.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/BasicQueryProcessor.cs index ae25c56..c0d0cdb 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/BasicQueryProcessor.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/BasicQueryProcessor.cs @@ -156,7 +156,7 @@ private static void RunTestWithLinks(Action> testAction) try { using var links = new UnitedMemoryLinks(tempDbFile); - var decoratedLinks = links.DecorateWithAutomaticUniquenessAndUsagesResolution(); + var decoratedLinks = links.DecorateWithAutomaticUniquenessAndUsagesRepointing(); testAction(decoratedLinks); } finally diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/ExtensibilityTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/ExtensibilityTests.cs index fef6b83..c3597eb 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/ExtensibilityTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/ExtensibilityTests.cs @@ -112,6 +112,7 @@ public void VersionControlDecoratorCanBeSubclassed() [InlineData(typeof(NamedLinks))] [InlineData(typeof(PinnedTypes))] [InlineData(typeof(UnicodeStringStorage))] + [InlineData(typeof(LinksUniquenessAndUsagesRepointingResolver))] public void PublicTypesStayOpenForExtension(Type type) { Assert.True(type.IsPublic, $"{type.Name} must be public."); diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/LinksUniquenessAndUsagesRepointingResolverTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/LinksUniquenessAndUsagesRepointingResolverTests.cs new file mode 100644 index 0000000..33fa43d --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/LinksUniquenessAndUsagesRepointingResolverTests.cs @@ -0,0 +1,135 @@ +using Platform.Data; +using Platform.Data.Doublets; +using Platform.Data.Doublets.Memory.United.Generic; +using DoubletLink = Platform.Data.Doublets.Link; + +namespace Foundation.Data.Doublets.Cli.Tests +{ + /// + /// An update into an existing doublet merges into it, and every link that used the merged-away address + /// is re-pointed at the surviving one (https://github.com/linksplatform/Data.Doublets/issues/515). + /// + public sealed class LinksUniquenessAndUsagesRepointingResolverTests : IDisposable + { + private readonly string _databaseFile = Path.GetTempFileName(); + private readonly UnitedMemoryLinks _store; + private readonly ILinks _links; + + public LinksUniquenessAndUsagesRepointingResolverTests() + { + _store = new UnitedMemoryLinks(_databaseFile); + _links = _store.DecorateWithAutomaticUniquenessAndUsagesRepointing(); + } + + public void Dispose() + { + _store.Dispose(); + File.Delete(_databaseFile); + } + + private uint Store(uint source, uint target) => _links.CreateAndUpdate(source, target); + + private uint Point() => _links.CreatePoint(); + + private void Update(uint index, uint source, uint target) => + _links.Update(new DoubletLink(index, _links.Constants.Any, _links.Constants.Any), new DoubletLink(index, source, target), null); + + private List AllLinks() => + _links.All(new DoubletLink(_links.Constants.Any, _links.Constants.Any, _links.Constants.Any)) + .Select(link => new DoubletLink(link)) + .ToList(); + + [Fact] + public void RepointUsagesReplacesOnlyTheHalfThatNamedTheMergedLink() + { + var merged = Point(); + var surviving = Point(); + var unrelated = Point(); + var usageAsSource = Store(merged, unrelated); + var usageAsTarget = Store(unrelated, merged); + var resolver = Assert.IsType>(_links); + + resolver.RepointUsages(merged, surviving, null); + + Assert.Equal( + new[] + { + new DoubletLink(merged, merged, merged), + new DoubletLink(surviving, surviving, surviving), + new DoubletLink(unrelated, unrelated, unrelated), + new DoubletLink(usageAsSource, surviving, unrelated), + new DoubletLink(usageAsTarget, unrelated, surviving), + }, + AllLinks()); + } + + [Fact] + public void AnUpdateIntoAnExistingDoubletRepointsTheUsagesOfTheMergedLink() + { + var surviving = Store(0, 0); + var merged = Point(); + Update(surviving, surviving, merged); + + Update(merged, surviving, merged); + + Assert.Equal(new[] { new DoubletLink(surviving, surviving, surviving) }, AllLinks()); + } + + [Fact] + public void AnUpdateReportsTheRepointedUsageAndTheDeletedLink() + { + var surviving = Store(0, 0); + var merged = Point(); + Update(surviving, surviving, merged); + var changes = new List<(DoubletLink Before, DoubletLink After)>(); + + _links.Update( + new DoubletLink(merged, _links.Constants.Any, _links.Constants.Any), + new DoubletLink(merged, surviving, merged), + (before, after) => + { + changes.Add((new DoubletLink(before), new DoubletLink(after))); + return _links.Constants.Continue; + }); + + Assert.Contains((new DoubletLink(surviving, surviving, merged), new DoubletLink(surviving, surviving, surviving)), changes); + Assert.Equal(new DoubletLink(0, 0, 0), changes[^1].After); + Assert.Equal(merged, changes[^1].Before.Index); + } + + [Fact] + public void AUsageOfTheMergedLinkInBothHalvesIsRepointedInOneUpdate() + { + var surviving = Point(); + var other = Point(); + var merged = Store(surviving, other); + var usage = Store(merged, merged); + + Update(merged, surviving, surviving); + + // The usage becomes (surviving surviving), which is the surviving link itself, so it merges in too. + Assert.Equal( + new[] { new DoubletLink(surviving, surviving, surviving), new DoubletLink(other, other, other) }, + AllLinks()); + Assert.False(_links.Exists(usage)); + } + + [Fact] + public void AUsageMergedAwayWhileAnEarlierOneIsRepointedIsSkipped() + { + var surviving = Point(); + var merged = Point(); + var usageAsSource = Store(merged, surviving); + var usageAsTarget = Store(surviving, merged); + // Re-pointing usageAsSource makes it the surviving link, so this one becomes (surviving merged), + // a duplicate of usageAsTarget, and merges into it before the loop reaches it. + var usageOfAUsage = Store(usageAsSource, merged); + + Update(merged, surviving, surviving); + + Assert.Equal(new[] { new DoubletLink(surviving, surviving, surviving) }, AllLinks()); + Assert.False(_links.Exists(usageAsTarget)); + Assert.False(_links.Exists(usageOfAUsage)); + } + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/MixedQueryProcessor.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/MixedQueryProcessor.cs index c386ce5..aa8e773 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/MixedQueryProcessor.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/MixedQueryProcessor.cs @@ -399,7 +399,7 @@ private static void RunTestWithLinks(Action> testAction) try { using var links = new UnitedMemoryLinks(tempDbFile); - var decoratedLinks = links.DecorateWithAutomaticUniquenessAndUsagesResolution(); + var decoratedLinks = links.DecorateWithAutomaticUniquenessAndUsagesRepointing(); testAction(decoratedLinks); } finally diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/NamedTypesDecoratorTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/NamedTypesDecoratorTests.cs index ec167ef..a85e6f8 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/NamedTypesDecoratorTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/NamedTypesDecoratorTests.cs @@ -36,7 +36,7 @@ private static void RunTestWithLinks(Action> testAction) { using var memory = new FileMappedResizableDirectMemory(tempFile, UnitedMemoryLinks.DefaultLinksSizeStep); using var unitedMemoryLinks = new UnitedMemoryLinks(memory); - var linksDecoratedWithAutomaticUniquenessResolution = unitedMemoryLinks.DecorateWithAutomaticUniquenessAndUsagesResolution(); + var linksDecoratedWithAutomaticUniquenessResolution = unitedMemoryLinks.DecorateWithAutomaticUniquenessAndUsagesRepointing(); testAction(linksDecoratedWithAutomaticUniquenessResolution); } finally diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/UnicodeStringStorageTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/UnicodeStringStorageTests.cs index b2793ff..c3edd18 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/UnicodeStringStorageTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/UnicodeStringStorageTests.cs @@ -247,7 +247,7 @@ private static void RunTestWithLinks(Action> testAction) var constants = new LinksConstants(enableExternalReferencesSupport: true); using var memory = new FileMappedResizableDirectMemory(tempDbFile, UnitedMemoryLinks.DefaultLinksSizeStep); using var links = new UnitedMemoryLinks(memory, UnitedMemoryLinks.DefaultLinksSizeStep, constants, Platform.Data.Doublets.Memory.IndexTreeType.Default); - var decoratedLinks = links.DecorateWithAutomaticUniquenessAndUsagesResolution(); + var decoratedLinks = links.DecorateWithAutomaticUniquenessAndUsagesRepointing(); testAction(decoratedLinks); } finally diff --git a/docs/case-studies/issue-100/README.md b/docs/case-studies/issue-100/README.md index e6416d7..5aef235 100644 --- a/docs/case-studies/issue-100/README.md +++ b/docs/case-studies/issue-100/README.md @@ -189,6 +189,12 @@ code involved. Filed as [Data.Doublets#515](https://github.com/linksplatform/Data.Doublets/issues/515) and recorded in the parity harness as its single `known_difference`. +**Update (issue #105):** the C# library no longer uses `MergeUsages`. Its +stores are composed with `DecorateWithAutomaticUniquenessAndUsagesRepointing()`, +whose `LinksUniquenessAndUsagesRepointingResolver` re-points usages itself, so +this scenario now agrees and the harness has no exemption left (see +[the issue #105 case study, §6.3](../issue-105/README.md#63-behaviour-differences-between-the-ports-query-processors)). + ### 5.6 The constants differ, and it is unreachable in practice `doublets` 0.5.0 re-exports `platform-data` 2.0.0's `LinksConstants`. @@ -345,7 +351,7 @@ trigger scenarios including the embedded store. ## 10. Risks and follow-ups -- **The `MergeUsages` exemption.** Until +- **The `MergeUsages` exemption** (worked around in issue #105, §5.5). Until [Data.Doublets#515](https://github.com/linksplatform/Data.Doublets/issues/515) is released, `update into duplicate` produces a corrupt link in C#. The harness will turn red the moment the languages agree, which is the signal to diff --git a/docs/case-studies/issue-100/evidence/cli-parity/run.sh b/docs/case-studies/issue-100/evidence/cli-parity/run.sh index 54526ba..e56d432 100755 --- a/docs/case-studies/issue-100/evidence/cli-parity/run.sh +++ b/docs/case-studies/issue-100/evidence/cli-parity/run.sh @@ -113,28 +113,6 @@ scenario() { fi } -# known_difference ... -# -# A scenario the two CLIs are *expected* to answer differently because of a -# defect in a dependency rather than in this repository. It does not count as a -# failure, but agreement does: the day the upstream fix lands, this turns red so -# the exemption gets removed instead of quietly outliving its reason. -known_difference() { - local name="$1" reason="$2"; shift 2 - run_both "$@" - EXTRA=(); TRIGGER_SETUP=() - - if agree; then - failures=$((failures + 1)) - echo "FAIL $name (the languages now agree -- drop the exemption)" - echo " reason on record: $reason" - else - echo "KNOWN $name" - echo " $reason" - report_divergence "$@" - fi -} - scenario "create" '() ((1 1))' scenario "duplicate create" '() ((1 1))' '() ((1 1))' scenario "update target" '() ((1 1))' '() ((2 2))' '((1: 1 1)) ((1: 1 2))' @@ -219,13 +197,16 @@ EXTRA=(--embed-triggers) trigger_scenario "trigger embedded in the main database" \ --always '(((1: 1 1)) ((1: 1 2)))' -- '() ((1: 1 1))' -known_difference "update into duplicate" \ - "Platform.Data.Doublets 0.18.1 MergeUsages corrupts the usages it repoints (see ../csharp-merge-usages), so C# leaves (2: 2 0) where doublets-rs rebases the usage onto the surviving link and leaves (2: 2 2)." \ - '() ((1 2) (2 1))' '((1: 1 2)) ((1: 2 1))' +# An update into an existing pair merges into it and re-points every usage of +# the merged-away link. C# used to blank the usage instead, through the +# MergeUsages of Platform.Data.Doublets 0.18.1 (see ../csharp-merge-usages, +# Data.Doublets#515); it now composes a resolver of its own (issue 105). +scenario "update into duplicate" '() ((1 2) (2 1))' '((1: 1 2)) ((1: 2 1))' +scenario "update into the pair using it" '() ((1 1) (2 2))' '((1: 1 1)) ((1: 1 2))' '((2: 2 2)) ((2: 1 2))' echo if [ "$failures" -eq 0 ]; then - echo "All scenarios match, except the known upstream differences listed above." + echo "All scenarios match." else echo "$failures scenario(s) diverge." fi diff --git a/docs/case-studies/issue-105/README.md b/docs/case-studies/issue-105/README.md index ca32d88..b5b98d4 100644 --- a/docs/case-studies/issue-105/README.md +++ b/docs/case-studies/issue-105/README.md @@ -375,19 +375,51 @@ prints the raw parser output, which was used to confirm this. ### 6.3 Behaviour differences between the ports' query processors -These are pre-existing behaviours, not introduced by the protocols. The -protocols carry whatever the processors return. - -- The C# `AdvancedMixedQueryProcessor` does not report the creation of a - *named* composite link as a change. As a result: - - `() ((a: a a))` gets an empty reply from a C# server. - - The same query gets `() ((a: a a))` from a Rust server. - - The link is created in both cases, and the listing shows it. -- Rust reports a failing query as `Query error: `, while C# reports - ``. -- Deleting a link also deletes the links that refer to it, in both ports. For - example, `((2: 2 2)) ()` with `(1: 1 2)` present replies with two - deletions. +A server replies with whatever its query processor reports, so every +difference between the processors became a difference between a C# and a Rust +server. Comparing the two `--changes` reports query by query +(`experiments/compare-cli-changes.sh`, and the parity harness in +[`../issue-100/evidence/cli-parity/run.sh`](../issue-100/evidence/cli-parity/run.sh), +now 41 scenarios that all agree) found these, all fixed here: + +- **Creations were not all reported.** The C# processor did not report a + named composite link it created, so `() ((a: a a))` got an empty reply from + a C# server and `() ((a: a a))` from a Rust one. Both now report every link + a query creates, once, as the creation of its final value. +- **New addresses were predicted wrongly.** Reference validation assumed a new + link gets the lowest free address, but both stores reuse the address freed + last first. Both now ask the store: create the links, note their addresses, + delete them again in reverse. +- **Errors were worded differently.** C# printed an unhandled exception with + its stack trace; it now prints one `Error: Query error: ...` or + `Error: Parse error: ...` line, as Rust does, and keeps the stack trace for + `--trace`. +- **A merge blanked usages in C#.** An update that turns a link into a + duplicate of another merges it into that other link, re-pointing whatever + used the merged-away address. After `() ((1 1) (2 2))` and + `((1: 1 1)) ((1: 1 2))`, the query `((2: 2 2)) ((2: 1 2))` merges 2 into 1: + + | | link 1 | link 2 | + |---|---|---| + | Rust (doublets-rs) | `(1: 1 1)`, its target re-pointed from 2 to 1 | `(2: 1 2)`, written by the query | + | C# before | `(1: 1 0)`, its target blanked | `(2: 1 2)` | + + The cause is upstream: `MergeUsages` in Platform.Data.Doublets 0.18.1 builds + `new Link(a, b)`, which binds to the `params` constructor and means + `(index: a, source: b, target: 0)`, not `(source: a, target: b)`. It was + already reported as + [Data.Doublets#515](https://github.com/linksplatform/Data.Doublets/issues/515) + during issue #100 and kept as the harness's one known difference. It is + still not released, so the C# library now composes its stores with + `DecorateWithAutomaticUniquenessAndUsagesRepointing()`: the same three + layers as `DecorateWithAutomaticUniquenessAndUsagesResolution()`, with + `LinksUniquenessAndUsagesRepointingResolver` on top, which replaces the + merged-away address in both halves of each usage in one update. + `LinksUniquenessAndUsagesRepointingResolverTests` fails on four of five + tests against the upstream resolver and passes against this one. + +Deleting a link also deletes the links that refer to it, in both ports. For +example, `((2: 2 2)) ()` with `(1: 1 2)` present replies with two deletions. ## 7. Risks and remaining limits diff --git a/rust/tests/doublets_resolver_semantics_tests.rs b/rust/tests/doublets_resolver_semantics_tests.rs index 77bc6e6..fdd16cc 100644 --- a/rust/tests/doublets_resolver_semantics_tests.rs +++ b/rust/tests/doublets_resolver_semantics_tests.rs @@ -27,11 +27,11 @@ fn dump(links: &impl Doublets) -> Vec<(u32, u32, u32)> { /// two, and every usage of the merged-away link is **rebased onto the /// survivor** rather than blanked. /// -/// This is the behaviour the `update into duplicate` scenario in -/// `docs/case-studies/issue-100/evidence/cli-parity/run.sh` records as -/// diverging from C#: `Platform.Data.Doublets` 0.18.1 corrupts the usage -/// instead (see `../csharp-merge-usages` in the same folder). If this test -/// ever fails, doublets-rs has moved and the exemption needs revisiting. +/// `Platform.Data.Doublets` 0.18.1 blanks the usage instead +/// (), so the C# +/// port composes its own `LinksUniquenessAndUsagesRepointingResolver` to get +/// this behaviour. If this test ever fails, doublets-rs has moved and the two +/// ports need comparing again. #[test] fn update_into_an_existing_pair_rebases_usages_onto_the_survivor() { let mut links = Store::::new(Global::new()) diff --git a/rust/tests/query_processor_csharp_parity_tests.rs b/rust/tests/query_processor_csharp_parity_tests.rs index 3e35a2e..3556e8d 100644 --- a/rust/tests/query_processor_csharp_parity_tests.rs +++ b/rust/tests/query_processor_csharp_parity_tests.rs @@ -362,6 +362,57 @@ fn test_update_into_existing_pair_reports_merge_matches_csharp() -> Result<()> { }) } +/// A link that used the merged-away address is re-pointed at the surviving one +/// rather than losing that half, as C# does since it stopped relying on the +/// `MergeUsages` of Platform.Data.Doublets 0.18.1 +/// (). +#[test] +fn test_update_into_existing_pair_repoints_usages_matches_csharp() -> Result<()> { + with_storage(|storage, processor| { + processor.process_query(storage, "() ((1 2) (2 1))")?; + + let changes = processor.process_query(storage, "((1: 1 2)) ((1: 2 1))")?; + + assert_eq!( + changes, + vec![ + (Some(Link::new(1, 1, 2)), Some(Link::new(1, 2, 1))), + (Some(Link::new(2, 2, 1)), Some(Link::new(2, 2, 2))), + ] + ); + assert_eq!( + sorted_links(storage), + vec![Link::new(1, 2, 1), Link::new(2, 2, 2)] + ); + Ok(()) + }) +} + +/// The link being merged away can be used by the very link it duplicates: the +/// usage is re-pointed at the survivor, and the query then writes its own link. +#[test] +fn test_update_into_a_pair_that_uses_it_repoints_the_usage_matches_csharp() -> Result<()> { + with_storage(|storage, processor| { + processor.process_query(storage, "() ((1 1) (2 2))")?; + processor.process_query(storage, "((1: 1 1)) ((1: 1 2))")?; + + let changes = processor.process_query(storage, "((2: 2 2)) ((2: 1 2))")?; + + assert_eq!( + changes, + vec![ + (Some(Link::new(1, 1, 2)), Some(Link::new(1, 1, 1))), + (Some(Link::new(2, 2, 2)), Some(Link::new(2, 1, 2))), + ] + ); + assert_eq!( + sorted_links(storage), + vec![Link::new(1, 1, 1), Link::new(2, 1, 2)] + ); + Ok(()) + }) +} + /// Deleting a named link cascades through the links that use it, and the names /// of the survivors are untouched. #[test] From 464ac0d8e94526b4a2f2bc7feb912bc3fbff9768 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 01:17:05 +0000 Subject: [PATCH 22/44] fix(csharp): a server shut down right after accepting a connection no longer crashes Shutdown disposes every registered client; a worker thread that had not started yet then set NoDelay on a disposed TcpClient, whose socket is null, and the unhandled NullReferenceException ended the process (it aborted the test run). The option is now set on the accepting thread, under the lock that keeps Shutdown out, and a refusal is traced instead of fatal, as in the Rust server. --- .../Protocol/LinksServer.cs | 16 +++++++++++++++- .../Protocol/LinksServerTests.cs | 12 ++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs index 0016a03..e0dcfa9 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs @@ -116,6 +116,8 @@ public void Serve(INamedTypesLinks links) client.Dispose(); break; } + // Here, where Shutdown cannot have disposed the client: the socket of a disposed client is null. + TryDisableNagle(client); _clients.Add(client); } var worker = new Thread(() => HandleConnection(client, links)) { IsBackground = true }; @@ -152,7 +154,6 @@ private void HandleConnection(TcpClient client, INamedTypesLinks links) { try { - client.NoDelay = true; using var stream = client.GetStream(); var reader = new LinoStreamReader(stream); while (!_stopping) @@ -196,6 +197,19 @@ private void HandleConnection(TcpClient client, INamedTypesLinks links) } } + /// Sends every reply at once; a connection that refuses still works, only slower, as in the Rust server. + private void TryDisableNagle(TcpClient client) + { + try + { + client.NoDelay = true; + } + catch (SocketException error) + { + Trace($"could not disable Nagle's algorithm: {error.Message}"); + } + } + private bool Accepts(MessageFormat format) => _options.Accept switch { AcceptedProtocols.Text => !format.IsBinary, diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs index 72f9245..f5fcef7 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs @@ -185,6 +185,18 @@ public void ShutdownStopsTheServer() Assert.ThrowsAny(() => client.Query("")); } + [Fact] + public void ShuttingDownRightAfterAConnectionArrivesIsClean() + { + // A connection the server has accepted but not started handling yet is + // closed by the shutdown; handling it must not crash the process. + for (var attempt = 0; attempt < 200; attempt++) + { + using var server = new RunningServer(); + using var client = server.Client(new TextLinoProtocol()); + } + } + [Fact] public void EndPointsAreParsed() { From 72228476847d780e6b96641a8f2ed4ade0ff261f Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 01:17:15 +0000 Subject: [PATCH 23/44] fix(rust): protocol writes reply with net changes, proven by a shared conversation A cascading delete takes different steps in the C# and the Rust stores (doublets-rs deletes a usage without clearing it first), so replying with the store's steps made the wire format port-specific. Update and delete replies are now simplified to the net change of every link, the way clink --changes reports a query. docs/protocol/links-operations.txt records one conversation with a fresh server; both ports replay it over every protocol, so a client of either port is proven to understand a server of the other. --- docs/protocol/links-operations.txt | 49 ++++++++++++++++++ .../20261005_120000_issue_105_remote_links.md | 5 ++ rust/src/protocol/links_operations.rs | 13 +++-- rust/src/protocol/remote_links.rs | 10 ++-- rust/tests/remote_links_tests.rs | 51 ++++++++++++++++--- 5 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 docs/protocol/links-operations.txt create mode 100644 rust/changelog.d/20261005_120000_issue_105_remote_links.md diff --git a/docs/protocol/links-operations.txt b/docs/protocol/links-operations.txt new file mode 100644 index 0000000..accb376 --- /dev/null +++ b/docs/protocol/links-operations.txt @@ -0,0 +1,49 @@ +# One conversation with a fresh links server, in the text protocol. +# +# Both ports replay it against their own server in their test suites +# (csharp: RemoteLinksTests, rust: remote_links_tests), so a client of either +# port is proven to understand a server of the other. +# +# `> request` is a request document; the lines up to the next request are its +# reply, and `! message` is an error reply carrying that message. + +> (create: (0 0)) +() ((1: 0 0)) +> (create: (1 1)) +() ((2: 1 1)) +> (update: (1 2 2)) +((1: 0 0)) ((1: 2 2)) +> (count: ()) +(count: 2) +> (count: (* 1)) +(count: 1) +> (count: (2)) +(count: 1) +> (each: (* * *)) +(1: 2 2) +(2: 1 1) +> (each: (* 2)) +(1: 2 2) +# set-name answers where the names store keeps the name, as INamedTypes does. +> (set-name: (2 'a pair')) +(link: 102) +> (get-name: 2) +(name: 'a pair') +> (get-by-name: 'a pair') +(link: 2) +> (remove-name: 2) +> (get-name: 2) +> (get-by-name: 'a pair') +> (update: (2 1 2)) +((2: 1 1)) ((2: 1 2)) +# Deleting 1 deletes 2, which uses it; every change is the net change of a +# link, whatever steps the store took to get there. +> (delete: 1) +((1: 2 2)) () +((2: 1 2)) () +> (count: ()) +(count: 0) +> (update: (7 1 1)) +! Link not found: 7 +> (delete: 7) +! Link not found: 7 diff --git a/rust/changelog.d/20261005_120000_issue_105_remote_links.md b/rust/changelog.d/20261005_120000_issue_105_remote_links.md new file mode 100644 index 0000000..0006c30 --- /dev/null +++ b/rust/changelog.d/20261005_120000_issue_105_remote_links.md @@ -0,0 +1,5 @@ +--- +bump: minor +--- + +Added `RemoteLinks`, which implements `doublets::Links`, `doublets::Doublets` and `NamedTypeLinks` for links that live behind a `LinksServer`. Code written against those traits, the `QueryProcessor` included, switches from a local file to a server by swapping one value. `LinksServer` now also answers `LinksOperation` documents, one per interface call: `(count: …)`, `(each: …)`, `(create: …)`, `(update: …)`, `(delete: …)`, `(get-name: …)`, `(set-name: …)`, `(get-by-name: …)` and `(remove-name: …)`. These are the documents the C# `RemoteLinks` sends, and both test suites replay the same recorded conversation, `docs/protocol/links-operations.txt`. A write replies with the net change of every link it touched: the steps of a cascading delete differ between the C# and the Rust stores, the net changes do not. diff --git a/rust/src/protocol/links_operations.rs b/rust/src/protocol/links_operations.rs index 117adae..c679109 100644 --- a/rust/src/protocol/links_operations.rs +++ b/rust/src/protocol/links_operations.rs @@ -12,7 +12,7 @@ //! | `(count: (index source target))` | `(count: N)` | //! | `(each: (index source target))` | one `(index: source target)` per match | //! | `(create: (source target))` | `() ((index: source target))` | -//! | `(update: (index source target))`| `((index: s t)) ((index: source target))` per change | +//! | `(update: (index source target))`| `((index: s t)) ((index: source target))` per changed link | //! | `(delete: index)` | `((index: source target)) ()` per removed link | //! | `(get-name: link)` | `(name: 'text')`, or nothing | //! | `(set-name: (link 'text'))` | `(link: N)` | @@ -24,9 +24,16 @@ //! `(index value)` links of that index whose source or target is `value`, //! and `(index source target)` matches part by part. `*` matches anything. //! Every reference in a reply is a number; failures are `(error: 'message')`. +//! +//! The changes of a `create`, `update` or `delete` are the net change of every +//! link it touched, the way `clink --changes` reports a query: a cascading +//! delete is one `((index: source target)) ()` per removed link, without the +//! intermediate steps by which the store behind the server got there, which +//! differ between the C# and the Rust stores. use super::error::{ProtocolError, ProtocolResult}; use super::mapping::LinoDocument; +use crate::changes_simplifier::simplify_changes; use crate::link::Link; use crate::named_type_links::NamedTypeLinks; use links_notation::LiNo; @@ -168,14 +175,14 @@ impl LinksOperation { changes.push((before, after)) })?; storage.save()?; - changes_document(&changes) + changes_document(&simplify_changes(changes)) } Self::Delete(index) => { let mut changes = Vec::new(); storage .delete_observed(*index, &mut |before, after| changes.push((before, after)))?; storage.save()?; - changes_document(&changes) + changes_document(&simplify_changes(changes)) } Self::GetName(index) => match storage.get_name(*index)? { Some(name) => vec![named("name", vec![LiNo::Ref(name)])], diff --git a/rust/src/protocol/remote_links.rs b/rust/src/protocol/remote_links.rs index 2ff93b3..0fb1f9c 100644 --- a/rust/src/protocol/remote_links.rs +++ b/rust/src/protocol/remote_links.rs @@ -19,10 +19,12 @@ //! # Ok::<(), anyhow::Error>(()) //! ``` //! -//! Every call is one [`LinksOperation`] round trip. The methods of the -//! upstream and CLI traits that cannot return an error panic when the -//! connection fails, the way a local store panics when its file does; the -//! inherent methods return every failure as a [`ProtocolError`]. +//! Every call is one [`LinksOperation`] round trip. An observer of a write +//! sees the net change of every link it touched, not the steps the store +//! behind the server took. The methods of the upstream and CLI traits that +//! cannot return an error panic when the connection fails, the way a local +//! store panics when its file does; the inherent methods return every failure +//! as a [`ProtocolError`]. use super::client::LinksClient; use super::error::{ProtocolError, ProtocolResult}; diff --git a/rust/tests/remote_links_tests.rs b/rust/tests/remote_links_tests.rs index cfd24a4..646ab45 100644 --- a/rust/tests/remote_links_tests.rs +++ b/rust/tests/remote_links_tests.rs @@ -10,7 +10,7 @@ use link_cli::protocol::{ links_operations::matches, parse_document, LinksOperation, LinoProtocol, RemoteLinks, ServerOptions, TextLinoProtocol, }; -use link_cli::{Link, NamedTypeLinks, NamedTypesDecorator, QueryProcessor}; +use link_cli::{simplify_changes, Link, NamedTypeLinks, NamedTypesDecorator, QueryProcessor}; use std::panic::{self, AssertUnwindSafe}; use tempfile::NamedTempFile; @@ -116,12 +116,10 @@ impl NamedTypeLinksCalls for S { .update_observed(5, 1, 1, &mut |before, after| changes.push((before, after))) .unwrap(); assert_eq!(merged, Link::new(5, 2, 2)); + // A server replies with the net change of every link it touched. assert_eq!( - changes, - vec![ - (Link::new(5, 2, 2), Link::new(5, 0, 0)), - (Link::new(5, 0, 0), Link::null()) - ] + simplify_changes(changes), + vec![(Link::new(5, 2, 2), Link::null())] ); assert!(!self.exists(5)); @@ -324,3 +322,44 @@ fn restrictions_match_like_the_raw_links_interface() { assert_eq!(matches(&link, restriction), *expected, "{restriction:?}"); } } + +/// The requests and replies of `docs/protocol/links-operations.txt`, which +/// the C# tests replay against the C# server too. +fn conversation() -> Vec<(&'static str, String)> { + let lines: Vec<&str> = include_str!("../../docs/protocol/links-operations.txt") + .lines() + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .collect(); + let mut exchanges = Vec::new(); + let mut start = 0; + while start < lines.len() { + let end = (start + 1..lines.len()) + .find(|&line| lines[line].starts_with("> ")) + .unwrap_or(lines.len()); + exchanges.push((&lines[start][2..], lines[start + 1..end].join("\n"))); + start = end; + } + exchanges +} + +#[test] +fn servers_answer_the_shared_conversation_over_every_protocol() { + let conversation = conversation(); + assert_eq!(conversation.len(), 19); + for protocol in protocols() { + let server = RunningServer::start(ServerOptions::default(), true); + let mut client = server.client(protocol); + for (request, reply) in &conversation { + match reply.strip_prefix("! ") { + Some(message) => { + let error = client.query_text(request).unwrap_err().to_string(); + assert!(error.ends_with(message), "{request}: {error}"); + } + None => assert_eq!( + (*request, client.query_text(request).unwrap()), + (*request, reply.clone()) + ), + } + } + } +} From acc9138f24e0d66ccf688fc8abbfb06514546493 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 01:17:15 +0000 Subject: [PATCH 24/44] feat(csharp): RemoteLinks, an INamedTypesLinks served over the LiNo protocols RemoteLinks implements INamedTypesLinks by sending one LinksOperation document per call, the same documents as the Rust RemoteLinks, so a local NamedTypesDecorator and a remote store are swappable. LinksServer answers operation documents next to substitution queries. The tests prove every interface call, the query processor and the raw ILinks extensions give the same answers locally and over every protocol, and replay the shared docs/protocol/links-operations.txt conversation. --- csharp/.changeset/issue-105-remote-links.md | 7 + .../Protocol/LinksOperation.cs | 433 ++++++++++++++++++ .../Protocol/LinksServer.cs | 23 +- .../Protocol/RemoteLinks.cs | 132 ++++++ .../Foundation.Data.Doublets.Cli.Tests.csproj | 4 + .../Protocol/LinksServerTests.cs | 54 +-- .../Protocol/RemoteLinksTests.cs | 428 +++++++++++++++++ .../Protocol/RunningServer.cs | 57 +++ 8 files changed, 1073 insertions(+), 65 deletions(-) create mode 100644 csharp/.changeset/issue-105-remote-links.md create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksOperation.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/RemoteLinks.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RemoteLinksTests.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RunningServer.cs diff --git a/csharp/.changeset/issue-105-remote-links.md b/csharp/.changeset/issue-105-remote-links.md new file mode 100644 index 0000000..43fa091 --- /dev/null +++ b/csharp/.changeset/issue-105-remote-links.md @@ -0,0 +1,7 @@ +--- +'Foundation.Data.Doublets.Cli': minor +--- + +Added `RemoteLinks`, an `INamedTypesLinks` whose links live behind a `LinksServer`. Code written against the links interface, the query processor included, switches from a local file to a server by swapping one value. `LinksServer` now also answers `LinksOperation` documents, one per interface call: `(count: …)`, `(each: …)`, `(create: …)`, `(update: …)`, `(delete: …)`, `(get-name: …)`, `(set-name: …)`, `(get-by-name: …)` and `(remove-name: …)`. These are the documents the Rust `RemoteLinks` sends, so either port's client works with either port's server. Both test suites replay the same recorded conversation, `docs/protocol/links-operations.txt`, to prove it. A write replies with the net change of every link it touched. + +Fixed a crash of a `LinksServer` shut down right after it accepted a connection: the connection thread set an option on the socket the shutdown had already disposed, and the `NullReferenceException` ended the process. diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksOperation.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksOperation.cs new file mode 100644 index 0000000..32514cf --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksOperation.cs @@ -0,0 +1,433 @@ +using System.Globalization; +using Platform.Data; +using Platform.Delegates; +using Platform.Data.Doublets; + +using DoubletLink = Platform.Data.Doublets.Link; +using LinoLink = Link.Foundation.Links.Notation.Link; + +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// +/// The parts of a link a or asks for; +/// a null part matches any value. +/// +/// +/// Matched like the raw links interface matches a query: no parts match every link, (index) one link, +/// (index value) links of that index whose source or target is value, and +/// (index source target) matches part by part. Every link is matched at most once, so a link that +/// uses value as both source and target is counted once, where UnitedMemoryLinks counts its +/// two usages. +/// +public sealed record LinksRestriction(IReadOnlyList Parts) +{ + /// The wire spelling of a part that matches any value. + public const string AnyPart = "*"; + + /// Matches every link. + public static LinksRestriction All { get; } = new(Array.Empty()); + + /// A restriction of the given parts. + public static LinksRestriction Of(params uint?[] parts) => new(parts); + + /// Whether matches this restriction. + public bool Matches(DoubletLink link) + { + static bool Is(uint? part, uint value) => part is not { } expected || expected == value; + return Parts switch + { + [] => true, + [var index] => Is(index, link.Index), + [var index, var value] => Is(index, link.Index) && (Is(value, link.Source) || Is(value, link.Target)), + [var index, var source, var target] => Is(index, link.Index) && Is(source, link.Source) && Is(target, link.Target), + _ => false, + }; + } + + /// + public bool Equals(LinksRestriction? other) => other is not null && Parts.SequenceEqual(other.Parts); + + /// + public override int GetHashCode() => Parts.Aggregate(Parts.Count, HashCode.Combine); + + /// The restriction as it is sent: (index source target) with * for any part. + public override string ToString() => LinoFormat.FormatLink(LinksOperation.Group(Parts.Select(PartLino))); + + internal static LinoLink PartLino(uint? part) => part is { } value ? LinksOperation.Number(value) : LinoFormat.Reference(AnyPart); +} + +/// +/// One call of the links interface () as a LiNo document, so a store +/// behind a is used exactly like a local one through . +/// +/// +/// +/// Every request is one top-level link named after the operation and holding exactly one value. The substitution +/// query language ignores that shape (a query needs a restriction and a substitution), so an operation +/// never collides with a query sent to the same server. The Rust LinksOperation uses the same documents, +/// so clients and servers of both ports talk to each other. +/// +/// +/// RequestReply +/// (count: (index source target))(count: N) +/// (each: (index source target))one (index: source target) per match, ordered by index +/// (create: (source target))() ((index: source target)) +/// (update: (index source target))((index: s t)) ((index: source target)) per change +/// (delete: index)((index: source target)) () per removed link +/// (get-name: link)(name: 'text'), or nothing +/// (set-name: (link 'text'))(link: N) +/// (get-by-name: 'text')(link: N), or nothing +/// (remove-name: link)nothing +/// +/// +/// Every reference in a reply is a number; failures are (error: 'message'). The changes of a +/// create, update or delete are the net change of every link it touched, the way +/// clink --changes reports a query: a cascading delete is one ((index: source target)) () per +/// removed link, without the intermediate steps by which the store behind the server got there. +/// +/// +public abstract record LinksOperation +{ + private LinksOperation() { } + + /// Number of links matching the restriction. + public sealed record Count(LinksRestriction Restriction) : LinksOperation + { + private protected override (string, LinoLink) Request => ("count", RestrictionLino(Restriction)); + + /// + public override IReadOnlyList Execute(INamedTypesLinks links) => + new[] { Named("count", Number((uint)Matching(links, Restriction).Count)) }; + } + + /// Every link matching the restriction, ordered by index. + public sealed record Each(LinksRestriction Restriction) : LinksOperation + { + private protected override (string, LinoLink) Request => ("each", RestrictionLino(Restriction)); + + /// + public override IReadOnlyList Execute(INamedTypesLinks links) => + Matching(links, Restriction).Select(LinkLino).ToList(); + } + + /// Creates a link. + /// + /// A store that keeps doublets unique merges a link created as an existing doublet into it, and the reply then + /// holds no creation. + /// + public sealed record Create(uint Source, uint Target) : LinksOperation + { + private protected override (string, LinoLink) Request => ("create", Numbers(Source, Target)); + + /// + public override IReadOnlyList Execute(INamedTypesLinks links) + { + var changes = Recorded(links, record => links.Create(null, record)); + if (Source != 0 || Target != 0) + { + var index = changes[^1].After.Index; + changes.AddRange(Recorded(links, record => + links.Update(new LinkAddress(index), new DoubletLink(index, Source, Target), record))); + } + return NetChangesDocument(changes); + } + } + + /// + /// Points an existing link at a new source and target; an update that would duplicate an existing link + /// merges into it instead. + /// + public sealed record Update(uint Index, uint Source, uint Target) : LinksOperation + { + private protected override (string, LinoLink) Request => ("update", Numbers(Index, Source, Target)); + + /// + public override IReadOnlyList Execute(INamedTypesLinks links) + { + ThrowIfMissing(links, Index); + return NetChangesDocument(Recorded(links, record => + links.Update(new LinkAddress(Index), new DoubletLink(Index, Source, Target), record))); + } + } + + /// Deletes a link, and every link that still refers to it. + public sealed record Delete(uint Index) : LinksOperation + { + private protected override (string, LinoLink) Request => ("delete", Number(Index)); + + /// + public override IReadOnlyList Execute(INamedTypesLinks links) + { + ThrowIfMissing(links, Index); + return NetChangesDocument(Recorded(links, record => links.Delete(new LinkAddress(Index), record))); + } + } + + /// The name of a link. + public sealed record GetName(uint Link) : LinksOperation + { + private protected override (string, LinoLink) Request => ("get-name", Number(Link)); + + /// + public override IReadOnlyList Execute(INamedTypesLinks links) => + links.GetName(Link) is { } name ? new[] { Named("name", LinoFormat.Reference(name)) } : Array.Empty(); + } + + /// Names a link. + public sealed record SetName(uint Link, string Name) : LinksOperation + { + private protected override (string, LinoLink) Request => + ("set-name", Group(new[] { Number(Link), LinoFormat.Reference(Name) })); + + /// + public override IReadOnlyList Execute(INamedTypesLinks links) => LinkReply(links.SetName(Link, Name)); + } + + /// The link with a name. + public sealed record GetByName(string Name) : LinksOperation + { + private protected override (string, LinoLink) Request => ("get-by-name", LinoFormat.Reference(Name)); + + /// + public override IReadOnlyList Execute(INamedTypesLinks links) => LinkReply(links.GetByName(Name)); + } + + /// Removes the name of a link. + public sealed record RemoveName(uint Link) : LinksOperation + { + private protected override (string, LinoLink) Request => ("remove-name", Number(Link)); + + /// + public override IReadOnlyList Execute(INamedTypesLinks links) + { + links.RemoveName(Link); + return Array.Empty(); + } + } + + /// The operation name and its one argument. + private protected abstract (string Name, LinoLink Argument) Request { get; } + + /// Runs the operation against and returns the reply document. + /// An update or delete names a link that does not exist. + public abstract IReadOnlyList Execute(INamedTypesLinks links); + + /// The request document for this operation. + public IReadOnlyList ToDocument() => new[] { Named(Request.Name, Request.Argument) }; + + /// + /// Recognises an operation request; null for any other document, such as a substitution query. + /// + /// The document names an operation but its argument is malformed. + public static LinksOperation? FromDocument(IReadOnlyList document) + { + ArgumentNullException.ThrowIfNull(document); + if (document is not [{ Id: { } name, Values: [var argument] }]) + { + return null; + } + switch (name) + { + case "count": + return new Count(ParseRestriction(argument)); + case "each": + return new Each(ParseRestriction(argument)); + case "create": + var created = ParseNumbers(argument, 2); + return new Create(created[0], created[1]); + case "update": + var updated = ParseNumbers(argument, 3); + return new Update(updated[0], updated[1], updated[2]); + case "delete": + return new Delete(ParseNumber(argument)); + case "get-name": + return new GetName(ParseNumber(argument)); + case "set-name": + return Parts(argument) is [var link, { Values: null, Id: { } text }] + ? new SetName(ParseNumber(link), text) + : throw MalformedArgument("set-name", argument); + case "get-by-name": + return argument is { Values: null, Id: { } wanted } + ? new GetByName(wanted) + : throw MalformedArgument("get-by-name", argument); + case "remove-name": + return new RemoveName(ParseNumber(argument)); + default: + return null; + } + } + + /// The count of a (count: N) reply. + public static uint ParseCount(IReadOnlyList document) + { + ArgumentNullException.ThrowIfNull(document); + return document is [{ Id: "count", Values: [var count] }] + ? ParseNumber(count) + : throw MalformedReply("count", document); + } + + /// The links of an each reply. + public static IReadOnlyList ParseLinks(IReadOnlyList document) + { + ArgumentNullException.ThrowIfNull(document); + return document.Select(ParseLink).ToList(); + } + + /// + /// The (before) (after) pairs of a create, update or delete reply; a missing side is + /// the null link. + /// + public static IReadOnlyList<(DoubletLink Before, DoubletLink After)> ParseChanges(IReadOnlyList document) + { + ArgumentNullException.ThrowIfNull(document); + return document + .Select(change => Parts(change) is [var before, var after] + ? (ParseChangeSide(before), ParseChangeSide(after)) + : throw MalformedReply("change", document)) + .ToList(); + } + + /// The name of a get-name reply, or null. + public static string? ParseName(IReadOnlyList document) + { + ArgumentNullException.ThrowIfNull(document); + return document switch + { + [] => null, + [{ Id: "name", Values: [{ Values: null, Id: var name }] }] => name, + _ => throw MalformedReply("name", document), + }; + } + + /// The link of a set-name or get-by-name reply, or null. + public static uint? ParseLinkReply(IReadOnlyList document) + { + ArgumentNullException.ThrowIfNull(document); + return document switch + { + [] => null, + [{ Id: "link", Values: [var link] }] => ParseNumber(link), + _ => throw MalformedReply("link", document), + }; + } + + /// A reply listing , one (before) (after) line each. + public static IReadOnlyList ChangesDocument(IEnumerable<(DoubletLink Before, DoubletLink After)> changes) => + changes.Select(change => ChangeLino(change.Before, change.After, LinkLino)).ToList(); + + /// + /// (before) (after), where a side is () for the null link and ((index: source target)) + /// otherwise, with every link written by . + /// + internal static LinoLink ChangeLino(DoubletLink before, DoubletLink after, Func linkLino) + { + LinoLink Side(DoubletLink link) => Group(link.IsNull() ? Array.Empty() : new[] { linkLino(link) }); + return Group(new[] { Side(before), Side(after) }); + } + + /// (index: source target) with plain numbers. + public static LinoLink LinkLino(DoubletLink link) => + Named(link.Index.ToString(CultureInfo.InvariantCulture), Number(link.Source), Number(link.Target)); + + internal static LinoLink Number(uint value) => LinoFormat.Reference(value.ToString(CultureInfo.InvariantCulture)); + + internal static LinoLink Group(IEnumerable values) => LinoFormat.Link(null, values.ToList()); + + private static LinoLink Named(string name, params LinoLink[] values) => LinoFormat.Link(name, values.ToList()); + + private static LinoLink Numbers(params uint[] numbers) => Group(numbers.Select(Number)); + + private static LinoLink RestrictionLino(LinksRestriction restriction) => Group(restriction.Parts.Select(LinksRestriction.PartLino)); + + private static IReadOnlyList LinkReply(uint link) => + link == 0 ? Array.Empty() : new[] { Named("link", Number(link)) }; + + private static List Matching(INamedTypesLinks links, LinksRestriction restriction) + { + var any = links.Constants.Any; + var candidates = restriction.Parts is [{ } index, ..] + ? (links.Exists(index) ? new[] { new DoubletLink(links.GetLink(index)) } : Array.Empty()) + : links.All(new DoubletLink(any, any, any)).Select(link => new DoubletLink(link)); + return candidates.Where(restriction.Matches).OrderBy(link => link.Index).ToList(); + } + + /// + /// The net change of every link a write touched: the steps a store takes to get there differ between the C# and + /// the Rust stores, the net changes do not. + /// + private static IReadOnlyList NetChangesDocument(List<(DoubletLink Before, DoubletLink After)> steps) => + ChangesDocument(ChangesSimplifier.SimplifyChanges(steps)); + + /// Runs and returns every change it reports, in order. + private static List<(DoubletLink Before, DoubletLink After)> Recorded(INamedTypesLinks links, Action> write) + { + var changes = new List<(DoubletLink Before, DoubletLink After)>(); + write((before, after) => + { + changes.Add((new DoubletLink(before), new DoubletLink(after))); + return links.Constants.Continue; + }); + return changes; + } + + private static void ThrowIfMissing(INamedTypesLinks links, uint index) + { + if (!links.Exists(index)) + { + throw new ArgumentException($"Link not found: {index}"); + } + } + + private static DoubletLink ParseChangeSide(LinoLink side) => side switch + { + { Id: null, Values: [] } => default, + { Id: null, Values: [var link] } => ParseLink(link), + // `((index: source target))` loses its wrapper when the side is the single named link itself. + { Id: not null, Values: not null } => ParseLink(side), + _ => throw LinoProtocolException.Malformed($"expected a change side, found {LinoFormat.FormatLink(side)}"), + }; + + private static DoubletLink ParseLink(LinoLink link) => link is { Id: { } index, Values: [var source, var target] } + ? new DoubletLink(ParseNumber(index), ParseNumber(source), ParseNumber(target)) + : throw LinoProtocolException.Malformed($"expected (index: source target), found {LinoFormat.FormatLink(link)}"); + + private static LinksRestriction ParseRestriction(LinoLink argument) + { + var parts = Parts(argument); + if (parts.Count > 3) + { + throw MalformedArgument("restriction", argument); + } + return new LinksRestriction(parts + .Select(part => part is { Values: null, Id: LinksRestriction.AnyPart } ? null : (uint?)ParseNumber(part)) + .ToList()); + } + + private static uint[] ParseNumbers(LinoLink argument, int count) + { + var numbers = Parts(argument).Select(ParseNumber).ToArray(); + return numbers.Length == count ? numbers : throw MalformedArgument($"{count} numbers", argument); + } + + /// + /// The values of an unnamed group; a lone reference is a group of one, since the canonical document model + /// unwraps (x) to x. + /// + private static IReadOnlyList Parts(LinoLink argument) => + argument is { Id: null, Values: { } values } ? values.ToList() : new[] { argument }; + + private static uint ParseNumber(LinoLink value) => value is { Values: null, Id: { } text } + ? ParseNumber(text) + : throw LinoProtocolException.Malformed($"expected a number, found {LinoFormat.FormatLink(value)}"); + + private static uint ParseNumber(string text) => + uint.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out var number) + ? number + : throw LinoProtocolException.Malformed($"expected a number, found '{text}'"); + + private static LinoProtocolException MalformedArgument(string expected, LinoLink argument) => + LinoProtocolException.Malformed($"expected {expected}, found {LinoFormat.FormatLink(argument)}"); + + private static LinoProtocolException MalformedReply(string expected, IReadOnlyList document) => + LinoProtocolException.Malformed($"expected a {expected} reply, found {LinoFormat.FormatDocument(document)}"); +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs index e0dcfa9..d4f4b0c 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs @@ -256,10 +256,11 @@ public static IReadOnlyList ErrorDocument(string message) => /// Executes one request against . /// /// - /// A non-empty document is a substitution query; the reply holds one - /// (before) (after) line per change, exactly like clink --changes. - /// The empty document asks for every link, one (index: source target) - /// per line. Failures produce an . + /// A is one call of the links interface, made by + /// . Any other non-empty document is a substitution + /// query; the reply holds one (before) (after) line per change, exactly + /// like clink --changes. The empty document asks for every link, one + /// (index: source target) per line. Failures produce an . /// public static IReadOnlyList ExecuteRequest( INamedTypesLinks links, @@ -270,6 +271,10 @@ public static IReadOnlyList ExecuteRequest( ArgumentNullException.ThrowIfNull(document); try { + if (LinksOperation.FromDocument(document) is { } operation) + { + return operation.Execute(links); + } if (document.Count == 0) { var any = links.Constants.Any; @@ -291,11 +296,7 @@ public static IReadOnlyList ExecuteRequest( }, }); return ChangesSimplifier.SimplifyChanges(changes) - .Select(change => LinoFormat.Link(null, new List - { - ChangeSide(links, change.Before), - ChangeSide(links, change.After), - })) + .Select(change => LinksOperation.ChangeLino(change.Before, change.After, link => LinkLino(links, link))) .ToList(); } catch (Exception error) when (error is not OutOfMemoryException) @@ -312,10 +313,6 @@ private static LinoLink LinkLino(INamedTypesLinks links, DoubletLink link) LinoFormat.Reference(ReferenceName(links, link.Target)), }); - // `()` for a missing side of a change, `((index: source target))` otherwise. - private static LinoLink ChangeSide(INamedTypesLinks links, DoubletLink link) => - LinoFormat.Link(null, link.IsNull() ? new List() : new List { LinkLino(links, link) }); - private static string ReferenceName(INamedTypesLinks links, uint id) => links.GetName(id) ?? id.ToString(System.Globalization.CultureInfo.InvariantCulture); } diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/RemoteLinks.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/RemoteLinks.cs new file mode 100644 index 0000000..9900f94 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/RemoteLinks.cs @@ -0,0 +1,132 @@ +using Platform.Data; +using Platform.Delegates; + +using DoubletLink = Platform.Data.Doublets.Link; +using LinoLink = Link.Foundation.Links.Notation.Link; + +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// +/// A links store served by a , usable wherever a local store is. +/// +/// +/// +/// It implements the same interface as a local store, , so code written +/// against it — the and every ILinks extension included — switches +/// from a local file to a server by swapping one value: +/// +/// +/// static void Run(INamedTypesLinks<uint> links) => +/// AdvancedMixedQueryProcessor.ProcessQuery(links, new() { Query = "() ((1 1))" }); +/// +/// using (var local = new NamedTypesDecorator<uint>("db.links")) Run(local); +/// using (var remote = RemoteLinks.Connect("127.0.0.1:8080", new TextLinoProtocol())) Run(remote); +/// +/// +/// Every call is one round trip, in the documents the Rust RemoteLinks sends, +/// so either port's client works with either port's server. A write handler sees the net change of every link a +/// write touched, not the steps the store behind the server took. A failure of the connection or of the server is a +/// . The calls are serialised, so one instance may be shared between threads. +/// +/// +public sealed class RemoteLinks : INamedTypesLinks, IDisposable +{ + private readonly LinksClient _client; + private readonly object _clientLock = new(); + + /// Uses an existing connection, which the store then owns. + public RemoteLinks(LinksClient client) + { + ArgumentNullException.ThrowIfNull(client); + _client = client; + } + + /// Connects to a server at an address written as host:port. + public static RemoteLinks Connect(string endPoint, ILinoProtocol protocol) => new(LinksClient.Connect(endPoint, protocol)); + + /// The constants of the UnitedMemoryLinks a server stores its links in. + public LinksConstants Constants { get; } = new(); + + /// Runs one operation on the server and returns its reply. + public IReadOnlyList Execute(LinksOperation operation) + { + ArgumentNullException.ThrowIfNull(operation); + lock (_clientLock) + { + return _client.Request(operation.ToDocument()); + } + } + + /// + public uint Count(IList? restriction) => + LinksOperation.ParseCount(Execute(new LinksOperation.Count(Restriction(restriction)))); + + /// + public uint Each(IList? restriction, ReadHandler? handler) + { + foreach (var link in LinksOperation.ParseLinks(Execute(new LinksOperation.Each(Restriction(restriction))))) + { + if (handler?.Invoke(link) == Constants.Break) + { + return Constants.Break; + } + } + return Constants.Continue; + } + + /// Creates an empty (index: 0 0) link, like every UnitedMemoryLinks. + public uint Create(IList? substitution, WriteHandler? handler) => + Replay(new LinksOperation.Create(0, 0), handler); + + /// Updates the link at the index of to (index source target). + /// The restriction or substitution is not of that shape. + public uint Update(IList? restriction, IList? substitution, WriteHandler? handler) => + (restriction, substitution) is ([var index, ..], [_, var source, var target]) + ? Replay(new LinksOperation.Update(index, source, target), handler) + : throw new ArgumentException("an update needs an index and an (index source target) substitution", nameof(substitution)); + + /// Deletes the link at the index of , and every link that refers to it. + /// The restriction holds no index. + public uint Delete(IList? restriction, WriteHandler? handler) => + restriction is [var index, ..] + ? Replay(new LinksOperation.Delete(index), handler) + : throw new ArgumentException("a delete needs an index", nameof(restriction)); + + /// + public string? GetName(uint link) => LinksOperation.ParseName(Execute(new LinksOperation.GetName(link))); + + /// + public uint SetName(uint link, string name) => + LinksOperation.ParseLinkReply(Execute(new LinksOperation.SetName(link, name))) + ?? throw new LinoProtocolException(LinoProtocolErrorKind.Malformed, "the set-name reply holds no link"); + + /// The link named , or 0 when there is none. + public uint GetByName(string name) => LinksOperation.ParseLinkReply(Execute(new LinksOperation.GetByName(name))) ?? 0; + + /// + public void RemoveName(uint link) => Execute(new LinksOperation.RemoveName(link)); + + /// Closes the connection. + public void Dispose() => _client.Dispose(); + + private LinksRestriction Restriction(IList? restriction) => + new((restriction ?? Array.Empty()).Select(part => part == Constants.Any ? null : (uint?)part).ToList()); + + /// + /// Runs a create, update or delete and feeds its changes to until + /// it asks to stop, passing a missing side as null the way UnitedMemoryLinks does. + /// + private uint Replay(LinksOperation operation, WriteHandler? handler) + { + foreach (var (before, after) in LinksOperation.ParseChanges(Execute(operation))) + { + if (handler?.Invoke(Side(before), Side(after)) == Constants.Break) + { + return Constants.Break; + } + } + return Constants.Continue; + } + + private static IList? Side(DoubletLink link) => link.IsNull() ? null : (IList)link; +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj b/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj index c50cfe8..28f0cf9 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj @@ -20,6 +20,10 @@ + + + + diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs index f5fcef7..3637da0 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs @@ -7,56 +7,6 @@ namespace Foundation.Data.Doublets.Cli.Tests.Protocol; /// LiNo substitution operations over TCP (issue #105). public sealed class LinksServerTests { - private sealed class RunningServer : IDisposable - { - private readonly string _databaseFilename = Path.GetTempFileName(); - private readonly LinksServer _server; - private readonly Thread _thread; - private Exception? _failure; - - public RunningServer(LinksServerOptions? options = null) - { - _server = LinksServer.Bind("127.0.0.1", 0, options); - _thread = new Thread(() => - { - try - { - using var links = new NamedTypesDecorator(_databaseFilename); - _server.Serve(links); - } - catch (Exception error) - { - _failure = error; - } - }) - { IsBackground = true }; - _thread.Start(); - } - - public int Port => _server.LocalEndPoint.Port; - - public LinksClient Client(ILinoProtocol protocol) => LinksClient.Connect("127.0.0.1", Port, protocol); - - public void Stop() - { - _server.Shutdown(); - Assert.True(_thread.Join(TimeSpan.FromSeconds(30)), "the server did not stop"); - Assert.Null(_failure); - } - - public void Dispose() - { - Stop(); - File.Delete(_databaseFilename); - File.Delete(NamedTypesDecorator.MakeNamesDatabaseFilename(_databaseFilename)); - } - } - - private static IEnumerable Protocols() => - LinoProtocolCodecTests.AllOptions() - .Select(options => (ILinoProtocol)new BinaryLinoProtocol(options)) - .Prepend(new TextLinoProtocol()); - private static TcpClient RawConnection(RunningServer server) { return new TcpClient("127.0.0.1", server.Port) { ReceiveTimeout = 10_000 }; @@ -65,7 +15,7 @@ private static TcpClient RawConnection(RunningServer server) [Fact] public void CrudWorksOverEveryProtocol() { - foreach (var protocol in Protocols()) + foreach (var protocol in RunningServer.Protocols()) { using var server = new RunningServer(); using var client = server.Client(protocol); @@ -102,7 +52,7 @@ public void TextAndBinaryRepliesAreIdenticalDocuments() public void QueryErrorsComeBackAsRemoteErrors() { using var server = new RunningServer(); - foreach (var protocol in Protocols()) + foreach (var protocol in RunningServer.Protocols()) { using var client = server.Client(protocol); var error = Assert.Throws(() => client.Query("((99: 1 1)) ()")); diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RemoteLinksTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RemoteLinksTests.cs new file mode 100644 index 0000000..9a752ac --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RemoteLinksTests.cs @@ -0,0 +1,428 @@ +using Foundation.Data.Doublets.Cli.Protocol; +using Platform.Data; +using Platform.Data.Doublets; +using Platform.Delegates; + +using DoubletLink = Platform.Data.Doublets.Link; + +namespace Foundation.Data.Doublets.Cli.Tests.Protocol; + +/// is a drop-in replacement for a local store over every protocol. +public sealed class RemoteLinksTests +{ + /// + /// The README walkthrough plus the cases that cascade: a merge into an existing doublet and the deletion of a + /// link that others use. + /// + private static readonly string[] Queries = + { + "() ((1 1))", + "() ((2 2))", + "((1: 1 1)) ((1: 1 2))", + "((1: 1 2)) ((1: 2 1))", + "((($i: $s $t)) (($i: $t $s)))", + "((2: 2 2)) ((2: 1 1))", + "() ((father: father father))", + "() ((mother: mother mother))", + "() ((child: father mother))", + "((child: father mother)) ((child: father mother))", + "((child: father mother)) ((child: mother father))", + "((father: father father)) ()", + "((1 *)) ()", + }; + + private sealed class LocalStore : IDisposable + { + private readonly string _databaseFilename = Path.GetTempFileName(); + + public LocalStore() => Links = new NamedTypesDecorator(_databaseFilename); + + public NamedTypesDecorator Links { get; } + + public void Dispose() + { + Links.Dispose(); + File.Delete(_databaseFilename); + File.Delete(NamedTypesDecorator.MakeNamesDatabaseFilename(_databaseFilename)); + } + } + + /// The net changes of every query, the way clink --changes reports them. + private static List> RunQueries(INamedTypesLinks links) => + Queries.Select(query => NetChanges(links, record => AdvancedMixedQueryProcessor.ProcessQuery(links, new AdvancedMixedQueryProcessor.Options + { + Query = query, + AutoCreateMissingReferences = true, + ChangesHandler = record, + }))).ToList(); + + private static List<(DoubletLink Before, DoubletLink After)> NetChanges(INamedTypesLinks links, Action> write) + { + var steps = new List<(DoubletLink Before, DoubletLink After)>(); + write((before, after) => + { + steps.Add((new DoubletLink(before), new DoubletLink(after))); + return links.Constants.Continue; + }); + return ChangesSimplifier.SimplifyChanges(steps).ToList(); + } + + /// Every link with its name, the way clink --after prints the store. + private static List NamedLinks(INamedTypesLinks links) + { + string Part(uint link) => links.GetName(link) ?? link.ToString(); + return links.All(new DoubletLink(links.Constants.Any, links.Constants.Any, links.Constants.Any)) + .Select(link => new DoubletLink(link)) + .Select(link => $"({Part(link.Index)}: {Part(link.Source)} {Part(link.Target)})") + .ToList(); + } + + [Fact] + public void TheQueryProcessorGivesTheSameResultsOnARemoteStore() + { + using var local = new LocalStore(); + var expectedChanges = RunQueries(local.Links); + var expectedLinks = NamedLinks(local.Links); + // Deleting father cascades to child, and ((1 *)) () deletes what is left of the numbered links. + Assert.Equal(new[] { "(mother: mother mother)" }, expectedLinks); + + foreach (var protocol in RunningServer.Protocols()) + { + using var server = new RunningServer(); + using var remote = server.Remote(protocol); + + Assert.Equal(expectedChanges, RunQueries(remote)); + Assert.Equal(expectedLinks, NamedLinks(remote)); + } + } + + /// + /// Runs every call, through the extensions code uses it by, and + /// returns what each call answered, so a remote store is held to exactly the answers of a local one. + /// + private static List Exercise(INamedTypesLinks links) + { + var answers = new List(); + void Write(Func, uint> write) => + answers.AddRange(NetChanges(links, record => answers.Add(write(record))).Cast()); + + answers.Add(links.Create()); + answers.Add(links.GetOrCreate(1u, 1u)); + answers.Add(links.GetOrCreate(1u, 1u)); + answers.Add(links.SearchOrDefault(1u, 1u)); + answers.Add(links.SearchOrDefault(2u, 2u)); + answers.Add(new DoubletLink(links.GetLink(2u))); + answers.Add(links.Exists(1u)); + answers.Add(links.Exists(9u)); + answers.Add(links.Count()); + + answers.Add(links.Create()); + answers.Add(links.Update(3u, 2u, 2u)); + Write(record => links.Update(new LinkAddress(3), new DoubletLink(3, 1, 1), record)); + answers.Add(links.Exists(3u)); + + answers.Add(links.SetName(2, "pair")); + answers.Add(links.GetName(2)); + answers.Add(links.GetByName("pair")); + answers.Add(links.GetByName("none")); + links.RemoveName(2); + answers.Add(links.GetName(2)); + + Write(record => links.Delete(new LinkAddress(1), record)); + answers.Add(links.Count()); + return answers; + } + + [Fact] + public void EveryNamedTypesLinksCallMatchesALocalStore() + { + using var local = new LocalStore(); + var expected = Exercise(local.Links); + // Updating (3: 2 2) into the existing (2: 1 1) merges it away. + Assert.Contains((new DoubletLink(3, 2, 2), default(DoubletLink)), expected); + // Deleting 1 deletes (2: 1 1), which uses it, too. + Assert.Equal( + new object?[] { (new DoubletLink(2, 1, 1), default(DoubletLink)), (new DoubletLink(1, 0, 0), default(DoubletLink)) }, + expected.SkipLast(1).TakeLast(2)); + + foreach (var protocol in RunningServer.Protocols()) + { + using var server = new RunningServer(); + using var remote = server.Remote(protocol); + + Assert.Equal(expected, Exercise(remote)); + } + } + + /// + /// The requests and replies of docs/protocol/links-operations.txt, which the Rust tests replay against + /// the Rust server too. + /// + private static IEnumerable<(string Request, string Reply)> Conversation() + { + var lines = File.ReadAllLines(Path.Combine(AppContext.BaseDirectory, "Protocol", "links-operations.txt")) + .Where(line => line.Length > 0 && !line.StartsWith('#')) + .ToList(); + for (var start = 0; start < lines.Count;) + { + var end = lines.FindIndex(start + 1, line => line.StartsWith("> ", StringComparison.Ordinal)); + end = end < 0 ? lines.Count : end; + yield return (lines[start][2..], string.Join('\n', lines.Skip(start + 1).Take(end - start - 1))); + start = end; + } + } + + [Fact] + public void ServersAnswerTheSharedConversationOverEveryProtocol() + { + var conversation = Conversation().ToList(); + Assert.Equal(19, conversation.Count); + foreach (var protocol in RunningServer.Protocols()) + { + using var server = new RunningServer(); + using var client = server.Client(protocol); + foreach (var (request, reply) in conversation) + { + if (reply.StartsWith("! ", StringComparison.Ordinal)) + { + var error = Assert.Throws(() => client.QueryText(request)); + Assert.EndsWith(reply[2..], error.Message); + } + else + { + Assert.Equal((request, reply), (request, client.QueryText(request))); + } + } + } + } + + [Fact] + public void TheRawLinksInterfaceWorksOverEveryProtocol() + { + foreach (var protocol in RunningServer.Protocols()) + { + using var server = new RunningServer(); + using var remote = server.Remote(protocol); + var constants = remote.Constants; + var any = constants.Any; + + var created = new List<(IList? Before, IList? After)>(); + var flow = remote.Create(null, (before, after) => + { + created.Add((before, after)); + return constants.Continue; + }); + Assert.Equal(constants.Continue, flow); + Assert.Equal(new (IList?, IList?)[] { (null, new DoubletLink(1, 0, 0)) }, created); + remote.Create(Array.Empty(), null); + + remote.Update(new uint[] { 1 }, new uint[] { 1, 1, 2 }, null); + remote.Update(new uint[] { 2 }, new uint[] { 2, 2, 1 }, null); + Assert.Equal(new DoubletLink(1, 1, 2), new DoubletLink(remote.GetLink(1u))); + + Assert.Equal(2u, remote.Count(null)); + Assert.Equal(2u, remote.Count(new[] { any })); + Assert.Equal(1u, remote.Count(new uint[] { 2 })); + Assert.Equal(0u, remote.Count(new uint[] { 3 })); + Assert.Equal(2u, remote.Count(new[] { any, 1u })); + Assert.Equal(1u, remote.Count(new[] { any, 1u, any })); + Assert.Equal(1u, remote.Count(new[] { any, any, 1u })); + Assert.Equal(0u, remote.Count(new[] { any, 2u, 2u })); + + var seen = new List(); + flow = remote.Each(new[] { any, any, any }, link => + { + seen.Add(new DoubletLink(link)); + return constants.Break; + }); + Assert.Equal(constants.Break, flow); + Assert.Equal(new[] { new DoubletLink(1, 1, 2) }, seen); + Assert.Equal(constants.Continue, remote.Each(null, null)); + + var deleted = new List<(IList? Before, IList? After)>(); + flow = remote.Delete(new uint[] { 1 }, (before, after) => + { + deleted.Add((before, after)); + return constants.Break; + }); + Assert.Equal(constants.Break, flow); + Assert.Single(deleted); + Assert.Equal(0u, remote.Count(null)); + } + } + + [Fact] + public void MisshapenCallsAreRejectedBeforeAnythingIsSent() + { + using var server = new RunningServer(); + using var remote = server.Remote(new TextLinoProtocol()); + + Assert.Throws(() => remote.Update(Array.Empty(), new uint[] { 1, 1, 1 }, null)); + Assert.Throws(() => remote.Update(new uint[] { 1 }, new uint[] { 1, 1 }, null)); + Assert.Throws(() => remote.Delete(null, null)); + Assert.Throws(() => remote.Execute(null!)); + Assert.Throws(() => new RemoteLinks(null!)); + } + + [Fact] + public void ServerFailuresAreProtocolExceptions() + { + using var server = new RunningServer(); + using var remote = server.Remote(new TextLinoProtocol()); + + var missing = Assert.Throws(() => remote.Update(7u, 1u, 1u)); + Assert.Equal(LinoProtocolErrorKind.Remote, missing.Kind); + Assert.EndsWith("Link not found: 7", missing.Message); + Assert.Throws(() => remote.Delete(7u)); + var tooLong = Assert.Throws(() => remote.Count(new uint[] { 1, 2, 3, 4 })); + Assert.Equal(LinoProtocolErrorKind.Remote, tooLong.Kind); + Assert.Equal(0u, remote.Count(null)); + } + + [Fact] + public void ALostConnectionThrows() + { + using var server = new RunningServer(); + using var remote = server.Remote(new TextLinoProtocol()); + server.Stop(); + + Assert.Throws(() => remote.Count(null)); + } + + [Fact] + public void OperationsRoundTripThroughTheirDocuments() + { + LinksOperation[] operations = + { + new LinksOperation.Count(LinksRestriction.All), + new LinksOperation.Count(LinksRestriction.Of(1)), + new LinksOperation.Each(LinksRestriction.Of(null, 2)), + new LinksOperation.Each(LinksRestriction.Of(1, null, 3)), + new LinksOperation.Create(1, 2), + new LinksOperation.Update(3, 4, 5), + new LinksOperation.Delete(6), + new LinksOperation.GetName(7), + new LinksOperation.SetName(8, "a name"), + new LinksOperation.GetByName("a name"), + new LinksOperation.RemoveName(9), + }; + foreach (var protocol in RunningServer.Protocols()) + { + foreach (var operation in operations) + { + using var wire = new MemoryStream(); + protocol.WriteDocument(wire, operation.ToDocument()); + var document = protocol.ReadDocument(LinoStreamReader.FromBytes(wire.ToArray()))!; + + Assert.Equal(operation, LinksOperation.FromDocument(document)); + } + } + } + + [Theory] + [InlineData("")] + [InlineData("() ((1 1))")] + [InlineData("((1: 1 1)) ()")] + [InlineData("(unknown: 1)")] + [InlineData("(count: 1 2)")] + [InlineData("(count: 1)\n(count: 1)")] + public void OnlyOperationShapedDocumentsAreOperations(string query) => + Assert.Null(LinksOperation.FromDocument(LinoFormat.ParseDocument(query))); + + [Theory] + [InlineData("(count: (1 2 3 4))")] + [InlineData("(each: (x))")] + [InlineData("(create: (1))")] + [InlineData("(update: (1 2))")] + [InlineData("(delete: x)")] + [InlineData("(delete: -1)")] + [InlineData("(get-name: (1 2))")] + [InlineData("(set-name: (1))")] + [InlineData("(set-name: (1 (a b)))")] + [InlineData("(get-by-name: (a b))")] + [InlineData("(remove-name: 4294967296)")] + public void MalformedOperationsAreRejected(string query) + { + var error = Assert.Throws(() => LinksOperation.FromDocument(LinoFormat.ParseDocument(query))); + Assert.Equal(LinoProtocolErrorKind.Malformed, error.Kind); + } + + [Theory] + [InlineData("(count: 1 2)")] + [InlineData("(count: x)")] + [InlineData("(1 2)")] + [InlineData("((1 2) x)")] + public void MalformedRepliesAreRejected(string reply) + { + var document = LinoFormat.ParseDocument(reply); + Assert.Throws(() => LinksOperation.ParseCount(document)); + Assert.Throws(() => LinksOperation.ParseChanges(document)); + } + + [Fact] + public void RepliesParseBackToWhatTheyCarry() + { + Assert.Equal(3u, LinksOperation.ParseCount(LinoFormat.ParseDocument("(count: 3)"))); + Assert.Equal( + new[] { new DoubletLink(1, 2, 3) }, + LinksOperation.ParseLinks(LinoFormat.ParseDocument("(1: 2 3)"))); + Assert.Equal( + new[] { (default(DoubletLink), new DoubletLink(1, 0, 0)), (new DoubletLink(1, 0, 0), default) }, + LinksOperation.ParseChanges(LinoFormat.ParseDocument("() ((1: 0 0))\n((1: 0 0)) ()"))); + Assert.Equal("a name", LinksOperation.ParseName(LinoFormat.ParseDocument("(name: 'a name')"))); + Assert.Null(LinksOperation.ParseName(Array.Empty>())); + Assert.Equal(5u, LinksOperation.ParseLinkReply(LinoFormat.ParseDocument("(link: 5)"))); + Assert.Null(LinksOperation.ParseLinkReply(Array.Empty>())); + } + + [Theory] + [InlineData(new uint[0], true)] + [InlineData(new[] { Any }, true)] + [InlineData(new uint[] { 1 }, true)] + [InlineData(new uint[] { 2 }, false)] + [InlineData(new[] { Any, 2u }, true)] + [InlineData(new[] { Any, 3u }, true)] + [InlineData(new[] { Any, 1u }, false)] + [InlineData(new uint[] { 1, 2, 3 }, true)] + [InlineData(new[] { Any, 3u, Any }, false)] + [InlineData(new[] { Any, Any, Any, Any }, false)] + public void RestrictionsMatchLikeTheRawLinksInterface(uint[] parts, bool matches) + { + var restriction = new LinksRestriction(parts.Select(part => part == Any ? null : (uint?)part).ToList()); + + Assert.Equal(matches, restriction.Matches(new DoubletLink(1, 2, 3))); + } + + private const uint Any = uint.MaxValue - 3; + + [Fact] + public void RestrictionsAreValuesWrittenAsLinoGroups() + { + Assert.Equal(LinksRestriction.Of(1, null, 3), new LinksRestriction(new uint?[] { 1, null, 3 }.ToList())); + Assert.Equal(LinksRestriction.Of(1, null, 3).GetHashCode(), LinksRestriction.Of(1, null, 3).GetHashCode()); + Assert.NotEqual(LinksRestriction.Of(1, null), LinksRestriction.Of(1, null, 3)); + Assert.False(LinksRestriction.All.Equals(null)); + Assert.Equal("(1 * 3)", LinksRestriction.Of(1, null, 3).ToString()); + } + + /// + /// Restricting by source or target visits a link once even when both halves match, where + /// UnitedMemoryLinks visits a self-referencing link once per usage. + /// + [Fact] + public void ALinkUsingAnAddressTwiceMatchesOnce() + { + using var local = new LocalStore(); + var point = local.Links.GetOrCreate(1u, 1u); + Assert.Equal(1u, point); + var any = local.Links.Constants.Any; + Assert.Equal(2u, local.Links.Count(new[] { any, point })); + + using var server = new RunningServer(); + using var remote = server.Remote(new TextLinoProtocol()); + remote.GetOrCreate(1u, 1u); + + Assert.Equal(1u, remote.Count(new[] { any, point })); + Assert.Equal(new[] { new DoubletLink(1, 1, 1) }, remote.All(new[] { any, point }).Select(link => new DoubletLink(link))); + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RunningServer.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RunningServer.cs new file mode 100644 index 0000000..6cdb639 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RunningServer.cs @@ -0,0 +1,57 @@ +using Foundation.Data.Doublets.Cli.Protocol; + +namespace Foundation.Data.Doublets.Cli.Tests.Protocol; + +/// A serving a fresh temporary store on a background thread. +internal sealed class RunningServer : IDisposable +{ + private readonly string _databaseFilename = Path.GetTempFileName(); + private readonly LinksServer _server; + private readonly Thread _thread; + private Exception? _failure; + + public RunningServer(LinksServerOptions? options = null) + { + _server = LinksServer.Bind("127.0.0.1", 0, options); + _thread = new Thread(() => + { + try + { + using var links = new NamedTypesDecorator(_databaseFilename); + _server.Serve(links); + } + catch (Exception error) + { + _failure = error; + } + }) + { IsBackground = true }; + _thread.Start(); + } + + public int Port => _server.LocalEndPoint.Port; + + public LinksClient Client(ILinoProtocol protocol) => LinksClient.Connect("127.0.0.1", Port, protocol); + + public RemoteLinks Remote(ILinoProtocol protocol) => new(Client(protocol)); + + public void Stop() + { + _server.Shutdown(); + Assert.True(_thread.Join(TimeSpan.FromSeconds(30)), "the server did not stop"); + Assert.Null(_failure); + } + + public void Dispose() + { + Stop(); + File.Delete(_databaseFilename); + File.Delete(NamedTypesDecorator.MakeNamesDatabaseFilename(_databaseFilename)); + } + + /// The text protocol and the binary protocol with every option. + public static IEnumerable Protocols() => + LinoProtocolCodecTests.AllOptions() + .Select(options => (ILinoProtocol)new BinaryLinoProtocol(options)) + .Prepend(new TextLinoProtocol()); +} From 48d362ec273671cd828eb1df92c954bf553170ab Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 01:18:35 +0000 Subject: [PATCH 25/44] docs: describe RemoteLinks and the links operations it sends --- README.md | 31 +++++++++++++++++++ docs/case-studies/issue-105/README.md | 43 +++++++++++++++++++++++++-- examples/tcp/README.md | 12 ++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b568479..8aa5af8 100644 --- a/README.md +++ b/README.md @@ -451,6 +451,37 @@ C# and Rust servers and clients work with each other: see [`examples/tcp/`](examples/tcp) and [docs/case-studies/issue-105](docs/case-studies/issue-105/README.md). +### Use a served database as a store + +In both libraries, `RemoteLinks` is a links store whose links live behind a +server. It implements the same interfaces as a local store: `INamedTypesLinks` +in C#, and `Links`, `Doublets` and `NamedTypeLinks` in Rust. So code written +against them, the query processor included, switches from a local file to a +server by swapping one value: + +```csharp +static void Run(INamedTypesLinks links) => + AdvancedMixedQueryProcessor.ProcessQuery(links, new() { Query = "() ((1 1))" }); + +using (var local = new NamedTypesDecorator("db.links")) Run(local); +using (var remote = RemoteLinks.Connect("127.0.0.1:7777", new TextLinoProtocol())) Run(remote); +``` + +```rust +fn run(store: &mut impl NamedTypeLinks) -> anyhow::Result<()> { + QueryProcessor::new(false).process_query(store, "() ((1 1))")?; + Ok(()) +} + +run(&mut NamedTypesDecorator::new("db.links", false)?)?; +run(&mut RemoteLinks::connect("127.0.0.1:7777", TextLinoProtocol::new())?)?; +``` + +Each call is one request, such as `(count: (* 1))`, `(update: (1 2 2))` or +`(set-name: (2 'a pair'))`. A write replies with the net change of every link +it touched. [`docs/protocol/links-operations.txt`](docs/protocol/links-operations.txt) +records one whole conversation, and both ports replay it in their tests. + ## Update single link Update link with index 1 and source 1 and target 1, changing target to 2. diff --git a/docs/case-studies/issue-105/README.md b/docs/case-studies/issue-105/README.md index b5b98d4..24e012e 100644 --- a/docs/case-studies/issue-105/README.md +++ b/docs/case-studies/issue-105/README.md @@ -28,7 +28,7 @@ decorator, in both the C# and the Rust port. | # | Requirement (from the issue text) | Where it is addressed | |---|---|---| -| R1 | "lightweight protocols to transfer links notation representing substitution operations to allow all CRUD operations over TCP/IP" | `LinksServer` / `LinksClient` in both ports. A request is a substitution query and the empty request reads everything (§4.4). `clink --serve` / `--connect` (§4.5) | +| R1 | "lightweight protocols to transfer links notation representing substitution operations to allow all CRUD operations over TCP/IP" | `LinksServer` / `LinksClient` in both ports. A request is a substitution query and the empty request reads everything (§4.4). `clink --serve` / `--connect` (§4.6). `RemoteLinks` serves the links interface itself (§4.5) | | R2 | "UTF-8 based text only links notation" | `TextLinoProtocol` (§4.1) | | R3 | "binary version of links notation, that will have number of links in a message/packet" | `BinaryLinoProtocol` and `LinksPacket`. The header carries N (doublets) and M (sequences) as LEB128 (§4.2) | | R4 | "0-256 links have 8 bit size. 257-65536 links have 16 bits size and so on" (8/16/32/64) | The width is the tier of the largest address: 1, 2, 4 or 8 bytes. `--progressive-widths` grows the width per address instead (§4.2.3) | @@ -267,7 +267,42 @@ server to one protocol. Requests run one at a time under a store lock. Each connection has its own thread for parsing and formatting. -### 4.5 Command line +### 4.5 Remote stores + +A substitution query is one way to use a store; the links interface is the +other. `RemoteLinks` implements that interface — `INamedTypesLinks` in +C#, `Links`, `Doublets` and `NamedTypeLinks` in Rust — by sending one +`LinksOperation` document per call. So a local store and a served one are +swappable, the query processor included: + +| Request | Reply | +|---|---| +| `(count: (index source target))` | `(count: N)` | +| `(each: (index source target))` | one `(index: source target)` per match | +| `(create: (source target))` | `() ((index: source target))` | +| `(update: (index source target))` | `((index: s t)) ((index: source target))` per changed link | +| `(delete: index)` | `((index: source target)) ()` per removed link | +| `(get-name: link)` | `(name: 'text')`, or nothing | +| `(set-name: (link 'text'))` | `(link: N)`, the address of the name | +| `(get-by-name: 'text')` | `(link: N)`, or nothing | +| `(remove-name: link)` | nothing | + +A restriction holds up to three parts, and `*` matches anything. An operation +holds exactly one value, and a substitution query needs a restriction *and* a +substitution, so the two never collide on one server. + +A write replies with the **net change** of every link it touched, the way +`clink --changes` reports a query, not with the steps the store took. The +steps differ between the ports: a cascading delete in C# clears a usage to +`(0 0)` before it removes it, while doublets-rs 0.5.0 removes it directly. +The net changes are the same in both, so the replies are too. + +[`docs/protocol/links-operations.txt`](../../protocol/links-operations.txt) +records one conversation with a fresh server, covering every operation, a +cascade and the errors. Both test suites replay it over every protocol, so a +client of either port is proven to understand a server of the other. + +### 4.6 Command line ```bash clink --db data.links --serve 127.0.0.1:7777 [--protocol text|binary] [--auto-create-missing-references] [--trace] @@ -340,9 +375,11 @@ structure, not that it is smaller. - `rust/tests/protocol_packet_tests.rs`: codecs, the golden vectors, the corpus round trip under all 8 option sets, limits, malformed input, and deep nesting in linear time. - `rust/tests/protocol_tcp_tests.rs`: CRUD over every protocol, mixed clients, errors, CRLF, malformed input, protocol restriction, concurrency and shutdown. - `rust/tests/cli_tcp_tests.rs`: `clink --serve` / `--connect` end to end. + - `rust/tests/remote_links_tests.rs`: the same proof for the Rust `RemoteLinks`, including the shared conversation. - C#: - `LinoProtocolCodecTests` (34 tests): the same corpus and golden vectors. - - `LinksServerTests` (9 tests): CRUD over every protocol, identical text and binary replies, errors, CRLF, malformed input, protocol restriction, concurrent clients and shutdown. + - `LinksServerTests` (10 tests): CRUD over every protocol, identical text and binary replies, errors, CRLF, malformed input, protocol restriction, concurrent clients, shutdown, and shutdown racing a new connection. + - `RemoteLinksTests`: every `INamedTypesLinks` call, the query processor and the raw `ILinks` interface give the same answers on a local store and over every protocol. It also covers the shared conversation, malformed operations and replies, and lost connections. - `CliTcpIntegrationTests` (8 tests): `clink --serve` / `--connect` end to end. - Cross-language interop: [`examples/tcp/run-interop.sh`](../../../examples/tcp/run-interop.sh) runs a Rust server with a C# client, and a C# server with a Rust client. It covers text and every binary option, and both directions print identical results. diff --git a/examples/tcp/README.md b/examples/tcp/README.md index 6fee685..31de366 100644 --- a/examples/tcp/README.md +++ b/examples/tcp/README.md @@ -50,5 +50,17 @@ the server with `nc 127.0.0.1 7777`: . ``` +Besides queries, a server answers one request per call of the links +interface, which is what `RemoteLinks` sends. After `() ((1 1) (2 2) (1 2))`, +two links use `1`: + +```bash +clink --connect 127.0.0.1:7777 '(count: (* 1))' +(count: 2) +``` + +[`docs/protocol/links-operations.txt`](../../docs/protocol/links-operations.txt) +lists every such request with its reply. + The wire formats are described in [`docs/case-studies/issue-105`](../../docs/case-studies/issue-105/README.md). From 3819bec3d024de60057e206a385beef6690379a2 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 01:21:20 +0000 Subject: [PATCH 26/44] test: prove that a delete cascades through usages in C#, and document why Rust already had processor-level tests for the cascade; C# now has the same two. HOW-IT-WORKS explains that a link never refers to an address that no longer exists. The concurrency test of the server now checks the reply to named creations, which the C# processor reports since d4cec67. --- .../AdvancedMixedQueryProcessor.Cascades.cs | 45 +++++++++++++++++++ .../Protocol/LinksServerTests.cs | 4 +- docs/HOW-IT-WORKS.md | 10 +++++ docs/case-studies/issue-105/README.md | 8 +++- 4 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.Cascades.cs diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.Cascades.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.Cascades.cs new file mode 100644 index 0000000..279c96c --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/AdvancedMixedQueryProcessor.Cascades.cs @@ -0,0 +1,45 @@ +// Deleting a link deletes every link that uses it, transitively: a link never refers to an address that does not +// exist. The Rust port does the same (test_delete_cascade_chain_matches_csharp). +using DoubletLink = Platform.Data.Doublets.Link; + +namespace Foundation.Data.Doublets.Cli.Tests.Tests +{ + public partial class AdvancedMixedQueryProcessor + { + [Fact] + public void DeleteCascadesToUsagesTest() + { + RunTestWithLinks(links => + { + ProcessQuery(links, "() ((1 1) (2 2) (1 2))"); + + var changes = ReportedChanges(links, "((2: 2 2)) ()"); + + Assert.Equal( + new[] + { + (new DoubletLink(2, 2, 2), default(DoubletLink)), + (new DoubletLink(3, 1, 2), default(DoubletLink)), + }, + changes.OrderBy(change => change.Item1.Index)); + Assert.Equal(new[] { new DoubletLink(1, 1, 1) }, GetAllLinks(links)); + }); + } + + [Fact] + public void DeleteCascadesThroughAChainOfUsagesTest() + { + RunTestWithLinks(links => + { + ProcessQuery(links, "() ((1 1) (2 2) (1 2))"); + ProcessQuery(links, "() ((3 3))"); + + var changes = ReportedChanges(links, "((1: 1 1)) ()"); + + Assert.Equal(new uint[] { 1, 3, 4 }, changes.Select(change => change.Item1.Index).Order()); + Assert.All(changes, change => Assert.True(change.Item2.IsNull())); + Assert.Equal(new[] { new DoubletLink(2, 2, 2) }, GetAllLinks(links)); + }); + } + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs index 3637da0..9921f82 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs @@ -112,10 +112,8 @@ public async Task ConcurrentClientsShareOneStore() using var client = server.Client(worker % 2 == 0 ? new TextLinoProtocol() : new BinaryLinoProtocol()); for (var item = 0; item < 5; item++) { - // The C# query processor does not report named creations as - // changes (unlike the Rust one), so only success is checked here. var name = $"w{worker}i{item}"; - client.Query($"() (({name}: {name} {name}))"); + Assert.Equal($"() (({name}: {name} {name}))", client.QueryText($"() (({name}: {name} {name}))")); } })).ToArray(); await Task.WhenAll(workers); diff --git a/docs/HOW-IT-WORKS.md b/docs/HOW-IT-WORKS.md index 43d05c2..aabf9f8 100644 --- a/docs/HOW-IT-WORKS.md +++ b/docs/HOW-IT-WORKS.md @@ -50,6 +50,16 @@ Delete: The substitution side is empty, so matched links are deleted. +Deleting a link also deletes every link that uses it, transitively, so a link +never refers to an address that no longer exists. After +`() ((1 1) (2 2) (1 2))`, the query `((2: 2 2)) ()` deletes `(3: 1 2)` too, +and `--changes` reports both: + +```text +((2: 2 2)) () +((3: 1 2)) () +``` + ## Pattern Elements Patterns can contain: diff --git a/docs/case-studies/issue-105/README.md b/docs/case-studies/issue-105/README.md index 24e012e..32ca473 100644 --- a/docs/case-studies/issue-105/README.md +++ b/docs/case-studies/issue-105/README.md @@ -455,8 +455,12 @@ now 41 scenarios that all agree) found these, all fixed here: `LinksUniquenessAndUsagesRepointingResolverTests` fails on four of five tests against the upstream resolver and passes against this one. -Deleting a link also deletes the links that refer to it, in both ports. For -example, `((2: 2 2)) ()` with `(1: 1 2)` present replies with two deletions. +Deleting a link also deletes the links that refer to it, transitively, in +both ports. This is by design, not a difference: a link never refers to an +address that no longer exists. For example, `((2: 2 2)) ()` with `(3: 1 2)` +present replies with two deletions. The behaviour is documented in +[HOW-IT-WORKS](../../HOW-IT-WORKS.md) and proven by the +`DeleteCascades*` tests in C# and the `*cascade*` tests in Rust. ## 7. Risks and remaining limits From 18fe07cd4416faf42c7f173cc2cb325a6e0e4e5f Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 01:50:23 +0000 Subject: [PATCH 27/44] feat(rust): configurable section-based binary links notation Packets are now sections of links that share one reference width and one arity range. Doublets with a uniform width stay the default (and keep the compact header); --arity (e.g. 2..3, 1..) and --packed-widths replace the sequence section and progressive widths. Packing plans sections with a linear DP and never emits more bytes than the uniform layout. Golden vectors in docs/protocol/binary-links-notation-vectors.txt pin the format for both ports. --- .../binary-links-notation-vectors.txt | 112 +++ ...20261004_120000_issue_105_tcp_protocols.md | 2 +- rust/examples/lino_binary_dump.rs | 55 +- rust/src/cli.rs | 42 +- rust/src/main.rs | 8 +- rust/src/protocol/mapping.rs | 349 ++++---- rust/src/protocol/mod.rs | 15 +- rust/src/protocol/packet.rs | 833 +++++++++++++----- rust/src/protocol/protocols.rs | 11 +- rust/tests/cli_tcp_tests.rs | 22 +- rust/tests/common/mod.rs | 39 +- rust/tests/protocol_packet_tests.rs | 535 ++++++++--- rust/tests/protocol_tcp_tests.rs | 6 +- 13 files changed, 1429 insertions(+), 600 deletions(-) create mode 100644 docs/protocol/binary-links-notation-vectors.txt diff --git a/docs/protocol/binary-links-notation-vectors.txt b/docs/protocol/binary-links-notation-vectors.txt new file mode 100644 index 0000000..5831901 --- /dev/null +++ b/docs/protocol/binary-links-notation-vectors.txt @@ -0,0 +1,112 @@ +# Golden vectors of binary links notation, version 1. +# +# Both ports check them in their test suites (csharp: LinoProtocolCodecTests, +# rust: protocol_packet_tests), so they write and read exactly these bytes. +# Fields are separated by tabs; bytes are hexadecimal. +# +# document +# A LiNo document (`\n` is a line break) encoded with the binary LiNo +# options: plain or external references, the arity range of links, and +# uniform or packed reference widths. +# +# links +# A packet of raw links laid out by `pack`, written as +# `address:reference reference…` separated by `;`, where `#v` is the +# external value `v` and a bare number is a link address. + +document plain 2 uniform 10 00 +document plain 2 packed 10 00 +document plain 2..3 uniform 10 00 +document plain 2..3 packed 10 00 +document plain 1.. uniform 10 00 +document plain 1.. packed 10 00 +document external 2 uniform 11 00 +document external 2 packed 11 00 +document external 2..3 uniform 11 00 +document external 2..3 packed 11 00 +document external 1.. uniform 11 00 +document external 1.. packed 11 00 +document () ((1 1)) plain 2 uniform 10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b +document () ((1 1)) plain 2 packed 10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b +document () ((1 1)) plain 2..3 uniform 10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b +document () ((1 1)) plain 2..3 packed 10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b +document () ((1 1)) plain 1.. uniform 12 04 24 05 02 10 01 20 01 10 01 02 01 06 06 07 00 08 09 +document () ((1 1)) plain 1.. packed 12 04 24 05 02 10 01 20 01 10 01 02 01 06 06 07 00 08 09 +document () ((1 1)) external 2 uniform 11 06 ff ff 06 00 04 07 00 08 09 00 04 0a +document () ((1 1)) external 2 packed 11 06 ff ff 06 00 04 07 00 08 09 00 04 0a +document () ((1 1)) external 2..3 uniform 11 06 ff ff 06 00 04 07 00 08 09 00 04 0a +document () ((1 1)) external 2..3 packed 11 06 ff ff 06 00 04 07 00 08 09 00 04 0a +document () ((1 1)) external 1.. uniform 13 01 1c 05 01 04 01 ff ff 00 06 01 00 07 00 08 +document () ((1 1)) external 1.. packed 13 01 1c 05 01 04 01 ff ff 00 06 01 00 07 00 08 +document ((1: 1 1)) ((1: 1 2)) plain 2 uniform 10 12 02 01 06 00 06 07 06 08 05 09 0a 00 04 0b 01 01 02 0d 0e 00 06 0f 06 10 05 11 12 00 04 13 0c 14 15 00 04 16 +document ((1: 1 1)) ((1: 1 2)) plain 2 packed 10 12 02 01 06 00 06 07 06 08 05 09 0a 00 04 0b 01 01 02 0d 0e 00 06 0f 06 10 05 11 12 00 04 13 0c 14 15 00 04 16 +document ((1: 1 1)) ((1: 1 2)) plain 2..3 uniform 10 12 02 01 06 00 06 07 06 08 05 09 0a 00 04 0b 01 01 02 0d 0e 00 06 0f 06 10 05 11 12 00 04 13 0c 14 15 00 04 16 +document ((1: 1 1)) ((1: 1 2)) plain 2..3 packed 10 12 02 01 06 00 06 07 06 08 05 09 0a 00 04 0b 01 01 02 0d 0e 00 06 0f 06 10 05 11 12 00 04 13 0c 14 15 00 04 16 +document ((1: 1 1)) ((1: 1 2)) plain 1.. uniform 12 02 24 05 03 18 03 06 02 01 01 01 02 07 03 05 06 06 06 00 09 03 05 06 06 08 00 0b 01 0a 0c 00 0d +document ((1: 1 1)) ((1: 1 2)) plain 1.. packed 12 02 24 05 03 18 03 06 02 01 01 01 02 07 03 05 06 06 06 00 09 03 05 06 06 08 00 0b 01 0a 0c 00 0d +document ((1: 1 1)) ((1: 1 2)) external 2 uniform 11 0f ff 00 ff 06 ff 07 05 08 09 00 04 0a fe 00 ff 0c ff 0d 05 0e 0f 00 04 10 0b 11 12 00 04 13 +document ((1: 1 1)) ((1: 1 2)) external 2 packed 11 0f ff 00 ff 06 ff 07 05 08 09 00 04 0a fe 00 ff 0c ff 0d 05 0e 0f 00 04 10 0b 11 12 00 04 13 +document ((1: 1 1)) ((1: 1 2)) external 2..3 uniform 11 0f ff 00 ff 06 ff 07 05 08 09 00 04 0a fe 00 ff 0c ff 0d 05 0e 0f 00 04 10 0b 11 12 00 04 13 +document ((1: 1 1)) ((1: 1 2)) external 2..3 packed 11 0f ff 00 ff 06 ff 07 05 08 09 00 04 0a fe 00 ff 0c ff 0d 05 0e 0f 00 04 10 0b 11 12 00 04 13 +document ((1: 1 1)) ((1: 1 2)) external 1.. uniform 13 01 1c 05 03 06 03 05 ff ff ff 00 06 03 05 ff ff fe 00 08 01 07 09 00 0a +document ((1: 1 1)) ((1: 1 2)) external 1.. packed 13 01 1c 05 03 06 03 05 ff ff ff 00 06 03 05 ff ff fe 00 08 01 07 09 00 0a +document hi plain 2 uniform 10 10 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 08 0b 0c 08 01 0a 0e 0b 0f 10 00 0d 11 03 12 13 00 04 14 +document hi plain 2 packed 10 10 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 08 0b 0c 08 01 0a 0e 0b 0f 10 00 0d 11 03 12 13 00 04 14 +document hi plain 2..3 uniform 12 03 24 05 0b 30 01 20 02 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 08 0b 0c 08 01 0a 0e 0b 0f 03 0d 10 11 00 04 12 +document hi plain 2..3 packed 12 03 24 05 0b 30 01 20 02 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 08 0b 0c 08 01 0a 0e 0b 0f 03 0d 10 11 00 04 12 +document hi plain 1.. uniform 12 03 24 05 0b 30 01 10 01 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 08 0b 0c 08 01 0a 0e 0b 0f 03 0d 10 11 +document hi plain 1.. packed 12 03 24 05 0b 30 01 10 01 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 08 0b 0c 08 01 0a 0e 0b 0f 03 0d 10 11 +document hi external 2 uniform 11 05 97 00 98 06 03 07 08 00 04 09 +document hi external 2 packed 11 05 97 00 98 06 03 07 08 00 04 09 +document hi external 2..3 uniform 13 02 34 05 01 20 02 03 98 97 06 00 04 07 +document hi external 2..3 packed 13 02 34 05 01 20 02 03 98 97 06 00 04 07 +document hi external 1.. uniform 13 02 34 05 01 10 01 03 98 97 06 +document hi external 1.. packed 13 02 34 05 01 10 01 03 98 97 06 +document (a b c) plain 2 uniform 10 19 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 01 0b 0c 0d 00 03 0e 0a 06 0b 10 11 00 03 12 06 01 0a 14 0b 15 16 00 03 17 18 00 13 19 0f 1a 04 1b 1c 00 04 1d +document (a b c) plain 2 packed 10 19 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 01 0b 0c 0d 00 03 0e 0a 06 0b 10 11 00 03 12 06 01 0a 14 0b 15 16 00 03 17 18 00 13 19 0f 1a 04 1b 1c 00 04 1d +document (a b c) plain 2..3 uniform 12 03 24 05 13 30 01 20 02 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 01 0b 0c 0d 00 03 0e 0a 06 0b 10 11 00 03 12 06 01 0a 14 0b 15 16 00 03 17 0f 13 18 19 00 04 1a +document (a b c) plain 2..3 packed 12 03 24 05 13 30 01 20 02 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 01 0b 0c 0d 00 03 0e 0a 06 0b 10 11 00 03 12 06 01 0a 14 0b 15 16 00 03 17 0f 13 18 19 00 04 1a +document (a b c) plain 1.. uniform 12 03 24 05 13 30 01 10 01 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 01 0b 0c 0d 00 03 0e 0a 06 0b 10 11 00 03 12 06 01 0a 14 0b 15 16 00 03 17 0f 13 18 19 +document (a b c) plain 1.. packed 12 03 24 05 13 30 01 10 01 01 01 06 06 07 07 08 08 09 09 0a 0a 0a 01 0b 0c 0d 00 03 0e 0a 06 0b 10 11 00 03 12 06 01 0a 14 0b 15 16 00 03 17 0f 13 18 19 +document (a b c) external 2 uniform 11 0c 9f 00 03 06 9e 00 03 08 9d 00 03 0a 0b 00 09 0c 07 0d 04 0e 0f 00 04 10 +document (a b c) external 2 packed 11 0c 9f 00 03 06 9e 00 03 08 9d 00 03 0a 0b 00 09 0c 07 0d 04 0e 0f 00 04 10 +document (a b c) external 2..3 uniform 13 03 24 05 06 30 01 20 02 9f 00 03 06 9e 00 03 08 9d 00 03 0a 07 09 0b 0c 00 04 0d +document (a b c) external 2..3 packed 13 03 24 05 06 30 01 20 02 9f 00 03 06 9e 00 03 08 9d 00 03 0a 07 09 0b 0c 00 04 0d +document (a b c) external 1.. uniform 13 03 24 05 06 30 01 10 01 9f 00 03 06 9e 00 03 08 9d 00 03 0a 07 09 0b 0c +document (a b c) external 1.. packed 13 03 24 05 06 30 01 10 01 9f 00 03 06 9e 00 03 08 9d 00 03 0a 07 09 0b 0c +document (1000 1) plain 2 uniform 10 13 01 01 06 06 07 07 08 08 09 09 0a 0a 0b 0b 0c 0c 0d 0d 0a 08 0b 0f 0c 10 0d 11 0e 12 02 13 02 01 14 15 16 00 04 17 +document (1000 1) plain 2 packed 10 13 01 01 06 06 07 07 08 08 09 09 0a 0a 0b 0b 0c 0c 0d 0d 0a 08 0b 0f 0c 10 0d 11 0e 12 02 13 02 01 14 15 16 00 04 17 +document (1000 1) plain 2..3 uniform 10 13 01 01 06 06 07 07 08 08 09 09 0a 0a 0b 0b 0c 0c 0d 0d 0a 08 0b 0f 0c 10 0d 11 0e 12 02 13 02 01 14 15 16 00 04 17 +document (1000 1) plain 2..3 packed 10 13 01 01 06 06 07 07 08 08 09 09 0a 0a 0b 0b 0c 0c 0d 0d 0a 08 0b 0f 0c 10 0d 11 0e 12 02 13 02 01 14 15 16 00 04 17 +document (1000 1) plain 1.. uniform 12 02 24 05 11 10 01 01 01 06 06 07 07 08 08 09 09 0a 0a 0b 0b 0c 0c 0d 0d 0a 08 0b 0f 0c 10 0d 11 0e 12 02 13 02 01 14 15 16 +document (1000 1) plain 1.. packed 12 02 24 05 11 10 01 01 01 06 06 07 07 08 08 09 09 0a 0a 0b 0b 0c 0c 0d 0d 0a 08 0b 0f 0c 10 0d 11 0e 12 02 13 02 01 14 15 16 +document (1000 1) external 2 uniform 15 03 18 fc ff ff 06 00 00 00 04 00 07 00 +document (1000 1) external 2 packed 15 03 18 fc ff ff 06 00 00 00 04 00 07 00 +document (1000 1) external 2..3 uniform 15 03 18 fc ff ff 06 00 00 00 04 00 07 00 +document (1000 1) external 2..3 packed 15 03 18 fc ff ff 06 00 00 00 04 00 07 00 +document (1000 1) external 1.. uniform 13 02 25 05 01 11 01 18 fc ff ff 06 00 +document (1000 1) external 1.. packed 13 02 25 05 01 10 01 18 fc ff ff 06 +document 1\n2 plain 2 uniform 10 04 02 01 01 01 02 07 06 08 +document 1\n2 plain 2 packed 10 04 02 01 01 01 02 07 06 08 +document 1\n2 plain 2..3 uniform 10 04 02 01 01 01 02 07 06 08 +document 1\n2 plain 2..3 packed 10 04 02 01 01 01 02 07 06 08 +document 1\n2 plain 1.. uniform 10 04 02 01 01 01 02 07 06 08 +document 1\n2 plain 1.. packed 10 04 02 01 01 01 02 07 06 08 +document 1\n2 external 2 uniform 11 01 ff fe +document 1\n2 external 2 packed 11 01 ff fe +document 1\n2 external 2..3 uniform 11 01 ff fe +document 1\n2 external 2..3 packed 11 01 ff fe +document 1\n2 external 1.. uniform 11 01 ff fe +document 1\n2 external 1.. packed 11 01 ff fe +links 1:1 1;2:1 2 3;3:2 2 2;10:3 9 plain uniform 12 03 20 01 30 02 24 06 01 01 01 01 02 03 02 02 02 03 09 +links 1:1 1;2:1 2 3;3:2 2 2;10:3 9 plain packed 12 03 20 01 30 02 24 06 01 01 01 01 02 03 02 02 02 03 09 +links 1:1 1;2:1 2;3:2 3;4:3 4;5:4 5;6:5 6;7:70000 1 plain uniform 12 01 22 07 01 00 00 00 01 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 03 00 00 00 03 00 00 00 04 00 00 00 04 00 00 00 05 00 00 00 05 00 00 00 06 00 00 00 70 11 01 00 01 00 00 00 +links 1:1 1;2:1 2;3:2 3;4:3 4;5:4 5;6:5 6;7:70000 1 plain packed 12 02 20 06 22 01 01 01 01 02 02 03 03 04 04 05 05 06 70 11 01 00 01 00 00 00 +links 1:#65;2:1 #66 #67;3:2 external uniform 13 03 10 01 30 01 10 01 bf 01 be bd 02 +links 1:#65;2:1 #66 #67;3:2 external packed 13 03 10 01 30 01 10 01 bf 01 be bd 02 +links 6:1 1;7:#1000 6 external uniform 15 02 01 00 01 00 18 fc 06 00 +links 6:1 1;7:#1000 6 external packed 15 02 01 00 01 00 18 fc 06 00 +links 1:1;2:1 1;3:1 1 1;4:1 1 1 1 plain uniform 12 01 18 03 04 00 01 01 01 01 02 01 01 01 03 01 01 01 01 +links 1:1;2:1 1;3:1 1 1;4:1 1 1 1 plain packed 12 01 18 03 04 00 01 01 01 01 02 01 01 01 03 01 01 01 01 +links 1:0 0;300:1 1;301:300 #5 external uniform 13 02 21 01 25 aa 02 02 00 00 00 00 01 00 01 00 2c 01 fb ff +links 1:0 0;300:1 1;301:300 #5 external packed 13 02 20 01 25 aa 02 02 00 00 01 00 01 00 2c 01 fb ff diff --git a/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md b/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md index e3f5a08..feb1686 100644 --- a/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md +++ b/rust/changelog.d/20261004_120000_issue_105_tcp_protocols.md @@ -2,6 +2,6 @@ bump: minor --- -Added LiNo substitution operations over TCP (issue #105): a dot-stuffed UTF-8 `TextLinoProtocol` and an interchangeable `BinaryLinoProtocol` whose packets carry the number of links and grow reference widths from 8 to 16, 32 and 64 bits, with optional external (Hybrid) references, a variable-length sequence section and progressive widths. `LinksServer`/`LinksClient` and the new `clink --serve`/`--connect` options (`--protocol`, `--external-references`, `--sequences`, `--progressive-widths`) run create, read, update and delete queries over either protocol. +Added LiNo substitution operations over TCP (issue #105): a dot-stuffed UTF-8 `TextLinoProtocol` and an interchangeable `BinaryLinoProtocol` whose packets are sections of links sharing one reference width (8, 16, 32 or 64 bits) and one arity range. Doublets with one uniform width are the default; external (Hybrid) references, any arity range such as `2..3` or `1..`, and packed per-section widths are options, and packed output is never larger than uniform output. The format is pinned by shared golden vectors in `docs/protocol/binary-links-notation-vectors.txt`. `LinksServer`/`LinksClient` and the new `clink --serve`/`--connect` options (`--protocol`, `--external-references`, `--arity`, `--packed-widths`) run create, read, update and delete queries over either protocol. Upgraded `links-notation` to 0.22.0. In earlier versions, parse time grew exponentially with nesting depth, so one short message could stall a server. diff --git a/rust/examples/lino_binary_dump.rs b/rust/examples/lino_binary_dump.rs index ee31f2a..1e2ce66 100644 --- a/rust/examples/lino_binary_dump.rs +++ b/rust/examples/lino_binary_dump.rs @@ -1,10 +1,11 @@ -//! Prints the binary LiNo encoding of a document under every option set. +//! Prints the binary LiNo encoding of a document under several option sets. //! //! ```text //! cargo run --example lino_binary_dump -- "() ((1 1))" +//! VERBOSE=1 cargo run --example lino_binary_dump -- "(a b c)" //! ``` -use link_cli::protocol::{encode_document, parse_document, BinaryLinoOptions}; +use link_cli::protocol::{encode_document, parse_document, ArityRange, BinaryLinoOptions}; fn main() { let text = std::env::args() @@ -13,26 +14,36 @@ fn main() { let document = parse_document(&text).expect("valid LiNo"); println!("document: {text}"); for external_references in [false, true] { - for sequences in [false, true] { - let options = BinaryLinoOptions { - external_references, - sequences, - progressive_widths: false, - }; - let packet = encode_document(&document, options).expect("encodable"); - let bytes = packet.to_bytes().expect("serializable"); - let hex = bytes - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::>() - .join(" "); - println!( - "external_references={external_references:<5} sequences={sequences:<5} {:>4} bytes: {hex}", - bytes.len() - ); - if std::env::var_os("VERBOSE").is_some() { - println!(" doublets: {:?}", packet.doublets); - println!(" sequences: {:?}", packet.sequences); + for arity in [ + ArityRange::DOUBLETS, + ArityRange::between(2, 3), + ArityRange::at_least(1), + ] { + for packed_widths in [false, true] { + let options = BinaryLinoOptions { + external_references, + arity, + packed_widths, + }; + let packet = encode_document(&document, options).expect("encodable"); + let bytes = packet.to_bytes().expect("serializable"); + let hex = bytes + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::>() + .join(" "); + println!( + "external={external_references:<5} arity={arity:<4} packed={packed_widths:<5} {:>4} bytes: {hex}", + bytes.len() + ); + if std::env::var_os("VERBOSE").is_some() { + for section in &packet.sections { + println!( + " gap {} arity {} width {}: {:?}", + section.gap, section.arity, section.width, section.links + ); + } + } } } } diff --git a/rust/src/cli.rs b/rust/src/cli.rs index 8fd168b..6c5c8cd 100644 --- a/rust/src/cli.rs +++ b/rust/src/cli.rs @@ -1,5 +1,6 @@ //! Command-line argument parsing for the `clink` binary. +use crate::protocol::ArityRange; use anyhow::{bail, Result}; use std::env; use std::ffi::OsString; @@ -42,8 +43,8 @@ pub struct Cli { pub connect: Option, pub protocol: Option, pub external_references: bool, - pub sequences: bool, - pub progressive_widths: bool, + pub arity: ArityRange, + pub packed_widths: bool, } impl Default for Cli { @@ -83,8 +84,8 @@ impl Default for Cli { connect: None, protocol: None, external_references: false, - sequences: false, - progressive_widths: false, + arity: ArityRange::DOUBLETS, + packed_widths: false, } } } @@ -291,12 +292,12 @@ impl Cli { cli.external_references = parse_bool("--external-references", value)?; continue; } - if let Some(value) = inline_value(&arg, &["--sequences"]) { - cli.sequences = parse_bool("--sequences", value)?; + if let Some(value) = inline_value(&arg, &["--arity"]) { + cli.arity = parse_arity(value)?; continue; } - if let Some(value) = inline_value(&arg, &["--progressive-widths"]) { - cli.progressive_widths = parse_bool("--progressive-widths", value)?; + if let Some(value) = inline_value(&arg, &["--packed-widths"]) { + cli.packed_widths = parse_bool("--packed-widths", value)?; continue; } @@ -404,11 +405,11 @@ impl Cli { "--external-references" => { cli.external_references = next_bool_value(&mut args, true)?; } - "--sequences" => { - cli.sequences = next_bool_value(&mut args, true)?; + "--arity" => { + cli.arity = parse_arity(&next_value(&mut args, &arg)?)?; } - "--progressive-widths" => { - cli.progressive_widths = next_bool_value(&mut args, true)?; + "--packed-widths" => { + cli.packed_widths = next_bool_value(&mut args, true)?; } "--" => { for value in args.by_ref() { @@ -516,10 +517,12 @@ impl Cli { " or 'any' for --serve (default: any)\n", " --external-references\n", " Binary protocol: send numbers and characters as external references\n", - " --sequences\n", - " Binary protocol: send lists in the variable-length sequence section\n", - " --progressive-widths\n", - " Binary protocol: widen references link by link instead of uniformly\n", + " --arity \n", + " Binary protocol: link lengths to use, 'n', 'min..max' or 'min..'\n", + " (default: 2, doublets only; e.g. 2..3 adds triplets, 1.. any length)\n", + " --packed-widths\n", + " Binary protocol: give each section of links the narrowest reference\n", + " width it needs instead of one width for the whole packet\n", " -h, --help\n", " Print help\n", " -V, --version\n", @@ -532,6 +535,13 @@ impl Cli { } } +fn parse_arity(value: &str) -> Result { + value + .trim() + .parse() + .map_err(|error| anyhow::anyhow!("invalid value for '--arity': {error}")) +} + fn inline_value<'a>(arg: &'a str, names: &[&str]) -> Option<&'a str> { names.iter().find_map(|name| { arg.strip_prefix(name) diff --git a/rust/src/main.rs b/rust/src/main.rs index 35d6691..408fb27 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -415,13 +415,13 @@ where Ok(()) } -/// The binary options selected by `--external-references`, `--sequences` and -/// `--progressive-widths`. +/// The binary options selected by `--external-references`, `--arity` and +/// `--packed-widths`. fn binary_options(cli: &Cli) -> BinaryLinoOptions { BinaryLinoOptions::default() .with_external_references(cli.external_references) - .with_sequences(cli.sequences) - .with_progressive_widths(cli.progressive_widths) + .with_arity(cli.arity) + .with_packed_widths(cli.packed_widths) } /// `--connect`: sends the query to a server and prints the reply. diff --git a/rust/src/protocol/mapping.rs b/rust/src/protocol/mapping.rs index bb89eec..cd01413 100644 --- a/rust/src/protocol/mapping.rs +++ b/rust/src/protocol/mapping.rs @@ -14,17 +14,21 @@ //! - A link with an id is `(Identified id values…)`. //! - The document is a list of its top-level links, stored last (the root). //! -//! Typed values and lists are encoded as a variable-length sequence -//! `[marker, elements…]` when the sequence section is enabled, and as the -//! doublet `(marker chain)` otherwise, where `chain` is the nil-terminated cons -//! list `(e1 (e2 (… (en 0))))`. A plain list in sequence mode has no marker. -//! Identical sub-links are emitted once and shared, because links are -//! content-addressed. +//! A link with two values is always a doublet. A list or typed value with +//! any other number of values is a single link of that many references when +//! [`BinaryLinoOptions::arity`] allows it (`[marker, elements…]` for typed +//! values, the bare elements for lists), and otherwise the doublet +//! `(marker chain)`, where `chain` is the nil-terminated cons list +//! `(e1 (e2 (… (en 0))))`. Identical sub-links are emitted once and shared, +//! because links are content-addressed. +//! +//! Links are numbered so that every link only refers to earlier ones: +//! doublets of plain doublets first, then the rest in creation order. use super::error::{ProtocolError, ProtocolResult}; use super::packet::{ - external_capacity, DecodeLimits, LinksPacket, Reference, FIRST_LINK_ADDRESS, IDENTIFIED, LIST, - NULL, NUMBER, ONE, STRING, + external_capacity, ArityRange, DecodeLimits, LinksPacket, Reference, FIRST_LINK_ADDRESS, + IDENTIFIED, LIST, NULL, NUMBER, ONE, STRING, }; use links_notation::LiNo; use std::collections::HashMap; @@ -41,12 +45,13 @@ pub struct BinaryLinoOptions { /// Send numbers and code points as Hybrid external references instead of /// in-band unary links. Halves the internal address range of each width. pub external_references: bool, - /// Use the variable-length sequence section for lists, strings and links - /// with ids instead of cons chains of doublets. - pub sequences: bool, - /// Let the reference width grow with the address instead of using the - /// width of the largest address for every reference. - pub progressive_widths: bool, + /// The link lengths the encoder may use. The default, exactly 2, sends + /// only doublets; `2..3` adds triplets and `1..` any length. The range + /// must include 2. + pub arity: ArityRange, + /// Give every section of the packet the narrowest width its links need + /// instead of one width for the whole packet. + pub packed_widths: bool, } impl BinaryLinoOptions { @@ -56,17 +61,35 @@ impl BinaryLinoOptions { self } - /// Enables or disables the sequence section. - pub fn with_sequences(mut self, enabled: bool) -> Self { - self.sequences = enabled; + /// Sets the link lengths the encoder may use. + pub fn with_arity(mut self, arity: ArityRange) -> Self { + self.arity = arity; self } - /// Enables or disables progressive reference widths. - pub fn with_progressive_widths(mut self, enabled: bool) -> Self { - self.progressive_widths = enabled; + /// Enables or disables packed widths. + pub fn with_packed_widths(mut self, enabled: bool) -> Self { + self.packed_widths = enabled; self } + + /// The options a peer most likely used to write `packet`, so a reply can + /// be written in the same style. + pub fn of_packet(packet: &LinksPacket) -> Self { + let (shortest, longest) = packet + .links() + .map(|(_, link)| link.len() as u64) + .fold((2, 2), |(shortest, longest), length| { + (shortest.min(length), longest.max(length)) + }); + let mut widths = packet.sections.iter().map(|section| section.width); + let first_width = widths.next(); + Self { + external_references: packet.external_references, + arity: ArityRange::between(shortest, longest), + packed_widths: widths.any(|width| Some(width) != first_width), + } + } } /// Converts a document into a packet. @@ -74,18 +97,19 @@ pub fn encode_document( document: &[LiNo], options: BinaryLinoOptions, ) -> ProtocolResult { + if !options.arity.contains(2) { + return Err(ProtocolError::Unencodable(format!( + "arity {} does not include doublets (2)", + options.arity + ))); + } let mut encoder = Encoder::new(options); if !document.is_empty() { let items = document .iter() .map(|link| encoder.encode(link)) .collect::>(); - if options.sequences { - encoder.sequences.push(items); - } else { - let chain = encoder.chain(&items); - encoder.doublets.push((Node::Internal(LIST), chain)); - } + encoder.list(items); } encoder.finish() } @@ -95,14 +119,14 @@ pub fn decode_document( packet: &LinksPacket, limits: &DecodeLimits, ) -> ProtocolResult { - let Some(root) = packet.last_address() else { + let decoder = Decoder::new(packet, limits)?; + let Some(root) = decoder.links.len().checked_sub(1) else { return Ok(Vec::new()); }; - let decoder = Decoder::new(packet, limits); - let root = Reference::Internal(root); - let items = match decoder.view(root)? { - View::Sequence(items) if !starts_with_marker(items) => items.to_vec(), - View::Doublet(Reference::Internal(LIST), chain) => decoder.chain(chain)?, + let root = Reference::Internal(FIRST_LINK_ADDRESS + root as u64); + let items = match decoder.view(root) { + View::Link(&[Reference::Internal(LIST), chain]) => decoder.chain(chain)?, + View::Link(items) if !starts_with_marker(items) => items.to_vec(), _ => return Err(ProtocolError::malformed("the root link is not a list")), }; let mut budget = limits.max_nodes; @@ -124,16 +148,17 @@ pub(crate) fn canonical_number(text: &str) -> Option { enum Node { Internal(u64), External(u64), + /// A doublet of plain doublets, markers and externals. Doublet(usize), - Sequence(usize), + /// Any other link; numbered after every [`Node::Doublet`]. + Tuple(usize), } struct Encoder { options: BinaryLinoOptions, - doublets: Vec<(Node, Node)>, - sequences: Vec>, - doublet_index: HashMap<(Node, Node), usize>, - sequence_index: HashMap, usize>, + doublets: Vec>, + tuples: Vec>, + created: HashMap, Node>, powers: Vec, } @@ -142,66 +167,70 @@ impl Encoder { Self { options, doublets: Vec::new(), - sequences: Vec::new(), - doublet_index: HashMap::new(), - sequence_index: HashMap::new(), + tuples: Vec::new(), + created: HashMap::new(), powers: vec![Node::Internal(ONE)], } } - fn doublet(&mut self, source: Node, target: Node) -> Node { - if let Some(&index) = self.doublet_index.get(&(source, target)) { - return Node::Doublet(index); - } - let index = self.doublets.len(); - self.doublets.push((source, target)); - self.doublet_index.insert((source, target), index); - Node::Doublet(index) - } - - fn sequence(&mut self, items: Vec) -> Node { - if let Some(&index) = self.sequence_index.get(&items) { - return Node::Sequence(index); + fn link(&mut self, items: Vec) -> Node { + if let Some(&node) = self.created.get(&items) { + return node; } - let index = self.sequences.len(); - self.sequences.push(items.clone()); - self.sequence_index.insert(items, index); - Node::Sequence(index) + let refers_to_tuple = items.iter().any(|item| matches!(item, Node::Tuple(_))); + let node = if items.len() == 2 && !refers_to_tuple { + self.doublets.push(items.clone()); + Node::Doublet(self.doublets.len() - 1) + } else { + self.tuples.push(items.clone()); + Node::Tuple(self.tuples.len() - 1) + }; + self.created.insert(items, node); + node } - /// A two-element link. Fixed doublets may only refer to fixed doublets, - /// so a pair holding a sequence becomes a two-element sequence. fn pair(&mut self, first: Node, second: Node) -> Node { - if matches!(first, Node::Sequence(_)) || matches!(second, Node::Sequence(_)) { - self.sequence(vec![first, second]) - } else { - self.doublet(first, second) - } + self.link(vec![first, second]) } fn chain(&mut self, items: &[Node]) -> Node { items .iter() .rev() - .fold(Node::Internal(NULL), |tail, &head| self.doublet(head, tail)) + .fold(Node::Internal(NULL), |tail, &head| self.pair(head, tail)) + } + + /// One link of `items` when the arity allows it, except that two items + /// are always a doublet. + fn fits_one_link(&self, items: usize) -> bool { + items != 2 && self.options.arity.contains(items as u64) } fn typed(&mut self, marker: u64, elements: Vec) -> Node { - if self.options.sequences { + if self.fits_one_link(elements.len() + 1) { let mut items = Vec::with_capacity(elements.len() + 1); items.push(Node::Internal(marker)); items.extend(elements); - self.sequence(items) + self.link(items) } else { let chain = self.chain(&elements); - self.doublet(Node::Internal(marker), chain) + self.pair(Node::Internal(marker), chain) + } + } + + fn list(&mut self, elements: Vec) -> Node { + match elements.as_slice() { + [] => Node::Internal(NULL), + &[first, second] => self.pair(first, second), + _ if self.fits_one_link(elements.len()) => self.link(elements), + _ => self.typed(LIST, elements), } } fn power(&mut self, exponent: usize) -> Node { while self.powers.len() <= exponent { let previous = *self.powers.last().expect("powers start with One"); - let next = self.doublet(previous, previous); + let next = self.pair(previous, previous); self.powers.push(next); } self.powers[exponent] @@ -215,7 +244,7 @@ impl Encoder { }; rest.iter() .rev() - .fold(last, |sum, &power| self.doublet(power, sum)) + .fold(last, |sum, &power| self.pair(power, sum)) } fn scalar(&mut self, value: u64) -> Node { @@ -232,7 +261,7 @@ impl Encoder { return Node::External(value); } let unary = self.unary(value); - return self.doublet(Node::Internal(NUMBER), unary); + return self.pair(Node::Internal(NUMBER), unary); } let code_points = text .chars() @@ -244,22 +273,10 @@ impl Encoder { fn encode(&mut self, link: &LiNo) -> Node { match link { LiNo::Ref(text) => self.reference(text), - LiNo::Link { id: None, values } => match values.as_slice() { - [] => Node::Internal(NULL), - [first, second] => { - let first = self.encode(first); - let second = self.encode(second); - self.pair(first, second) - } - values => { - let elements = values.iter().map(|value| self.encode(value)).collect(); - if self.options.sequences { - self.sequence(elements) - } else { - self.typed(LIST, elements) - } - } - }, + LiNo::Link { id: None, values } => { + let elements = values.iter().map(|value| self.encode(value)).collect(); + self.list(elements) + } LiNo::Link { id: Some(id), values, @@ -274,31 +291,31 @@ impl Encoder { fn finish(self) -> ProtocolResult { let doublet_count = self.doublets.len() as u64; - let resolve = |node: Node| match node { + let resolve = |node: &Node| match *node { Node::Internal(address) => Reference::Internal(address), Node::External(value) => Reference::External(value), Node::Doublet(index) => Reference::Internal(FIRST_LINK_ADDRESS + index as u64), - Node::Sequence(index) => { + Node::Tuple(index) => { Reference::Internal(FIRST_LINK_ADDRESS + doublet_count + index as u64) } }; - let mut packet = LinksPacket { - external_references: self.options.external_references, - sequences_section: self.options.sequences, - min_width: 1, - doublets: self - .doublets - .iter() - .map(|&(source, target)| (resolve(source), resolve(target))) - .collect(), - sequences: self - .sequences - .iter() - .map(|items| items.iter().map(|&item| resolve(item)).collect()) - .collect(), - }; - packet.min_width = packet.required_min_width(!self.options.progressive_widths)?; - Ok(packet) + let links: Vec<(u64, Vec)> = self + .doublets + .iter() + .chain(&self.tuples) + .enumerate() + .map(|(index, items)| { + ( + FIRST_LINK_ADDRESS + index as u64, + items.iter().map(resolve).collect(), + ) + }) + .collect(); + LinksPacket::pack( + self.options.external_references, + &links, + self.options.packed_widths, + ) } } @@ -306,67 +323,78 @@ enum View<'a> { Null, Marker(u64), External(u64), - Doublet(Reference, Reference), - Sequence(&'a [Reference]), + Link(&'a [Reference]), +} + +fn is_marker(reference: Reference) -> bool { + matches!(reference, Reference::Internal(address) if (ONE..FIRST_LINK_ADDRESS).contains(&address)) } fn starts_with_marker(items: &[Reference]) -> bool { - matches!(items.first(), Some(Reference::Internal(address)) if (ONE..FIRST_LINK_ADDRESS).contains(address)) + items.first().copied().is_some_and(is_marker) } struct Decoder<'a> { - packet: &'a LinksPacket, limits: &'a DecodeLimits, - /// `unary[i]` is the number doublet `i` denotes, if it is a unary number. + /// `links[i]` is the link at address `FIRST_LINK_ADDRESS + i`. + links: Vec<&'a [Reference]>, + /// `unary[i]` is the number link `i` denotes, if it is a unary number. unary: Vec>, } impl<'a> Decoder<'a> { - fn new(packet: &'a LinksPacket, limits: &'a DecodeLimits) -> Self { - // Links only refer backwards, so one forward pass evaluates every - // unary number without recursion. - let mut unary: Vec> = Vec::with_capacity(packet.doublets.len()); - for &(source, target) in &packet.doublets { + fn new(packet: &'a LinksPacket, limits: &'a DecodeLimits) -> ProtocolResult { + let mut links: Vec<&'a [Reference]> = Vec::new(); + let mut unary: Vec> = Vec::new(); + for (address, link) in packet.links() { + let expected = FIRST_LINK_ADDRESS + links.len() as u64; + if address != expected { + return Err(ProtocolError::malformed(format!( + "a LiNo packet stores its links contiguously from address \ + {FIRST_LINK_ADDRESS}, found link {address} where {expected} belongs" + ))); + } + if let Some(&target) = link.iter().find( + |&&reference| matches!(reference, Reference::Internal(target) if target >= address), + ) { + return Err(ProtocolError::malformed(format!( + "link {address} refers to {target:?}, which is not an earlier link" + ))); + } + // Links only refer backwards, so one forward pass evaluates every + // unary number without recursion. let value_of = |reference: Reference| match reference { Reference::Internal(NULL) => Some(0), Reference::Internal(ONE) => Some(1), - Reference::Internal(address) if address >= FIRST_LINK_ADDRESS => unary - .get((address - FIRST_LINK_ADDRESS) as usize) - .copied() - .flatten(), + Reference::Internal(address) if address >= FIRST_LINK_ADDRESS => { + unary[(address - FIRST_LINK_ADDRESS) as usize] + } + _ => None, + }; + let value = match *link { + [source, target] => value_of(source) + .zip(value_of(target)) + .and_then(|(source, target)| source.checked_add(target)), _ => None, }; - let value = value_of(source) - .zip(value_of(target)) - .and_then(|(source, target)| source.checked_add(target)); unary.push(value); + links.push(link); } - Self { - packet, + Ok(Self { limits, + links, unary, - } + }) } - fn view(&self, reference: Reference) -> ProtocolResult> { - let address = match reference { - Reference::External(value) => return Ok(View::External(value)), - Reference::Internal(NULL) => return Ok(View::Null), - Reference::Internal(address) if address < FIRST_LINK_ADDRESS => { - return Ok(View::Marker(address)) + fn view(&self, reference: Reference) -> View<'a> { + match reference { + Reference::External(value) => View::External(value), + Reference::Internal(NULL) => View::Null, + Reference::Internal(address) if address < FIRST_LINK_ADDRESS => View::Marker(address), + Reference::Internal(address) => { + View::Link(self.links[(address - FIRST_LINK_ADDRESS) as usize]) } - Reference::Internal(address) => address - FIRST_LINK_ADDRESS, - }; - let doublets = self.packet.doublets.len() as u64; - if address < doublets { - let (source, target) = self.packet.doublets[address as usize]; - Ok(View::Doublet(source, target)) - } else { - self.packet - .sequences - .get((address - doublets) as usize) - .map(|items| View::Sequence(items.as_slice())) - .ok_or_else(|| ProtocolError::malformed(format!("dangling reference {address}"))) } } @@ -375,34 +403,26 @@ impl<'a> Decoder<'a> { Reference::External(value) => Ok(value), Reference::Internal(NULL) => Ok(0), Reference::Internal(ONE) => Ok(1), - Reference::Internal(address) if address >= FIRST_LINK_ADDRESS => self - .unary - .get((address - FIRST_LINK_ADDRESS) as usize) - .copied() - .flatten() + Reference::Internal(address) if address >= FIRST_LINK_ADDRESS => self.unary + [(address - FIRST_LINK_ADDRESS) as usize] .ok_or_else(|| ProtocolError::malformed("expected a unary number")), _ => Err(ProtocolError::malformed("expected a unary number")), } } - /// The elements of a typed value given in doublet form: a cons chain, - /// optionally ending in a sequence holding the remaining elements. + /// The elements of the cons list `(e1 (e2 (… (en 0))))`. fn chain(&self, mut tail: Reference) -> ProtocolResult> { let mut elements = Vec::new(); loop { - match self.view(tail)? { + match self.view(tail) { View::Null => return Ok(elements), - View::Doublet(head, next) => { + View::Link(&[head, next]) => { if elements.len() >= self.limits.max_nodes { return Err(ProtocolError::LimitExceeded("chain too long".into())); } elements.push(head); tail = next; } - View::Sequence(items) => { - elements.extend_from_slice(items); - return Ok(elements); - } _ => return Err(ProtocolError::malformed("broken element chain")), } } @@ -484,7 +504,7 @@ impl<'a> Decoder<'a> { *budget = budget .checked_sub(1) .ok_or_else(|| ProtocolError::LimitExceeded("too many LiNo nodes".into()))?; - match self.view(reference)? { + match self.view(reference) { View::Null => Ok(LiNo::Link { id: None, values: Vec::new(), @@ -493,20 +513,17 @@ impl<'a> Decoder<'a> { View::Marker(marker) => Err(ProtocolError::malformed(format!( "marker {marker} used as a value" ))), - View::Doublet(Reference::Internal(NUMBER), value) => { + View::Link(&[Reference::Internal(NUMBER), value]) => { self.typed(NUMBER, &[value], depth, budget) } - View::Doublet(Reference::Internal(marker), chain) - if (ONE..FIRST_LINK_ADDRESS).contains(&marker) => + View::Link(&[Reference::Internal(marker), chain]) + if is_marker(Reference::Internal(marker)) => { let elements = self.chain(chain)?; self.typed(marker, &elements, depth, budget) } - View::Doublet(source, target) => self.list(&[source, target], depth, budget), - View::Sequence(items) => match items.split_first() { - Some((&Reference::Internal(marker), elements)) - if (ONE..FIRST_LINK_ADDRESS).contains(&marker) => - { + View::Link(items) => match items.split_first() { + Some((&Reference::Internal(marker), elements)) if starts_with_marker(items) => { self.typed(marker, elements, depth, budget) } _ => self.list(items, depth, budget), diff --git a/rust/src/protocol/mod.rs b/rust/src/protocol/mod.rs index c758758..a53a3e9 100644 --- a/rust/src/protocol/mod.rs +++ b/rust/src/protocol/mod.rs @@ -5,13 +5,14 @@ //! //! - [`TextLinoProtocol`]: UTF-8 LiNo text, each message ended by a line //! holding only `.` (SMTP-style dot-stuffing keeps the framing unambiguous). -//! - [`BinaryLinoProtocol`]: a self-delimiting [`LinksPacket`] whose header -//! carries the number of links and whose reference width grows with the -//! number of links (8, 16, 32 or 64 bits). Its optional features — +//! - [`BinaryLinoProtocol`]: a self-delimiting [`LinksPacket`] made of +//! sections of links that share one reference width (1, 2, 4 or 8 bytes) +//! and one [arity range](ArityRange). By default every link is a doublet +//! and the whole packet uses the narrowest width that fits; the options //! [external references](BinaryLinoOptions::external_references), -//! [the sequence section](BinaryLinoOptions::sequences) and -//! [progressive widths](BinaryLinoOptions::progressive_widths) — are all -//! off by default and can be switched on one by one. +//! [arity](BinaryLinoOptions::arity) (for example `2..3` or `1..`) and +//! [packed widths](BinaryLinoOptions::packed_widths) can be switched on +//! one by one. //! //! Both implement [`LinoProtocol`], so a [`LinoConnection`] or a //! [`LinksClient`] switches protocol by swapping one value. A @@ -48,7 +49,7 @@ pub use error::{ProtocolError, ProtocolResult}; pub use format::{format_document, format_link, format_reference, parse_document}; pub use links_operations::LinksOperation; pub use mapping::{decode_document, encode_document, BinaryLinoOptions, LinoDocument}; -pub use packet::{DecodeLimits, LinksPacket, Reference}; +pub use packet::{ArityRange, DecodeLimits, LinksPacket, Reference, Section}; pub use protocols::{ is_binary_start, read_any_document, BinaryLinoProtocol, LinoConnection, LinoProtocol, MessageFormat, TextLinoProtocol, diff --git a/rust/src/protocol/packet.rs b/rust/src/protocol/packet.rs index 7d311e7..5b6ab0e 100644 --- a/rust/src/protocol/packet.rs +++ b/rust/src/protocol/packet.rs @@ -1,29 +1,44 @@ -//! The binary links packet: the wire format of the binary LiNo protocol. +//! Binary links notation: a self-delimiting packet of links. +//! +//! A packet stores links, each a tuple of one or more references, at +//! implicit consecutive addresses. It knows nothing about LiNo; the LiNo +//! protocol ([`crate::protocol::encode_document`]) and the store archive +//! ([`crate::protocol::archive`]) are two uses of it. //! //! ```text -//! byte 0 0x10 | flags high nibble 1 = format version 1 -//! bit 0 external references (Hybrid encoding) -//! bit 1 sequence section present -//! bits 2-3 log2 of the minimum width in bytes -//! LEB128 N number of fixed doublets -//! LEB128 M number of sequences (only when bit 1 is set) -//! N times source target one fixed doublet, refs `width(a)` bytes each -//! M times size ref_1 … ref_n one sequence, size and refs `width(a)` bytes each +//! byte 0 0x10 | flags high nibble 1 = format version 1 +//! bit 0 external references (Hybrid encoding) +//! bit 1 explicit layout +//! bits 2-3 log2 of the width (compact layout only) +//! +//! compact layout (bit 1 clear): one section of doublets right after the markers +//! LEB128 N number of links, each `source target` +//! +//! explicit layout (bit 1 set): +//! LEB128 S number of sections, then S section headers: +//! LEB128 shape bits 0-1 log2 of the reference width in bytes +//! bit 2 a gap follows +//! bit 3 variable arity (else every link +//! holds exactly min_arity references) +//! bits 4+ min_arity, at least 1 +//! LEB128 gap addresses skipped before the section (if bit 2) +//! LEB128 extra_arity 0 = no maximum, else max - min (if bit 3) +//! LEB128 count number of links in the section +//! +//! links, section by section; a link in a variable-arity section starts +//! with LEB128 (length - min_arity); every reference is `width` bytes, +//! little-endian //! ``` //! -//! Addresses are implicit: `0` is null, `1..=5` are the reserved marker points -//! (never transmitted), the fixed doublets occupy `6..6+N` and the sequences -//! follow them, so the first sequence is "last fixed link + 1". +//! Address `0` is null. The first section starts at `1 + gap` and every +//! other section at `previous end + gap`, so gaps leave holes. The compact +//! layout is exactly one section with gap 5 (the LiNo marker points +//! `1..=5`), arity 2 and the header width. //! -//! Every reference stored in the link at address `a` uses -//! `width(a) = max(min_width, tier(a))` bytes, little-endian, where `tier(a)` -//! is the smallest of 1, 2, 4 and 8 bytes able to hold `a`. Because a link may -//! only refer to links *before* it, `tier(a)` always fits every internal -//! reference it can contain. With `min_width` equal to the tier of the largest -//! address the whole packet uses one uniform width (`0..256` addresses use -//! 8-bit references, `256..65536` 16-bit, and so on); with a smaller -//! `min_width` the width grows progressively with the address, the way LZW -//! code widths grow with the dictionary. +//! Each section has its own width: the narrowest of 1, 2, 4 and 8 bytes +//! that holds every reference in it, so links that only refer to small +//! addresses stay small wherever they live. [`LinksPacket::pack`] chooses +//! the sections. //! //! With external references enabled the top bit of a reference marks it as //! external, exactly like `Platform.Data.Hybrid`: value `v ≥ 1` is stored as @@ -31,15 +46,23 @@ //! That halves the internal range of every width (`0..128` for 8-bit, …). use super::error::{ProtocolError, ProtocolResult}; +use std::fmt; use std::io::{self, Read, Write}; +use std::str::FromStr; /// The high nibble of the header byte. Text messages never start with a byte /// in `0x10..=0x1F`, so the header doubles as a protocol detector. pub const BINARY_VERSION_1: u8 = 0x10; const FLAG_EXTERNAL_REFERENCES: u8 = 0b0001; -const FLAG_SEQUENCES: u8 = 0b0010; +const FLAG_EXPLICIT_LAYOUT: u8 = 0b0010; const WIDTH_SHIFT: u8 = 2; +const WIDTH_BITS: u8 = 0b1100; + +const SHAPE_WIDTH_BITS: u64 = 0b0011; +const SHAPE_HAS_GAP: u64 = 0b0100; +const SHAPE_VARIABLE_ARITY: u64 = 0b1000; +const SHAPE_MIN_ARITY_SHIFT: u32 = 4; /// Null link address. pub const NULL: u64 = 0; @@ -53,16 +76,19 @@ pub const STRING: u64 = 3; pub const LIST: u64 = 4; /// Marker point `5`: `(Identified id values…)` is a link with an id. pub const IDENTIFIED: u64 = 5; -/// Address of the first transmitted link. +/// Address of the first link after the marker points. pub const FIRST_LINK_ADDRESS: u64 = 6; +/// The addresses the compact layout skips: the marker points. +const COMPACT_GAP: u64 = FIRST_LINK_ADDRESS - 1; + /// The reference widths, in bytes, that a packet may use. pub const WIDTHS: [u8; 4] = [1, 2, 4, 8]; /// One reference inside a packet. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum Reference { - /// An address inside the packet (`0` null, `1..=5` markers, then links). + /// A link address (`0` is null). Internal(u64), /// An external value, e.g. a number or a Unicode code point. External(u64), @@ -73,13 +99,127 @@ impl Reference { pub const NULL: Reference = Reference::Internal(NULL); } +/// How many references the links of a section hold: `min..=max`, where +/// `max = None` means no upper bound. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ArityRange { + /// Fewest references in a link, at least 1. + pub min: u64, + /// Most references in a link, `None` for no limit. + pub max: Option, +} + +impl ArityRange { + /// Links of exactly two references. + pub const DOUBLETS: ArityRange = ArityRange::exactly(2); + + /// Links of exactly `arity` references. + pub const fn exactly(arity: u64) -> Self { + Self { + min: arity, + max: Some(arity), + } + } + + /// Links of at least `min` references. + pub const fn at_least(min: u64) -> Self { + Self { min, max: None } + } + + /// Links of `min..=max` references. + pub const fn between(min: u64, max: u64) -> Self { + Self { + min, + max: Some(max), + } + } + + /// True when a link of `length` references fits the range. + pub fn contains(&self, length: u64) -> bool { + length >= self.min && self.max.is_none_or(|max| length <= max) + } + + /// True when every link has the same number of references, so links + /// need no length prefix. + pub fn is_fixed(&self) -> bool { + self.max == Some(self.min) + } + + fn validate(&self) -> Result<(), String> { + if self.min == 0 { + return Err("arity must be at least 1".into()); + } + if self.min > u64::MAX >> SHAPE_MIN_ARITY_SHIFT { + return Err(format!("arity {} is too large", self.min)); + } + if self.max.is_some_and(|max| max < self.min) { + return Err(format!("arity range {self} is empty")); + } + Ok(()) + } + + /// `0` for no maximum, otherwise `max - min`; only for variable arities. + fn extra(&self) -> u64 { + self.max.map_or(0, |max| max - self.min) + } + + fn from_shape(min: u64, extra: Option) -> ProtocolResult { + let range = match extra { + None => Self::exactly(min), + Some(0) => Self::at_least(min), + Some(extra) => Self::between( + min, + min.checked_add(extra) + .ok_or_else(|| ProtocolError::malformed("arity range overflows 64 bits"))?, + ), + }; + range.validate().map_err(ProtocolError::malformed)?; + Ok(range) + } +} + +impl Default for ArityRange { + fn default() -> Self { + Self::DOUBLETS + } +} + +impl fmt::Display for ArityRange { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.max { + Some(max) if max == self.min => write!(formatter, "{max}"), + Some(max) => write!(formatter, "{}..{max}", self.min), + None => write!(formatter, "{}..", self.min), + } + } +} + +impl FromStr for ArityRange { + type Err = String; + + /// Parses `n`, `min..max` (inclusive) or `min..` (no maximum). + fn from_str(text: &str) -> Result { + let number = |part: &str| { + part.parse::() + .map_err(|_| format!("invalid arity '{text}': expected n, min..max or min..")) + }; + let range = match text.split_once("..") { + None => Self::exactly(number(text)?), + Some((min, "")) => Self::at_least(number(min)?), + Some((min, max)) => Self::between(number(min)?, number(max)?), + }; + range.validate()?; + Ok(range) + } +} + /// Safety limits applied while decoding untrusted input. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct DecodeLimits { - /// Maximum `N + M`. + /// Maximum number of links in a packet. pub max_links: u64, - /// Maximum total number of references inside all sequences. - pub max_sequence_items: u64, + /// Maximum number of references in all links of a packet. + pub max_references: u64, /// Maximum number of LiNo nodes a packet may expand to. pub max_nodes: usize, /// Maximum LiNo nesting depth. @@ -92,7 +232,7 @@ impl Default for DecodeLimits { fn default() -> Self { Self { max_links: 1 << 22, - max_sequence_items: 1 << 24, + max_references: 1 << 24, max_nodes: 1 << 22, max_depth: 1024, max_text_bytes: 64 << 20, @@ -100,19 +240,41 @@ impl Default for DecodeLimits { } } +impl DecodeLimits { + /// Limits for trusted input such as a store archive: only the address + /// space bounds the packet. + pub fn unlimited() -> Self { + Self { + max_links: u64::MAX, + max_references: u64::MAX, + max_nodes: usize::MAX, + max_depth: usize::MAX, + max_text_bytes: usize::MAX, + } + } +} + +/// A run of links at consecutive addresses sharing an arity range and a +/// reference width. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Section { + /// Addresses skipped before the first link of the section. + pub gap: u64, + /// The number of references each link may hold. + pub arity: ArityRange, + /// Bytes per reference: 1, 2, 4 or 8. + pub width: u8, + /// The links, in address order. + pub links: Vec>, +} + /// A decoded binary links packet. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct LinksPacket { /// Header bit 0: references may be external (Hybrid encoding). pub external_references: bool, - /// Header bit 1: the packet carries a sequence section (possibly empty). - pub sequences_section: bool, - /// Minimum reference width in bytes: 1, 2, 4 or 8. - pub min_width: u8, - /// Fixed doublets at addresses `6..6+N`. - pub doublets: Vec<(Reference, Reference)>, - /// Variable-length sequences at addresses `6+N..6+N+M`. - pub sequences: Vec>, + /// The links, section by section. + pub sections: Vec
, } /// Largest internal address that fits in `width` bytes. @@ -130,11 +292,6 @@ pub fn external_capacity(width: u8) -> u64 { (1u64 << (u32::from(width) * 8 - 1)) - 1 } -/// Largest unsigned value (a sequence size) that fits in `width` bytes. -pub fn unsigned_capacity(width: u8) -> u64 { - internal_capacity(width, false) -} - /// The narrowest width able to hold the internal address `address`. pub fn address_tier(address: u64, external_references: bool) -> u8 { WIDTHS @@ -176,118 +333,207 @@ fn width_code(width: u8) -> ProtocolResult { .ok_or_else(|| ProtocolError::Unencodable(format!("invalid width {width}"))) } -impl LinksPacket { - /// Address of the fixed doublet with zero-based index `index`. - pub fn doublet_address(&self, index: usize) -> u64 { - FIRST_LINK_ADDRESS + index as u64 - } +fn width_from_code(code: u64) -> ProtocolResult { + usize::try_from(code) + .ok() + .and_then(|code| WIDTHS.get(code).copied()) + .ok_or_else(|| ProtocolError::malformed(format!("invalid width code {code}"))) +} - /// Address of the sequence with zero-based index `index`. - pub fn sequence_address(&self, index: usize) -> u64 { - FIRST_LINK_ADDRESS + self.doublets.len() as u64 + index as u64 +/// The narrowest width able to hold `reference`. +pub fn reference_width(reference: Reference, external_references: bool) -> ProtocolResult { + match reference { + Reference::Internal(address) => { + if address > internal_capacity(8, external_references) { + return Err(ProtocolError::Unencodable(format!( + "address {address} exceeds the internal range" + ))); + } + Ok(address_tier(address, external_references)) + } + Reference::External(value) => { + if !external_references { + return Err(ProtocolError::Unencodable( + "external reference in a packet without external references".into(), + )); + } + WIDTHS + .into_iter() + .find(|&width| external_capacity(width) >= value) + .ok_or_else(|| { + ProtocolError::Unencodable(format!("external value {value} exceeds 63 bits")) + }) + } } +} - /// The highest address in the packet, or `None` for an empty packet. - pub fn last_address(&self) -> Option { - let count = self.doublets.len() + self.sequences.len(); - (count > 0).then(|| FIRST_LINK_ADDRESS + count as u64 - 1) +impl Section { + fn write_header(&self, out: &mut Vec) -> ProtocolResult<()> { + let mut shape = + u64::from(width_code(self.width)?) | (self.arity.min << SHAPE_MIN_ARITY_SHIFT); + if self.gap != 0 { + shape |= SHAPE_HAS_GAP; + } + if !self.arity.is_fixed() { + shape |= SHAPE_VARIABLE_ARITY; + } + write_leb128(out, shape); + if self.gap != 0 { + write_leb128(out, self.gap); + } + if !self.arity.is_fixed() { + write_leb128(out, self.arity.extra()); + } + write_leb128(out, self.links.len() as u64); + Ok(()) } - /// Width used by every reference of the link at `address`. - pub fn width_at(&self, address: u64) -> u8 { - self.min_width - .max(address_tier(address, self.external_references)) + /// Reads a section header, returning the still empty section and its + /// link count. + fn read_header(reader: &mut dyn Read) -> ProtocolResult<(Self, u64)> { + let shape = read_leb128(reader)?; + let width = width_from_code(shape & SHAPE_WIDTH_BITS)?; + let gap = if shape & SHAPE_HAS_GAP != 0 { + read_leb128(reader)? + } else { + 0 + }; + let extra = if shape & SHAPE_VARIABLE_ARITY != 0 { + Some(read_leb128(reader)?) + } else { + None + }; + let arity = ArityRange::from_shape(shape >> SHAPE_MIN_ARITY_SHIFT, extra)?; + let count = read_leb128(reader)?; + let section = Section { + gap, + arity, + width, + links: Vec::new(), + }; + Ok((section, count)) } - /// The smallest `min_width` able to encode the packet. - /// - /// With `uniform` set, the result is at least the tier of the last - /// address, so every reference in the packet has the same width. - pub fn required_min_width(&self, uniform: bool) -> ProtocolResult { - let mut needed = 1u8; - let mut note = |address: u64, need: u8| { - if need > address_tier(address, self.external_references) { - needed = needed.max(need); - } - }; - for (index, &(source, target)) in self.doublets.iter().enumerate() { - let address = self.doublet_address(index); - note( - address, - self.reference_need(source)? - .max(self.reference_need(target)?), - ); - } - for (index, items) in self.sequences.iter().enumerate() { - let address = self.sequence_address(index); - let size_need = WIDTHS - .into_iter() - .find(|&width| unsigned_capacity(width) >= items.len() as u64) - .unwrap_or(8); - let mut need = size_need; - for &item in items { - need = need.max(self.reference_need(item)?); + fn validate(&self) -> ProtocolResult<()> { + width_code(self.width)?; + self.arity.validate().map_err(ProtocolError::Unencodable)?; + for link in &self.links { + if !self.arity.contains(link.len() as u64) { + return Err(ProtocolError::Unencodable(format!( + "a link of {} references in a section of arity {}", + link.len(), + self.arity + ))); } - note(address, need); } - if uniform { - if let Some(last) = self.last_address() { - needed = needed.max(address_tier(last, self.external_references)); - } + Ok(()) + } +} + +impl LinksPacket { + /// An empty packet. + pub fn new(external_references: bool) -> Self { + Self { + external_references, + sections: Vec::new(), } - Ok(needed) } - fn reference_need(&self, reference: Reference) -> ProtocolResult { - match reference { - Reference::Internal(_) => Ok(1), - Reference::External(value) => { - if !self.external_references { - return Err(ProtocolError::Unencodable( - "external reference in a packet without external references".into(), - )); - } - WIDTHS - .into_iter() - .find(|&width| external_capacity(width) >= value) - .ok_or_else(|| { - ProtocolError::Unencodable(format!( - "external value {value} exceeds 63 bits" - )) - }) - } + /// Lays out `links` — `(address, references)` in ascending address + /// order — in as few bytes as the format allows. + /// + /// Holes between addresses start new sections. Without `packed_widths` + /// every section uses the width of the widest reference, so all + /// references have the same size. With it each section gets the + /// narrowest width its links need and sections split wherever that saves + /// bytes; the result is never larger than the uniform one. + pub fn pack( + external_references: bool, + links: &[(u64, Vec)], + packed_widths: bool, + ) -> ProtocolResult { + let planner = SectionPlanner::new(external_references, links)?; + let uniform_layout = planner.plan(false); + let uniform = Self::lay_out(external_references, links, &uniform_layout); + if !packed_widths { + return Ok(uniform); } + let packed_layout = planner.plan(true); + if packed_layout == uniform_layout { + return Ok(uniform); + } + let packed = Self::lay_out(external_references, links, &packed_layout); + Ok(if packed.to_bytes()?.len() < uniform.to_bytes()?.len() { + packed + } else { + uniform + }) } - fn header_byte(&self) -> ProtocolResult { - let mut header = BINARY_VERSION_1 | (width_code(self.min_width)? << WIDTH_SHIFT); - if self.external_references { - header |= FLAG_EXTERNAL_REFERENCES; + /// Splits `links` into sections of `(link count, width)`. + fn lay_out( + external_references: bool, + links: &[(u64, Vec)], + layout: &[(usize, u8)], + ) -> Self { + let mut sections = Vec::with_capacity(layout.len()); + let mut next_address = 1u64; + let mut remaining = links; + for &(count, width) in layout { + let (members, rest) = remaining.split_at(count); + remaining = rest; + let start = members[0].0; + let (shortest, longest) = members + .iter() + .map(|(_, link)| link.len() as u64) + .fold((u64::MAX, 0), |(shortest, longest), length| { + (shortest.min(length), longest.max(length)) + }); + sections.push(Section { + gap: start - next_address, + arity: ArityRange::between(shortest, longest), + width, + links: members.iter().map(|(_, link)| link.clone()).collect(), + }); + next_address = start + count as u64; } - if self.sequences_section { - header |= FLAG_SEQUENCES; + Self { + external_references, + sections, } - Ok(header) } - fn raw_reference(&self, reference: Reference, address: u64, width: u8) -> ProtocolResult { - match reference { - Reference::Internal(target) => { - if target >= address { - return Err(ProtocolError::Unencodable(format!( - "link {address} refers forward to {target}" - ))); - } - Ok(target) - } - Reference::External(value) => { - if !self.external_references || value > external_capacity(width) { - return Err(ProtocolError::Unencodable(format!( - "external value {value} does not fit {width} byte(s) at link {address}" - ))); - } - Ok(encode_external(value, width)) + /// Every link with its address, in address order. + pub fn links(&self) -> impl Iterator + '_ { + let mut next_address = 1u64; + self.sections.iter().flat_map(move |section| { + let start = next_address.saturating_add(section.gap); + next_address = start.saturating_add(section.links.len() as u64); + section + .links + .iter() + .enumerate() + .map(move |(index, link)| (start + index as u64, link.as_slice())) + }) + } + + /// The number of links in the packet. + pub fn link_count(&self) -> u64 { + self.sections + .iter() + .map(|section| section.links.len() as u64) + .sum() + } + + fn is_compact(&self) -> bool { + match self.sections.as_slice() { + [] => true, + [only] => { + only.gap == COMPACT_GAP + && only.arity == ArityRange::DOUBLETS + && !only.links.is_empty() } + _ => false, } } @@ -300,40 +546,61 @@ impl LinksPacket { /// Writes the packet to `writer`. pub fn write_to(&self, writer: &mut dyn Write) -> ProtocolResult<()> { - if !self.sequences_section && !self.sequences.is_empty() { - return Err(ProtocolError::Unencodable( - "sequences in a packet without a sequence section".into(), - )); - } - let mut out = vec![self.header_byte()?]; - write_leb128(&mut out, self.doublets.len() as u64); - if self.sequences_section { - write_leb128(&mut out, self.sequences.len() as u64); - } - for (index, &(source, target)) in self.doublets.iter().enumerate() { - let address = self.doublet_address(index); - let width = self.width_at(address); - write_raw(&mut out, self.raw_reference(source, address, width)?, width); - write_raw(&mut out, self.raw_reference(target, address, width)?, width); - } - for (index, items) in self.sequences.iter().enumerate() { - let address = self.sequence_address(index); - let width = self.width_at(address); - let size = items.len() as u64; - if size > unsigned_capacity(width) { - return Err(ProtocolError::Unencodable(format!( - "sequence size {size} does not fit {width} byte(s) at link {address}" - ))); + let mut header = BINARY_VERSION_1; + if self.external_references { + header |= FLAG_EXTERNAL_REFERENCES; + } + let mut out = Vec::new(); + let mut next_address = 1u64; + for section in &self.sections { + section.validate()?; + next_address = next_address + .checked_add(section.gap) + .and_then(|start| start.checked_add(section.links.len() as u64)) + .ok_or_else(|| ProtocolError::Unencodable("addresses overflow 64 bits".into()))?; + } + if self.is_compact() { + let width = self.sections.first().map_or(1, |section| section.width); + header |= width_code(width)? << WIDTH_SHIFT; + out.push(header); + write_leb128(&mut out, self.link_count()); + } else { + out.push(header | FLAG_EXPLICIT_LAYOUT); + write_leb128(&mut out, self.sections.len() as u64); + for section in &self.sections { + section.write_header(&mut out)?; } - write_raw(&mut out, size, width); - for &item in items { - write_raw(&mut out, self.raw_reference(item, address, width)?, width); + } + for section in &self.sections { + for link in §ion.links { + if !section.arity.is_fixed() { + write_leb128(&mut out, link.len() as u64 - section.arity.min); + } + for &reference in link { + write_raw( + &mut out, + self.raw_reference(reference, section.width)?, + section.width, + ); + } } } writer.write_all(&out)?; Ok(()) } + fn raw_reference(&self, reference: Reference, width: u8) -> ProtocolResult { + if reference_width(reference, self.external_references)? > width { + return Err(ProtocolError::Unencodable(format!( + "{reference:?} does not fit {width} byte(s)" + ))); + } + Ok(match reference { + Reference::Internal(address) => address, + Reference::External(value) => encode_external(value, width), + }) + } + /// Parses a complete packet; trailing bytes are an error. pub fn from_bytes(bytes: &[u8], limits: &DecodeLimits) -> ProtocolResult { let mut cursor = bytes; @@ -360,73 +627,205 @@ impl LinksPacket { "unsupported binary header byte 0x{header:02X}" ))); } - let mut packet = LinksPacket { - external_references: header & FLAG_EXTERNAL_REFERENCES != 0, - sequences_section: header & FLAG_SEQUENCES != 0, - min_width: WIDTHS[usize::from((header >> WIDTH_SHIFT) & 0b11)], - ..LinksPacket::default() - }; - let doublet_count = read_leb128(reader)?; - let sequence_count = if packet.sequences_section { - read_leb128(reader)? + let mut packet = LinksPacket::new(header & FLAG_EXTERNAL_REFERENCES != 0); + let mut counts = Vec::new(); + if header & FLAG_EXPLICIT_LAYOUT == 0 { + let width = width_from_code(u64::from((header & WIDTH_BITS) >> WIDTH_SHIFT))?; + let count = read_leb128(reader)?; + if count > 0 { + packet.sections.push(Section { + gap: COMPACT_GAP, + arity: ArityRange::DOUBLETS, + width, + links: Vec::new(), + }); + counts.push(count); + } } else { - 0 - }; - let total = doublet_count - .checked_add(sequence_count) + if header & WIDTH_BITS != 0 { + return Err(ProtocolError::malformed( + "the explicit layout keeps the header width bits clear", + )); + } + let section_count = read_leb128(reader)?; + if section_count > limits.max_links { + return Err(Self::too_many_links(limits)); + } + let mut next_address = 1u64; + for _ in 0..section_count { + let (section, count) = Section::read_header(reader)?; + next_address = next_address + .checked_add(section.gap) + .and_then(|start| start.checked_add(count)) + .ok_or_else(|| ProtocolError::malformed("addresses overflow 64 bits"))?; + packet.sections.push(section); + counts.push(count); + } + } + counts + .iter() + .try_fold(0u64, |total, &count| total.checked_add(count)) .filter(|&total| total <= limits.max_links) - .ok_or_else(|| { - ProtocolError::LimitExceeded(format!( - "packet declares {doublet_count} + {sequence_count} links, limit is {}", - limits.max_links - )) - })?; - packet.doublets.reserve(total.min(4096) as usize); - for index in 0..doublet_count as usize { - let address = packet.doublet_address(index); - let width = packet.width_at(address); - let source = packet.read_reference(reader, address, width)?; - let target = packet.read_reference(reader, address, width)?; - packet.doublets.push((source, target)); - } - let mut items_left = limits.max_sequence_items; - for index in 0..sequence_count as usize { - let address = packet.sequence_address(index); - let width = packet.width_at(address); - let size = read_raw(reader, width)?; - items_left = items_left.checked_sub(size).ok_or_else(|| { - ProtocolError::LimitExceeded(format!( - "sequence items exceed the limit of {}", - limits.max_sequence_items - )) - })?; - let mut items = Vec::with_capacity(size.min(4096) as usize); - for _ in 0..size { - items.push(packet.read_reference(reader, address, width)?); + .ok_or_else(|| Self::too_many_links(limits))?; + let mut references_left = limits.max_references; + let external_references = packet.external_references; + for (section, count) in packet.sections.iter_mut().zip(counts) { + section.links.reserve(count.min(4096) as usize); + for _ in 0..count { + let length = if section.arity.is_fixed() { + section.arity.min + } else { + let length = read_leb128(reader)? + .checked_add(section.arity.min) + .filter(|&length| section.arity.contains(length)) + .ok_or_else(|| { + ProtocolError::malformed(format!( + "link length outside the section arity {}", + section.arity + )) + })?; + length + }; + references_left = references_left.checked_sub(length).ok_or_else(|| { + ProtocolError::LimitExceeded(format!( + "references exceed the limit of {}", + limits.max_references + )) + })?; + let mut link = Vec::with_capacity(length.min(4096) as usize); + for _ in 0..length { + let raw = read_raw(reader, section.width)?; + link.push( + match decode_external(raw, section.width).filter(|_| external_references) { + Some(value) => Reference::External(value), + None => Reference::Internal(raw), + }, + ); + } + section.links.push(link); } - packet.sequences.push(items); } Ok(Some(packet)) } - fn read_reference( - &self, - reader: &mut dyn Read, - address: u64, - width: u8, - ) -> ProtocolResult { - let raw = read_raw(reader, width)?; - if self.external_references { - if let Some(value) = decode_external(raw, width) { - return Ok(Reference::External(value)); + fn too_many_links(limits: &DecodeLimits) -> ProtocolError { + ProtocolError::LimitExceeded(format!( + "packet declares more than {} links", + limits.max_links + )) + } +} + +/// Estimated bytes of a section header (shape and count), used to weigh a split. +const SECTION_HEADER_ESTIMATE: u64 = 2; +/// Estimated extra bytes of a variable-arity section: its `extra_arity` +/// header field, and the length prefix of each link. +const VARIABLE_ARITY_ESTIMATE: u64 = 1; +const UNREACHABLE: u64 = u64::MAX / 4; +/// Planner states: a width code (0..4) times fixed (0) or variable (1) arity. +const STATES: usize = WIDTHS.len() * 2; + +/// Splits links into sections with a linear dynamic program. +/// +/// After link `k`, `cost[s]` is the fewest estimated bytes for links +/// `0..=k` with link `k` in a section of state `s`. A link either continues +/// the section of the previous link (same state, no hole between them and, +/// for a fixed arity, the same length) or opens a new section after the +/// cheapest previous state, paying for a section header. +struct SectionPlanner<'a> { + links: &'a [(u64, Vec)], + needs: Vec, +} + +impl<'a> SectionPlanner<'a> { + fn new(external_references: bool, links: &'a [(u64, Vec)]) -> ProtocolResult { + let mut needs = Vec::with_capacity(links.len()); + let mut previous_address = 0u64; + for (address, link) in links { + if *address <= previous_address { + return Err(ProtocolError::Unencodable(format!( + "link addresses must ascend from 1, got {address} after {previous_address}" + ))); + } + if link.is_empty() { + return Err(ProtocolError::Unencodable(format!( + "link {address} has no references" + ))); } + previous_address = *address; + let mut need = 1u8; + for &reference in link { + need = need.max(reference_width(reference, external_references)?); + } + needs.push(need); } - if raw >= address { - return Err(ProtocolError::malformed(format!( - "link {address} refers to {raw}, which is not an earlier link" - ))); + Ok(Self { links, needs }) + } + + fn first_cheapest(costs: &[u64; STATES]) -> usize { + let mut best = 0; + for state in 1..STATES { + if costs[state] < costs[best] { + best = state; + } + } + best + } + + /// The sections as `(link count, width)` pairs; with `packed_widths` + /// every width may be used, otherwise only the widest one needed. + fn plan(&self, packed_widths: bool) -> Vec<(usize, u8)> { + if self.links.is_empty() { + return Vec::new(); + } + let widest = self.needs.iter().copied().max().unwrap_or(1); + let allowed_widths = WIDTHS.map(|width| packed_widths || width == widest); + let mut opens_section = vec![0u8; self.links.len()]; + let mut previous_best = vec![0u8; self.links.len()]; + let mut costs = [UNREACHABLE; STATES]; + for (index, (address, link)) in self.links.iter().enumerate() { + let best = Self::first_cheapest(&costs); + previous_best[index] = best as u8; + let cheapest_before = if index == 0 { 0 } else { costs[best] }; + let continues = index > 0 && self.links[index - 1].0 + 1 == *address; + let same_length = index > 0 && self.links[index - 1].1.len() == link.len(); + let mut next = [UNREACHABLE; STATES]; + for state in 0..STATES { + let width_index = state / 2; + let variable = state % 2 == 1; + let width = WIDTHS[width_index]; + if !allowed_widths[width_index] || width < self.needs[index] { + continue; + } + let variable_estimate = if variable { VARIABLE_ARITY_ESTIMATE } else { 0 }; + let body = link.len() as u64 * u64::from(width) + variable_estimate; + let opening_cost = cheapest_before + SECTION_HEADER_ESTIMATE + variable_estimate; + let continuing_cost = if continues && (variable || same_length) { + costs[state] + } else { + UNREACHABLE + }; + if continuing_cost <= opening_cost { + next[state] = continuing_cost + body; + } else { + next[state] = opening_cost + body; + opens_section[index] |= 1 << state; + } + } + costs = next; + } + let mut sections = Vec::new(); + let mut state = Self::first_cheapest(&costs); + let mut end = self.links.len(); + for index in (0..self.links.len()).rev() { + if opens_section[index] & (1 << state) != 0 { + sections.push((end - index, WIDTHS[state / 2])); + end = index; + state = usize::from(previous_best[index]); + } } - Ok(Reference::Internal(raw)) + sections.reverse(); + sections } } diff --git a/rust/src/protocol/protocols.rs b/rust/src/protocol/protocols.rs index 5e0a319..308db6f 100644 --- a/rust/src/protocol/protocols.rs +++ b/rust/src/protocol/protocols.rs @@ -8,7 +8,7 @@ use super::error::{ProtocolError, ProtocolResult}; use super::format::{format_document, parse_document}; use super::mapping::{decode_document, encode_document, BinaryLinoOptions, LinoDocument}; -use super::packet::{address_tier, DecodeLimits, LinksPacket, BINARY_VERSION_1}; +use super::packet::{DecodeLimits, LinksPacket, BINARY_VERSION_1}; use links_notation::LiNo; use std::fmt::Debug; use std::io::{BufRead, BufReader, Read, Write}; @@ -235,14 +235,7 @@ pub fn read_any_document( let Some(packet) = LinksPacket::read_from(reader, limits)? else { return Ok(None); }; - let progressive = packet - .last_address() - .is_some_and(|last| packet.min_width < address_tier(last, packet.external_references)); - let options = BinaryLinoOptions { - external_references: packet.external_references, - sequences: packet.sequences_section, - progressive_widths: progressive, - }; + let options = BinaryLinoOptions::of_packet(&packet); let document = decode_document(&packet, limits)?; Ok(Some((document, MessageFormat::Binary(options)))) } diff --git a/rust/tests/cli_tcp_tests.rs b/rust/tests/cli_tcp_tests.rs index c11f059..68e0e1c 100644 --- a/rust/tests/cli_tcp_tests.rs +++ b/rust/tests/cli_tcp_tests.rs @@ -1,6 +1,7 @@ //! `clink --serve` and `clink --connect` end to end (issue #105). use link_cli::cli::{Cli, CliCommand}; +use link_cli::protocol::ArityRange; use std::io::{BufRead, BufReader}; use std::process::{Child, Command, Output, Stdio}; use tempfile::TempDir; @@ -20,20 +21,24 @@ fn parses_tcp_options() { "127.0.0.1:7878", "--protocol=binary", "--external-references", - "--sequences", - "true", - "--progressive-widths=on", + "--arity", + "2..3", + "--packed-widths=on", ]); assert_eq!(cli.serve.as_deref(), Some("127.0.0.1:7878")); assert_eq!(cli.protocol.as_deref(), Some("binary")); assert!(cli.external_references); - assert!(cli.sequences); - assert!(cli.progressive_widths); + assert_eq!(cli.arity, ArityRange::between(2, 3)); + assert!(cli.packed_widths); + + let cli = parse_run(&["clink", "--connect=localhost:1", "--arity=1.."]); + assert_eq!(cli.arity, ArityRange::at_least(1)); let cli = parse_run(&["clink", "--connect=localhost:1", "() ((1 1))"]); assert_eq!(cli.connect.as_deref(), Some("localhost:1")); assert_eq!(cli.query_arg.as_deref(), Some("() ((1 1))")); - assert!(!cli.external_references && !cli.sequences && !cli.progressive_widths); + assert!(!cli.external_references && !cli.packed_widths); + assert_eq!(cli.arity, ArityRange::DOUBLETS); assert!(Cli::help_text().contains("--serve ")); assert!(Cli::help_text().contains("--connect ")); } @@ -106,7 +111,8 @@ fn clients_query_a_served_database_over_both_protocols() { assert_eq!( server.stdout(&[ "--external-references", - "--sequences", + "--arity", + "1..", "((1: 1 1)) ((1: 1 2))" ]), "((1: 1 1)) ((1: 1 2))\n" @@ -124,7 +130,7 @@ fn clients_query_a_served_database_over_both_protocols() { assert!(!failure.status.success()); assert!(String::from_utf8_lossy(&failure.stderr).contains("server error")); - let conflicting = server.connect(&["--protocol", "text", "--sequences", ""]); + let conflicting = server.connect(&["--protocol", "text", "--arity", "2..3", ""]); assert!(!conflicting.status.success()); } diff --git a/rust/tests/common/mod.rs b/rust/tests/common/mod.rs index 9989f30..9b65cd2 100644 --- a/rust/tests/common/mod.rs +++ b/rust/tests/common/mod.rs @@ -3,8 +3,8 @@ #![allow(dead_code)] // Every test crate compiles this module but uses only part of it. use link_cli::protocol::{ - BinaryLinoOptions, BinaryLinoProtocol, LinksClient, LinksServer, LinoProtocol, ServerOptions, - TextLinoProtocol, + ArityRange, BinaryLinoOptions, BinaryLinoProtocol, LinksClient, LinksServer, LinoProtocol, + ServerOptions, TextLinoProtocol, }; use link_cli::{LinkStorage, NamedTypesDecorator}; use std::net::SocketAddr; @@ -63,20 +63,33 @@ impl Drop for RunningServer { } } -pub fn protocols() -> Vec> { - let mut protocols: Vec> = vec![Box::new(TextLinoProtocol::new())]; +/// Every combination of the binary options the tests cover: plain and +/// external references, doublets, doublets and triplets, and any arity, with +/// uniform and packed widths. +pub fn binary_options() -> Vec { + let mut options = Vec::new(); for external_references in [false, true] { - for sequences in [false, true] { - for progressive_widths in [false, true] { - protocols.push(Box::new(BinaryLinoProtocol::with_options( - BinaryLinoOptions { - external_references, - sequences, - progressive_widths, - }, - ))); + for arity in [ + ArityRange::DOUBLETS, + ArityRange::between(2, 3), + ArityRange::at_least(1), + ] { + for packed_widths in [false, true] { + options.push(BinaryLinoOptions { + external_references, + arity, + packed_widths, + }); } } } + options +} + +pub fn protocols() -> Vec> { + let mut protocols: Vec> = vec![Box::new(TextLinoProtocol::new())]; + for options in binary_options() { + protocols.push(Box::new(BinaryLinoProtocol::with_options(options))); + } protocols } diff --git a/rust/tests/protocol_packet_tests.rs b/rust/tests/protocol_packet_tests.rs index f1cf343..51a9451 100644 --- a/rust/tests/protocol_packet_tests.rs +++ b/rust/tests/protocol_packet_tests.rs @@ -1,13 +1,16 @@ //! Binary and text LiNo protocol codecs (issue #105). +mod common; + +use common::binary_options; use link_cli::external_reference; use link_cli::protocol::packet::{ address_tier, decode_external, encode_external, external_capacity, internal_capacity, }; use link_cli::protocol::{ decode_document, encode_document, format_document, format_reference, parse_document, - read_any_document, BinaryLinoOptions, BinaryLinoProtocol, DecodeLimits, LinksPacket, - LinoProtocol, MessageFormat, ProtocolError, Reference, TextLinoProtocol, + read_any_document, ArityRange, BinaryLinoOptions, BinaryLinoProtocol, DecodeLimits, + LinksPacket, LinoProtocol, MessageFormat, ProtocolError, Reference, Section, TextLinoProtocol, }; use std::io::Cursor; @@ -41,22 +44,6 @@ const CORPUS: &[&str] = &[ "(type: type type)", ]; -fn all_options() -> Vec { - let mut options = Vec::new(); - for external_references in [false, true] { - for sequences in [false, true] { - for progressive_widths in [false, true] { - options.push(BinaryLinoOptions { - external_references, - sequences, - progressive_widths, - }); - } - } - } - options -} - fn hex(bytes: &[u8]) -> String { bytes .iter() @@ -71,69 +58,103 @@ fn unhex(text: &str) -> Vec { .collect() } -fn encode_hex(text: &str, external_references: bool, sequences: bool) -> String { - let binary = BinaryLinoProtocol::with_options(BinaryLinoOptions { - external_references, - sequences, - progressive_widths: false, - }); - hex(&binary.encode(&parse_document(text).unwrap()).unwrap()) -} - -/// Golden vectors shared with the C# test suite: both implementations must -/// produce exactly these bytes. -const GOLDEN: &[(&str, bool, bool, &str)] = &[ - ( - "() ((1 1))", - false, - false, - "10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b", - ), - ( - "() ((1 1))", - false, - true, - "12 02 03 02 01 06 06 01 07 02 00 08 01 09", - ), - ( - "() ((1 1))", - true, - false, - "11 06 ff ff 06 00 04 07 00 08 09 00 04 0a", - ), - ( - "() ((1 1))", - true, - true, - "13 01 03 ff ff 01 06 02 00 07 01 08", - ), - ("hi", true, false, "11 05 97 00 98 06 03 07 08 00 04 09"), - ("hi", true, true, "13 00 02 03 03 98 97 01 06"), - ("", false, false, "10 00"), - ("", true, true, "13 00 00"), -]; +fn references_option(field: &str) -> bool { + match field { + "plain" => false, + "external" => true, + other => panic!("unknown references option {other:?}"), + } +} + +fn widths_option(field: &str) -> bool { + match field { + "uniform" => false, + "packed" => true, + other => panic!("unknown widths option {other:?}"), + } +} + +/// `address:reference reference…` separated by `;`, `#v` being external. +fn parse_links(text: &str) -> Vec<(u64, Vec)> { + text.split(';') + .map(|link| { + let (address, references) = link.split_once(':').unwrap(); + let references = references + .split(' ') + .map(|reference| match reference.strip_prefix('#') { + Some(value) => Reference::External(value.parse().unwrap()), + None => Reference::Internal(reference.parse().unwrap()), + }) + .collect(); + (address.parse().unwrap(), references) + }) + .collect() +} + +/// The golden vectors shared with the C# test suite: both implementations +/// must write and read exactly these bytes. +fn golden_vectors() -> impl Iterator> { + include_str!("../../docs/protocol/binary-links-notation-vectors.txt") + .lines() + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(|line| line.split('\t').collect()) +} #[test] -fn golden_vectors_are_stable() { - for &(text, external_references, sequences, expected) in GOLDEN { +fn golden_documents_are_stable() { + let mut checked = 0; + for fields in golden_vectors().filter(|fields| fields[0] == "document") { + let [_, text, references, arity, widths, expected] = fields[..] else { + panic!("bad document vector {fields:?}"); + }; + let text = text.replace("\\n", "\n"); + let options = BinaryLinoOptions { + external_references: references_option(references), + arity: arity.parse().unwrap(), + packed_widths: widths_option(widths), + }; + let document = parse_document(&text).unwrap(); + let binary = BinaryLinoProtocol::with_options(options); assert_eq!( - encode_hex(text, external_references, sequences), + hex(&binary.encode(&document).unwrap()), expected, - "{text:?} external={external_references} sequences={sequences}" - ); - let binary = BinaryLinoProtocol::new(); - assert_eq!( - binary.decode(&unhex(expected)).unwrap(), - parse_document(text).unwrap() + "{text:?} with {options:?}" ); + assert_eq!(binary.decode(&unhex(expected)).unwrap(), document); + checked += 1; + } + assert!(checked >= 80, "only {checked} document vectors"); +} + +#[test] +fn golden_packets_are_stable() { + let mut checked = 0; + for fields in golden_vectors().filter(|fields| fields[0] == "links") { + let [_, links, references, widths, expected] = fields[..] else { + panic!("bad links vector {fields:?}"); + }; + let links = parse_links(links); + let packet = + LinksPacket::pack(references_option(references), &links, widths_option(widths)) + .unwrap(); + assert_eq!(hex(&packet.to_bytes().unwrap()), expected, "{links:?}"); + let read = LinksPacket::from_bytes(&unhex(expected), &DecodeLimits::default()).unwrap(); + assert_eq!(read, packet); + let read_links = read + .links() + .map(|(address, link)| (address, link.to_vec())) + .collect::>(); + assert_eq!(read_links, links); + checked += 1; } + assert!(checked >= 10, "only {checked} links vectors"); } #[test] fn every_option_set_round_trips_the_corpus() { for text in CORPUS { let document = parse_document(text).unwrap(); - for options in all_options() { + for options in binary_options() { let binary = BinaryLinoProtocol::with_options(options); let bytes = binary.encode(&document).unwrap(); assert!((0x10..=0x1F).contains(&bytes[0])); @@ -255,68 +276,293 @@ fn many_links(count: usize) -> String { .join("\n") } +fn section_widths(packet: &LinksPacket) -> Vec { + packet + .sections + .iter() + .map(|section| section.width) + .collect() +} + #[test] -fn uniform_width_grows_past_256_addresses() { +fn uniform_widths_grow_past_256_addresses_and_packed_widths_stay_small() { let document = parse_document(&many_links(3)).unwrap(); let small = encode_document(&document, BinaryLinoOptions::default()).unwrap(); - assert!(small.last_address().unwrap() <= 255); - assert_eq!(small.min_width, 1); + assert_eq!(section_widths(&small), [1]); let document = parse_document(&many_links(400)).unwrap(); let uniform = encode_document(&document, BinaryLinoOptions::default()).unwrap(); - assert!(uniform.last_address().unwrap() > 255); - assert_eq!(uniform.min_width, 2); + assert_eq!(section_widths(&uniform), [2]); let uniform_bytes = uniform.to_bytes().unwrap(); let header = 1 + 2; // header byte + two-byte LEB128 count - assert_eq!(uniform_bytes.len(), header + uniform.doublets.len() * 4); + assert_eq!( + uniform_bytes.len() as u64, + header + uniform.link_count() * 2 * 2 + ); - let progressive = encode_document( + let packed = encode_document( &document, - BinaryLinoOptions::default().with_progressive_widths(true), + BinaryLinoOptions::default().with_packed_widths(true), ) .unwrap(); - assert_eq!(progressive.min_width, 1); - let progressive_bytes = progressive.to_bytes().unwrap(); - assert!(progressive_bytes.len() < uniform_bytes.len()); + assert_eq!(section_widths(&packed)[0], 1); + assert!(section_widths(&packed).contains(&2)); + let packed_bytes = packed.to_bytes().unwrap(); + assert!(packed_bytes.len() < uniform_bytes.len()); let limits = DecodeLimits::default(); - for bytes in [uniform_bytes, progressive_bytes] { + for bytes in [uniform_bytes, packed_bytes] { let packet = LinksPacket::from_bytes(&bytes, &limits).unwrap(); assert_eq!(decode_document(&packet, &limits).unwrap(), document); } } #[test] -fn large_external_values_raise_the_minimum_width() { +fn packed_widths_never_take_more_bytes_than_uniform_ones() { + let mut corpus = CORPUS + .iter() + .map(|text| text.to_string()) + .collect::>(); + corpus.push(many_links(200)); + corpus.push("(1000 1)".into()); + for text in &corpus { + let document = parse_document(text).unwrap(); + for options in binary_options() + .into_iter() + .filter(|options| options.packed_widths) + { + let packed = encode_document(&document, options).unwrap(); + let uniform = encode_document(&document, options.with_packed_widths(false)).unwrap(); + assert!( + packed.to_bytes().unwrap().len() <= uniform.to_bytes().unwrap().len(), + "{text:?} with {options:?}" + ); + } + } +} + +#[test] +fn large_external_values_widen_only_their_section() { let options = BinaryLinoOptions::default() .with_external_references(true) - .with_progressive_widths(true); - let packet = encode_document(&parse_document("(1000 1)").unwrap(), options).unwrap(); - assert_eq!(packet.min_width, 2); + .with_packed_widths(true); let packet = encode_document(&parse_document("(100 1)").unwrap(), options).unwrap(); - assert_eq!(packet.min_width, 1); + assert_eq!(section_widths(&packet), [1]); + let packet = encode_document(&parse_document("(70000 1)").unwrap(), options).unwrap(); + assert_eq!(section_widths(&packet), [4, 1]); // Beyond 63 bits the number falls back to in-band unary links. let packet = encode_document(&parse_document("18446744073709551615").unwrap(), options).unwrap(); - assert!(packet.doublets.len() > 60); + assert!(packet.link_count() > 60); +} + +#[test] +fn arity_options_choose_doublets_triplets_or_any_length() { + let document = parse_document("(a b c)\n(d e)").unwrap(); + let arities = |arity: ArityRange| { + let options = BinaryLinoOptions::default() + .with_external_references(true) + .with_arity(arity); + let packet = encode_document(&document, options).unwrap(); + assert_eq!( + BinaryLinoProtocol::new() + .decode(&packet.to_bytes().unwrap()) + .unwrap(), + document + ); + packet + .links() + .map(|(_, link)| link.len()) + .collect::>() + }; + assert!(arities(ArityRange::DOUBLETS) + .iter() + .all(|&length| length == 2)); + // `(a b c)` is one triplet; the strings `(String code point)` stay doublets. + let triplets = arities(ArityRange::between(2, 3)); + assert_eq!(triplets.iter().filter(|&&length| length == 3).count(), 1); + assert!(triplets.iter().all(|&length| length == 2 || length == 3)); + assert_eq!(arities(ArityRange::at_least(1)), triplets); + // Only any length turns a one-item list, here the root, into one link. + let single = parse_document("a").unwrap(); + let options = BinaryLinoOptions::default().with_external_references(true); + let lengths = |options: BinaryLinoOptions| { + let packet = encode_document(&single, options).unwrap(); + packet + .links() + .map(|(_, link)| link.len()) + .collect::>() + }; + assert_eq!(lengths(options), [2, 2, 2, 2]); + assert_eq!( + lengths(options.with_arity(ArityRange::at_least(1))), + [2, 2, 1] + ); + // Long lists only become single links when the range allows their length. + let long = parse_document("(a b c d e f g h)").unwrap(); + for (arity, single_link) in [ + (ArityRange::between(2, 3), false), + (ArityRange::between(2, 8), true), + (ArityRange::at_least(2), true), + ] { + let options = BinaryLinoOptions::default() + .with_external_references(true) + .with_arity(arity); + let packet = encode_document(&long, options).unwrap(); + let longest = packet.links().map(|(_, link)| link.len()).max().unwrap(); + assert_eq!(longest == 8, single_link, "{arity}"); + } +} + +#[test] +fn arities_without_doublets_are_unencodable() { + for arity in [ + ArityRange::exactly(3), + ArityRange::exactly(1), + ArityRange::at_least(3), + ] { + assert!(matches!( + encode_document( + &parse_document("(a b)").unwrap(), + BinaryLinoOptions::default().with_arity(arity) + ), + Err(ProtocolError::Unencodable(_)) + )); + } +} + +#[test] +fn arity_ranges_parse_and_display() { + for (text, range) in [ + ("2", ArityRange::DOUBLETS), + ("2..3", ArityRange::between(2, 3)), + ("1..", ArityRange::at_least(1)), + ("3..3", ArityRange::exactly(3)), + ] { + assert_eq!(text.parse::(), Ok(range)); + } + assert_eq!(ArityRange::between(3, 3).to_string(), "3"); + assert_eq!(ArityRange::between(2, 3).to_string(), "2..3"); + assert_eq!(ArityRange::at_least(1).to_string(), "1.."); + assert_eq!(ArityRange::default(), ArityRange::DOUBLETS); + for invalid in ["", "0", "0..2", "3..2", "a", "1..b", "..3", "-1", " 2"] { + assert!(invalid.parse::().is_err(), "{invalid:?}"); + } + assert!(ArityRange::at_least(2).contains(1_000)); + assert!(!ArityRange::between(2, 3).contains(4)); + assert!(ArityRange::exactly(2).is_fixed() && !ArityRange::at_least(2).is_fixed()); +} + +#[test] +fn reply_options_follow_the_received_packet() { + let document = parse_document("(a b c) 70000").unwrap(); + let options = BinaryLinoOptions::default() + .with_external_references(true) + .with_arity(ArityRange::between(2, 3)) + .with_packed_widths(true); + let packet = encode_document(&document, options).unwrap(); + assert_eq!(BinaryLinoOptions::of_packet(&packet), options); + let packet = encode_document(&document, BinaryLinoOptions::default()).unwrap(); + assert_eq!( + BinaryLinoOptions::of_packet(&packet), + BinaryLinoOptions::default() + ); } #[test] fn hand_built_packets_decode() { // One doublet (One One) is 2^1 in unary, so `(Number 7)` is the number 2. + let links = [(1, 1), (2, 6), (7, 0), (4, 8)] + .map(|(source, target)| vec![Reference::Internal(source), Reference::Internal(target)]); let packet = LinksPacket { - min_width: 1, - doublets: vec![ - (Reference::Internal(1), Reference::Internal(1)), - (Reference::Internal(2), Reference::Internal(6)), - (Reference::Internal(7), Reference::NULL), - (Reference::Internal(4), Reference::Internal(8)), - ], - ..LinksPacket::default() + external_references: false, + sections: vec![Section { + gap: 5, + arity: ArityRange::DOUBLETS, + width: 1, + links: links.to_vec(), + }], }; let bytes = packet.to_bytes().unwrap(); assert_eq!(hex(&bytes), "10 04 01 01 02 06 07 00 04 08"); let protocol = BinaryLinoProtocol::new(); assert_eq!(format_document(&protocol.decode(&bytes).unwrap()), "2"); + + // The same links as a variable section of width 2 use the explicit layout. + let packet = LinksPacket { + sections: vec![Section { + arity: ArityRange::at_least(1), + width: 2, + ..packet.sections[0].clone() + }], + ..packet + }; + let bytes = packet.to_bytes().unwrap(); + assert_eq!( + hex(&bytes), + "12 01 1d 05 00 04 01 01 00 01 00 01 02 00 06 00 01 07 00 00 00 01 04 00 08 00" + ); + assert_eq!(format_document(&protocol.decode(&bytes).unwrap()), "2"); +} + +#[test] +fn packing_rejects_links_it_cannot_lay_out() { + let unencodable = |links: &[(u64, Vec)], external_references: bool| { + assert!( + matches!( + LinksPacket::pack(external_references, links, true), + Err(ProtocolError::Unencodable(_)) + ), + "{links:?}" + ); + }; + let link = |references: &[u64]| { + references + .iter() + .copied() + .map(Reference::Internal) + .collect() + }; + unencodable(&[(0, link(&[1]))], false); // address 0 is null + unencodable(&[(2, link(&[1])), (1, link(&[1]))], false); // descending + unencodable(&[(2, link(&[1])), (2, link(&[1]))], false); // repeated + unencodable(&[(1, Vec::new())], false); // no references + unencodable(&[(1, vec![Reference::External(1)])], false); + unencodable(&[(1, vec![Reference::External(1 << 63)])], true); + unencodable(&[(1, link(&[1 << 63]))], true); // the top bit marks externals + assert!(LinksPacket::pack(false, &[(1, link(&[u64::MAX]))], true).is_ok()); +} + +#[test] +fn writing_rejects_sections_that_do_not_hold_their_links() { + let section = |arity: ArityRange, width: u8, links: Vec>| LinksPacket { + external_references: false, + sections: vec![Section { + gap: 0, + arity, + width, + links, + }], + }; + let unencodable = |packet: LinksPacket| { + assert!( + matches!(packet.to_bytes(), Err(ProtocolError::Unencodable(_))), + "{packet:?}" + ); + }; + let pair = vec![Reference::Internal(1), Reference::Internal(1)]; + unencodable(section(ArityRange::exactly(3), 1, vec![pair.clone()])); + unencodable(section(ArityRange::exactly(0), 1, Vec::new())); + unencodable(section(ArityRange::between(3, 2), 1, Vec::new())); + unencodable(section(ArityRange::exactly(1 << 62), 1, Vec::new())); + unencodable(section(ArityRange::DOUBLETS, 3, vec![pair.clone()])); + unencodable(section( + ArityRange::DOUBLETS, + 1, + vec![vec![Reference::Internal(256), Reference::NULL]], + )); + let mut gaps = section(ArityRange::DOUBLETS, 1, vec![pair]); + gaps.sections[0].gap = u64::MAX; + unencodable(gaps); } fn expect_error(bytes: &[u8], limits: DecodeLimits) -> ProtocolError { @@ -349,53 +595,78 @@ fn malformed_packets_are_rejected() { malformed(&[0x10, 0x01, 0x01, 0x01]); // root is not a list malformed(&[0x10, 0x02, 0x00, 0x02, 0x04, 0x06]); // marker 2 used as a value malformed(&[0x10, 0x01, 0x04, 0x02]); // a bare marker in a list chain - // (String (0x110000)) is not a valid code point. - let invalid = LinksPacket { - external_references: true, - min_width: 4, - doublets: vec![ - (Reference::External(0x11_0000), Reference::NULL), - (Reference::Internal(3), Reference::Internal(6)), - (Reference::Internal(7), Reference::NULL), - (Reference::Internal(4), Reference::Internal(8)), - ], - ..LinksPacket::default() - }; - malformed(&invalid.to_bytes().unwrap()); + malformed(&[0x16, 0x00]); // the explicit layout keeps the width bits clear + malformed(&[0x12, 0x01, 0x04, 0x05, 0x00]); // arity 0 + malformed(&[0x12, 0x01, 0x18, 0x01, 0x01, 0x05, 0x00]); // length 6 in arity 1..2 + malformed(&[ + 0x12, 0x01, 0xf8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, + ]); // arity overflows + malformed(&[ + 0x12, 0x02, 0x04, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x01, 0x20, + 0x01, + ]); // addresses overflow + malformed(&[0x12, 0x01, 0x24, 0x06, 0x01, 0x01, 0x01]); // LiNo links start at 6, not 7 + malformed(&[ + 0x12, 0x02, 0x24, 0x05, 0x01, 0x24, 0x01, 0x01, 0x01, 0x01, 0x04, 0x06, + ]); // a hole + // (String (0x110000)) is not a valid code point. + let links = [ + vec![Reference::External(0x11_0000), Reference::NULL], + vec![Reference::Internal(3), Reference::Internal(6)], + vec![Reference::Internal(7), Reference::NULL], + vec![Reference::Internal(4), Reference::Internal(8)], + ]; + let links = (6..).zip(links).collect::>(); + malformed( + &LinksPacket::pack(true, &links, false) + .unwrap() + .to_bytes() + .unwrap(), + ); } #[test] fn hostile_packets_hit_limits() { - let limits = DecodeLimits { + let limited = |bytes: &[u8], limits: DecodeLimits| { + assert!( + matches!(expect_error(bytes, limits), ProtocolError::LimitExceeded(_)), + "{}", + hex(bytes) + ); + }; + let few_links = DecodeLimits { max_links: 8, ..DecodeLimits::default() }; - assert!(matches!( - expect_error(&[0x10, 0x09], limits), - ProtocolError::LimitExceeded(_) - )); + limited(&[0x10, 0x09], few_links); + limited(&[0x12, 0x09], few_links); // more sections than links allowed + limited(&[0x12, 0x02, 0x20, 0x05, 0x20, 0x05], few_links); // 10 in all + let few_references = DecodeLimits { + max_references: 5, + ..DecodeLimits::default() + }; + limited(&[0x10, 0x03, 1, 1, 1, 1, 1, 1], few_references); + limited(&[0x12, 0x01, 0x18, 0x00, 0x01, 0x05], few_references); // one link of 6 // A doubling chain `d(k) = (d(k-1) d(k-1))` expands to 2^k nodes. - let mut doublets = vec![(Reference::NULL, Reference::NULL)]; + let mut links = vec![vec![Reference::NULL, Reference::NULL]]; for address in 6..60u64 { - doublets.push((Reference::Internal(address), Reference::Internal(address))); + links.push(vec![ + Reference::Internal(address), + Reference::Internal(address), + ]); } - let last = 5 + doublets.len() as u64; - doublets.push((Reference::Internal(last), Reference::NULL)); - doublets.push((Reference::Internal(4), Reference::Internal(last + 1))); - let bomb = LinksPacket { - min_width: 1, - doublets, - ..LinksPacket::default() - }; + let last = 5 + links.len() as u64; + links.push(vec![Reference::Internal(last), Reference::NULL]); + links.push(vec![Reference::Internal(4), Reference::Internal(last + 1)]); + let links = (6..).zip(links).collect::>(); + let bomb = LinksPacket::pack(false, &links, false).unwrap(); let small_budget = DecodeLimits { max_nodes: 1 << 12, ..DecodeLimits::default() }; - assert!(matches!( - expect_error(&bomb.to_bytes().unwrap(), small_budget), - ProtocolError::LimitExceeded(_) - )); + limited(&bomb.to_bytes().unwrap(), small_budget); let deep = format!("{}x{}", "(y ".repeat(40), ")".repeat(40)); let bytes = BinaryLinoProtocol::new() @@ -405,10 +676,8 @@ fn hostile_packets_hit_limits() { max_depth: 10, ..DecodeLimits::default() }; - assert!(matches!( - expect_error(&bytes, shallow), - ProtocolError::LimitExceeded(_) - )); + limited(&bytes, shallow); + assert!(DecodeLimits::unlimited().max_links == u64::MAX); } #[test] @@ -469,9 +738,7 @@ fn text_messages_accept_crlf_and_reject_truncation() { #[test] fn protocols_are_detected_per_message() { let document = parse_document("() ((1 1))").unwrap(); - let options = BinaryLinoOptions::default() - .with_external_references(true) - .with_sequences(true); + let options = BinaryLinoOptions::default().with_external_references(true); let mut wire = Vec::new(); TextLinoProtocol::new() .write_document(&mut wire, &document) diff --git a/rust/tests/protocol_tcp_tests.rs b/rust/tests/protocol_tcp_tests.rs index e8da460..aed09ec 100644 --- a/rust/tests/protocol_tcp_tests.rs +++ b/rust/tests/protocol_tcp_tests.rs @@ -4,8 +4,8 @@ mod common; use common::{protocols, RunningServer}; use link_cli::protocol::{ - AcceptedProtocols, BinaryLinoOptions, BinaryLinoProtocol, LinksClient, ProtocolError, - ServerOptions, TextLinoProtocol, + AcceptedProtocols, ArityRange, BinaryLinoOptions, BinaryLinoProtocol, LinksClient, + ProtocolError, ServerOptions, TextLinoProtocol, }; use std::io::{BufRead, BufReader, Read, Write}; use std::net::TcpStream; @@ -60,7 +60,7 @@ fn text_and_binary_replies_are_identical_documents() { let mut binary = server.client(BinaryLinoProtocol::with_options( BinaryLinoOptions::default() .with_external_references(true) - .with_sequences(true), + .with_arity(ArityRange::at_least(1)), )); text.query("() ((child: father mother))").unwrap(); binary.query("() (('two words': 'it''s' \"x\"))").unwrap(); From 3158d3321c8cb38e40d4bc48a08e894f12fe8ad6 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:06:10 +0000 Subject: [PATCH 28/44] feat(csharp): section-based binary links notation matching the Rust golden vectors Port the configurable binary links notation to C#: packets are sections of links sharing one reference width and one ArityRange (doublets by default, any range such as 2..3 or 1..), with uniform or packed per-section widths chosen by the same planner as Rust. The CLI replaces --sequences and --progressive-widths with --arity and --packed-widths. The C# tests now replay docs/protocol/binary-links-notation-vectors.txt, so both ports are proven to write and read identical bytes. --- csharp/.changeset/issue-105-lino-over-tcp.md | 4 +- .../Protocol/ArityRange.cs | 148 +++++ .../Protocol/LinksPacket.cs | 521 ++++++++++++------ .../Protocol/LinoMapping.cs | 393 +++++++------ .../Protocol/LinoProtocols.cs | 5 +- .../Protocol/SectionPlanner.cs | 128 +++++ .../Foundation.Data.Doublets.Cli.Tests.csproj | 3 +- .../Protocol/BinaryLinksNotationTests.cs | 431 +++++++++++++++ .../Protocol/CliTcpIntegrationTests.cs | 19 +- .../Protocol/LinksServerTests.cs | 2 +- .../Protocol/LinoProtocolCodecTests.cs | 192 +------ .../Foundation.Data.Doublets.Cli/Program.cs | 30 +- 12 files changed, 1292 insertions(+), 584 deletions(-) create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/ArityRange.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/SectionPlanner.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs diff --git a/csharp/.changeset/issue-105-lino-over-tcp.md b/csharp/.changeset/issue-105-lino-over-tcp.md index 8955a7e..64f024d 100644 --- a/csharp/.changeset/issue-105-lino-over-tcp.md +++ b/csharp/.changeset/issue-105-lino-over-tcp.md @@ -5,8 +5,8 @@ Added LiNo substitution operations over TCP (issue #105). There are two protocols, and either can replace the other: - `TextLinoProtocol`: dot-stuffed UTF-8 messages. -- `BinaryLinoProtocol`: each packet carries its link count, and reference widths grow from 8 to 16, 32 and 64 bits. Optional decorators add external (Hybrid) references, a variable-length sequence section and progressive widths. +- `BinaryLinoProtocol`: each packet is a list of sections of links that share one reference width (8, 16, 32 or 64 bits) and one arity range. The default is doublets with one uniform width. Options add external (Hybrid) references, any arity range such as `2..3` or `1..` (`ArityRange`), and packed per-section widths; packed output is never larger than uniform output. Shared golden vectors in `docs/protocol/binary-links-notation-vectors.txt` pin the format. -`LinksServer` and `LinksClient` run create, read, update and delete queries over either protocol. The new `clink --serve` and `--connect` options do the same from the command line, together with `--protocol`, `--external-references`, `--sequences` and `--progressive-widths`. The wire format is byte-for-byte the same as the Rust port's, so C# and Rust servers and clients work with each other. +`LinksServer` and `LinksClient` run create, read, update and delete queries over either protocol. The new `clink --serve` and `--connect` options do the same from the command line, together with `--protocol`, `--external-references`, `--arity` and `--packed-widths`. The wire format is byte-for-byte the same as the Rust port's, so C# and Rust servers and clients work with each other. Upgraded Link.Foundation.Links.Notation to 0.22.0. In earlier versions, parse time grew exponentially with nesting depth, so one short message could stall a server. diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/ArityRange.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/ArityRange.cs new file mode 100644 index 0000000..78820e2 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/ArityRange.cs @@ -0,0 +1,148 @@ +using System.Diagnostics.CodeAnalysis; +using System.Globalization; + +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// +/// How many references the links of a hold: +/// Min..=Max, where a null means no upper bound. +/// +/// Fewest references in a link, at least 1. +/// Most references in a link, null for no limit. +public readonly record struct ArityRange(ulong Min, ulong? Max) +{ + /// The largest minimum arity a section shape can hold. + public const ulong MaxMin = ulong.MaxValue >> 4; + + /// Links of exactly two references. + public static ArityRange Doublets { get; } = Exactly(2); + + /// Links of exactly references. + public static ArityRange Exactly(ulong arity) => new(arity, arity); + + /// Links of at least references. + public static ArityRange AtLeast(ulong min) => new(min, null); + + /// Links of min..=max references. + public static ArityRange Between(ulong min, ulong max) => new(min, max); + + /// True when a link of references fits the range. + public bool Contains(ulong length) => length >= Min && (Max is not { } max || length <= max); + + /// True when every link has the same number of references, so links need no length prefix. + public bool IsFixed => Max == Min; + + /// 0 for no maximum, otherwise Max - Min; only for variable arities. + internal ulong Extra => Max is { } max ? max - Min : 0; + + /// Null for a usable range, otherwise why it is not. + internal string? Problem() + { + if (Min == 0) + { + return "arity must be at least 1"; + } + if (Min > MaxMin) + { + return $"arity {Min} is too large"; + } + if (Max is { } max && max < Min) + { + return $"arity range {this} is empty"; + } + return null; + } + + /// The range a section header describes; 0 means no maximum. + internal static ArityRange FromShape(ulong min, ulong? extra) + { + ArityRange range; + if (extra is not { } added) + { + range = Exactly(min); + } + else if (added == 0) + { + range = AtLeast(min); + } + else + { + if (min > ulong.MaxValue - added) + { + throw LinoProtocolException.Malformed("arity range overflows 64 bits"); + } + range = Between(min, min + added); + } + if (range.Problem() is { } problem) + { + throw LinoProtocolException.Malformed(problem); + } + return range; + } + + /// Formats the range as n, min..max or min... + public override string ToString() => Max switch + { + { } max when max == Min => max.ToString(CultureInfo.InvariantCulture), + { } max => $"{Min}..{max}", + null => $"{Min}..", + }; + + /// Parses n, min..max (inclusive) or min.. (no maximum). + /// The text is not a usable arity range. + public static ArityRange Parse(string text) => + TryParse(text, out var range, out var error) ? range : throw new FormatException(error); + + /// Parses n, min..max (inclusive) or min.. (no maximum). + public static bool TryParse([NotNullWhen(true)] string? text, out ArityRange range) => + TryParse(text, out range, out _); + + /// Parses an arity range, explaining in why it failed. + public static bool TryParse([NotNullWhen(true)] string? text, out ArityRange range, [NotNullWhen(false)] out string? error) + { + range = default; + error = $"invalid arity '{text}': expected n, min..max or min.."; + if (text is null) + { + return false; + } + var separator = text.IndexOf("..", StringComparison.Ordinal); + if (separator < 0) + { + if (!TryParseNumber(text, out var arity)) + { + return false; + } + range = Exactly(arity); + } + else + { + var maxText = text[(separator + 2)..]; + if (!TryParseNumber(text[..separator], out var min)) + { + return false; + } + if (maxText.Length == 0) + { + range = AtLeast(min); + } + else if (TryParseNumber(maxText, out var max)) + { + range = Between(min, max); + } + else + { + return false; + } + } + error = range.Problem(); + return error is null; + } + + // Like Rust's u64::from_str: ASCII digits with an optional leading '+'. + private static bool TryParseNumber(string text, out ulong value) + { + var digits = text.StartsWith('+') ? text[1..] : text; + return ulong.TryParse(digits, NumberStyles.None, CultureInfo.InvariantCulture, out value); + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs index 1dfb96a..3b1ac45 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs @@ -22,11 +22,11 @@ public readonly record struct PacketReference(bool IsExternal, ulong Value) /// Safety limits applied while decoding untrusted input. public sealed record DecodeLimits { - /// Maximum N + M. + /// Maximum number of links in a packet. public ulong MaxLinks { get; init; } = 1UL << 22; - /// Maximum total number of references inside all sequences. - public ulong MaxSequenceItems { get; init; } = 1UL << 24; + /// Maximum number of references in all links of a packet. + public ulong MaxReferences { get; init; } = 1UL << 24; /// Maximum number of LiNo nodes a packet may expand to. public long MaxNodes { get; init; } = 1L << 22; @@ -39,28 +39,124 @@ public sealed record DecodeLimits /// The default limits. public static DecodeLimits Default { get; } = new(); + + /// Limits for trusted input such as a store archive: only the address space bounds the packet. + public static DecodeLimits Unlimited { get; } = new() + { + MaxLinks = ulong.MaxValue, + MaxReferences = ulong.MaxValue, + MaxNodes = long.MaxValue, + MaxDepth = int.MaxValue, + MaxTextBytes = long.MaxValue, + }; +} + +/// A run of links at consecutive addresses sharing an arity range and a reference width. +public sealed class Section +{ + /// Addresses skipped before the first link of the section. + public ulong Gap { get; set; } + + /// The number of references each link may hold. + public ArityRange Arity { get; set; } = ArityRange.Doublets; + + /// Bytes per reference: 1, 2, 4 or 8. + public byte Width { get; set; } = 1; + + /// The links, in address order. + public List Links { get; } = new(); + + internal void WriteHeader(List output) + { + var shape = LinksPacket.WidthCode(Width) | (Arity.Min << LinksPacket.ShapeMinArityShift); + if (Gap != 0) + { + shape |= LinksPacket.ShapeHasGap; + } + if (!Arity.IsFixed) + { + shape |= LinksPacket.ShapeVariableArity; + } + LinksPacket.WriteLeb128(output, shape); + if (Gap != 0) + { + LinksPacket.WriteLeb128(output, Gap); + } + if (!Arity.IsFixed) + { + LinksPacket.WriteLeb128(output, Arity.Extra); + } + LinksPacket.WriteLeb128(output, (ulong)Links.Count); + } + + /// Reads a section header, returning the still empty section and its link count. + internal static (Section Section, ulong Count) ReadHeader(LinoStreamReader reader) + { + var shape = LinksPacket.ReadLeb128(reader); + var width = LinksPacket.WidthFromCode(shape & LinksPacket.ShapeWidthBits); + var gap = (shape & LinksPacket.ShapeHasGap) != 0 ? LinksPacket.ReadLeb128(reader) : 0; + ulong? extra = (shape & LinksPacket.ShapeVariableArity) != 0 ? LinksPacket.ReadLeb128(reader) : null; + var arity = ArityRange.FromShape(shape >> LinksPacket.ShapeMinArityShift, extra); + var count = LinksPacket.ReadLeb128(reader); + return (new Section { Gap = gap, Arity = arity, Width = width }, count); + } + + internal void Validate() + { + LinksPacket.WidthCode(Width); + if (Arity.Problem() is { } problem) + { + throw LinoProtocolException.Unencodable(problem); + } + foreach (var link in Links) + { + if (!Arity.Contains((ulong)link.Length)) + { + throw LinoProtocolException.Unencodable( + $"a link of {link.Length} references in a section of arity {Arity}"); + } + } + } } /// -/// The binary links packet: the wire format of the binary LiNo protocol. +/// Binary links notation: a self-delimiting packet of links. /// /// +/// A packet stores links, each a tuple of one or more references, at +/// implicit consecutive addresses. It knows nothing about LiNo; the LiNo +/// protocol () is one use of it. /// -/// byte 0 0x10 | flags high nibble 1 = format version 1 -/// bit 0 external references (Hybrid encoding) -/// bit 1 sequence section present -/// bits 2-3 log2 of the minimum width in bytes -/// LEB128 N number of fixed doublets -/// LEB128 M number of sequences (only when bit 1 is set) -/// N times source target one fixed doublet, refs width(a) bytes each -/// M times size ref_1 … ref_n one sequence, size and refs width(a) bytes each +/// byte 0 0x10 | flags high nibble 1 = format version 1 +/// bit 0 external references (Hybrid encoding) +/// bit 1 explicit layout +/// bits 2-3 log2 of the width (compact layout only) +/// +/// compact layout (bit 1 clear): one section of doublets right after the markers +/// LEB128 N number of links, each `source target` +/// +/// explicit layout (bit 1 set): +/// LEB128 S number of sections, then S section headers: +/// LEB128 shape bits 0-1 log2 of the reference width in bytes +/// bit 2 a gap follows +/// bit 3 variable arity (else every link +/// holds exactly min_arity references) +/// bits 4+ min_arity, at least 1 +/// LEB128 gap addresses skipped before the section (if bit 2) +/// LEB128 extra_arity 0 = no maximum, else max - min (if bit 3) +/// LEB128 count number of links in the section +/// +/// links, section by section; a link in a variable-arity section starts +/// with LEB128 (length - min_arity); every reference is `width` bytes, +/// little-endian /// -/// Addresses are implicit: 0 is null, 1..5 are the reserved marker points -/// (never transmitted), the fixed doublets occupy 6..6+N and the sequences -/// follow them. Every reference of the link at address a uses -/// width(a) = max(min_width, tier(a)) bytes, little-endian, where tier(a) is -/// the smallest of 1, 2, 4 and 8 bytes able to hold a. With external -/// references enabled the top bit marks a reference as external, exactly like +/// Address 0 is null. The first section starts at 1 + gap and every +/// other section at previous end + gap, so gaps leave holes. The +/// compact layout is exactly one section with gap 5 (the LiNo marker points +/// 1..5), arity 2 and the header width. Each section has its own width: the +/// narrowest of 1, 2, 4 and 8 bytes that holds every reference in it; +/// chooses the sections. With external references +/// enabled the top bit of a reference marks it as external, exactly like /// Platform.Data.Hybrid<T>, which halves every internal range. /// The byte layout is identical to the Rust link_cli::protocol::LinksPacket. /// @@ -81,30 +177,35 @@ public sealed class LinksPacket public const ulong List = 4; /// Marker 5: (Identified id values…) is a link with an id. public const ulong Identified = 5; - /// Address of the first transmitted link. + /// Address of the first link after the marker points. public const ulong FirstLinkAddress = 6; private const byte FlagExternalReferences = 0b0001; - private const byte FlagSequences = 0b0010; + private const byte FlagExplicitLayout = 0b0010; private const int WidthShift = 2; + private const byte WidthBits = 0b1100; - /// The reference widths, in bytes, that a packet may use. - public static IReadOnlyList Widths { get; } = new byte[] { 1, 2, 4, 8 }; + internal const ulong ShapeWidthBits = 0b0011; + internal const ulong ShapeHasGap = 0b0100; + internal const ulong ShapeVariableArity = 0b1000; + internal const int ShapeMinArityShift = 4; - /// Header bit 0: references may be external (Hybrid encoding). - public bool ExternalReferences { get; set; } + /// The addresses the compact layout skips: the marker points. + private const ulong CompactGap = FirstLinkAddress - 1; - /// Header bit 1: the packet carries a sequence section (possibly empty). - public bool SequencesSection { get; set; } + private static readonly byte[] WidthValues = { 1, 2, 4, 8 }; - /// Minimum reference width in bytes: 1, 2, 4 or 8. - public byte MinWidth { get; set; } = 1; + /// The reference widths, in bytes, that a packet may use. + public static IReadOnlyList Widths => WidthValues; + + /// An empty packet. + public LinksPacket(bool externalReferences = false) => ExternalReferences = externalReferences; - /// Fixed doublets at addresses 6..6+N. - public List<(PacketReference Source, PacketReference Target)> Doublets { get; } = new(); + /// Header bit 0: references may be external (Hybrid encoding). + public bool ExternalReferences { get; set; } - /// Variable-length sequences at addresses 6+N..6+N+M. - public List> Sequences { get; } = new(); + /// The links, section by section. + public List
Sections { get; } = new(); /// Largest internal address that fits in bytes. public static ulong InternalCapacity(byte width, bool externalReferences) @@ -116,13 +217,10 @@ public static ulong InternalCapacity(byte width, bool externalReferences) /// Largest external value that fits in bytes. public static ulong ExternalCapacity(byte width) => (1UL << (width * 8 - 1)) - 1; - /// Largest unsigned value (a sequence size) that fits in bytes. - public static ulong UnsignedCapacity(byte width) => InternalCapacity(width, false); - /// The narrowest width able to hold the internal address. public static byte AddressTier(ulong address, bool externalReferences) { - foreach (var width in Widths) + foreach (var width in WidthValues) { if (InternalCapacity(width, externalReferences) >= address) { @@ -132,9 +230,11 @@ public static byte AddressTier(ulong address, bool externalReferences) return 8; } + private static ulong WidthMask(byte width) => InternalCapacity(width, false); + /// Encodes an external value at the way Hybrid<T> does. public static ulong EncodeExternal(ulong value, byte width) => - value == 0 ? 1UL << (width * 8 - 1) : (0UL - value) & UnsignedCapacity(width); + value == 0 ? 1UL << (width * 8 - 1) : (0UL - value) & WidthMask(width); /// Decodes a raw value; returns true and the value for externals. public static bool TryDecodeExternal(ulong raw, byte width, out ulong value) @@ -147,78 +247,39 @@ public static bool TryDecodeExternal(ulong raw, byte width, out ulong value) } if (raw > externalZero) { - value = (0UL - raw) & UnsignedCapacity(width); + value = (0UL - raw) & WidthMask(width); return true; } return false; } - /// Address of the fixed doublet with zero-based index. - public ulong DoubletAddress(int index) => FirstLinkAddress + (ulong)index; - - /// Address of the sequence with zero-based index. - public ulong SequenceAddress(int index) => FirstLinkAddress + (ulong)Doublets.Count + (ulong)index; - - /// The highest address in the packet, or null for an empty packet. - public ulong? LastAddress + internal static ulong WidthCode(byte width) { - get - { - var count = Doublets.Count + Sequences.Count; - return count > 0 ? FirstLinkAddress + (ulong)count - 1 : null; - } + var code = Array.IndexOf(WidthValues, width); + return code >= 0 ? (ulong)code : throw LinoProtocolException.Unencodable($"invalid width {width}"); } - /// Width used by every reference of the link at . - public byte WidthAt(ulong address) => Math.Max(MinWidth, AddressTier(address, ExternalReferences)); + internal static byte WidthFromCode(ulong code) => + code < (ulong)WidthValues.Length + ? WidthValues[code] + : throw LinoProtocolException.Malformed($"invalid width code {code}"); - /// - /// The smallest min_width able to encode the packet; with - /// at least the tier of the last address, so every reference has the same width. - /// - public byte RequiredMinWidth(bool uniform) + /// The narrowest width able to hold . + public static byte ReferenceWidth(PacketReference reference, bool externalReferences) { - byte needed = 1; - void Note(ulong address, byte need) - { - if (need > AddressTier(address, ExternalReferences)) - { - needed = Math.Max(needed, need); - } - } - for (var index = 0; index < Doublets.Count; index++) - { - var (source, target) = Doublets[index]; - Note(DoubletAddress(index), Math.Max(ReferenceNeed(source), ReferenceNeed(target))); - } - for (var index = 0; index < Sequences.Count; index++) + if (!reference.IsExternal) { - var items = Sequences[index]; - var need = Widths.FirstOrDefault(width => UnsignedCapacity(width) >= (ulong)items.Count, (byte)8); - foreach (var item in items) + if (reference.Value > InternalCapacity(8, externalReferences)) { - need = Math.Max(need, ReferenceNeed(item)); + throw LinoProtocolException.Unencodable($"address {reference.Value} exceeds the internal range"); } - Note(SequenceAddress(index), need); + return AddressTier(reference.Value, externalReferences); } - if (uniform && LastAddress is { } last) - { - needed = Math.Max(needed, AddressTier(last, ExternalReferences)); - } - return needed; - } - - private byte ReferenceNeed(PacketReference reference) - { - if (!reference.IsExternal) - { - return 1; - } - if (!ExternalReferences) + if (!externalReferences) { throw LinoProtocolException.Unencodable("external reference in a packet without external references"); } - foreach (var width in Widths) + foreach (var width in WidthValues) { if (ExternalCapacity(width) >= reference.Value) { @@ -228,84 +289,158 @@ private byte ReferenceNeed(PacketReference reference) throw LinoProtocolException.Unencodable($"external value {reference.Value} exceeds 63 bits"); } - private byte HeaderByte() + /// + /// Lays out — addresses with their references, in + /// ascending address order — in as few bytes as the format allows. + /// + /// + /// Holes between addresses start new sections. Without + /// every section uses the width of the + /// widest reference, so all references have the same size. With it each + /// section gets the narrowest width its links need and sections split + /// wherever that saves bytes; the result is never larger than the uniform one. + /// + public static LinksPacket Pack( + bool externalReferences, + IReadOnlyList<(ulong Address, PacketReference[] References)> links, + bool packedWidths) { - var code = Widths.ToList().IndexOf(MinWidth); - if (code < 0) + ArgumentNullException.ThrowIfNull(links); + var planner = new SectionPlanner(externalReferences, links); + var uniformLayout = planner.Plan(false); + var uniform = LayOut(externalReferences, links, uniformLayout); + if (!packedWidths) { - throw LinoProtocolException.Unencodable($"invalid width {MinWidth}"); + return uniform; } - var header = (byte)(BinaryVersion1 | (code << WidthShift)); - if (ExternalReferences) + var packedLayout = planner.Plan(true); + if (packedLayout.SequenceEqual(uniformLayout)) { - header |= FlagExternalReferences; + return uniform; } - if (SequencesSection) - { - header |= FlagSequences; - } - return header; + var packed = LayOut(externalReferences, links, packedLayout); + return packed.ToBytes().Length < uniform.ToBytes().Length ? packed : uniform; } - private ulong RawReference(PacketReference reference, ulong address, byte width) + /// Splits into sections of (link count, width). + private static LinksPacket LayOut( + bool externalReferences, + IReadOnlyList<(ulong Address, PacketReference[] References)> links, + IReadOnlyList<(int Count, byte Width)> layout) { - if (!reference.IsExternal) - { - if (reference.Value >= address) + var packet = new LinksPacket(externalReferences); + var nextAddress = 1UL; + var first = 0; + foreach (var (count, width) in layout) + { + var start = links[first].Address; + var shortest = ulong.MaxValue; + var longest = 0UL; + var section = new Section { Gap = start - nextAddress, Width = width }; + for (var index = first; index < first + count; index++) { - throw LinoProtocolException.Unencodable($"link {address} refers forward to {reference.Value}"); + var length = (ulong)links[index].References.Length; + shortest = Math.Min(shortest, length); + longest = Math.Max(longest, length); + section.Links.Add(links[index].References); } - return reference.Value; + section.Arity = ArityRange.Between(shortest, longest); + packet.Sections.Add(section); + nextAddress = start + (ulong)count; + first += count; } - if (!ExternalReferences || reference.Value > ExternalCapacity(width)) + return packet; + } + + /// Every link with its address, in address order. + public IEnumerable<(ulong Address, PacketReference[] References)> Links() + { + var nextAddress = 1UL; + foreach (var section in Sections) { - throw LinoProtocolException.Unencodable( - $"external value {reference.Value} does not fit {width} byte(s) at link {address}"); + var start = SaturatingAdd(nextAddress, section.Gap); + nextAddress = SaturatingAdd(start, (ulong)section.Links.Count); + for (var index = 0; index < section.Links.Count; index++) + { + yield return (start + (ulong)index, section.Links[index]); + } } - return EncodeExternal(reference.Value, width); } + private static ulong SaturatingAdd(ulong left, ulong right) => + left > ulong.MaxValue - right ? ulong.MaxValue : left + right; + + /// The number of links in the packet. + public ulong LinkCount => Sections.Aggregate(0UL, (total, section) => total + (ulong)section.Links.Count); + + private bool IsCompact => Sections.Count switch + { + 0 => true, + 1 => Sections[0].Gap == CompactGap && Sections[0].Arity == ArityRange.Doublets && Sections[0].Links.Count > 0, + _ => false, + }; + /// Serializes the packet. public byte[] ToBytes() { - if (!SequencesSection && Sequences.Count > 0) + var header = BinaryVersion1; + if (ExternalReferences) { - throw LinoProtocolException.Unencodable("sequences in a packet without a sequence section"); + header |= FlagExternalReferences; } - var output = new List { HeaderByte() }; - WriteLeb128(output, (ulong)Doublets.Count); - if (SequencesSection) + var nextAddress = 1UL; + foreach (var section in Sections) { - WriteLeb128(output, (ulong)Sequences.Count); + section.Validate(); + var count = (ulong)section.Links.Count; + if (section.Gap > ulong.MaxValue - nextAddress || count > ulong.MaxValue - nextAddress - section.Gap) + { + throw LinoProtocolException.Unencodable("addresses overflow 64 bits"); + } + nextAddress += section.Gap + count; } - for (var index = 0; index < Doublets.Count; index++) + var output = new List(); + if (IsCompact) { - var address = DoubletAddress(index); - var width = WidthAt(address); - var (source, target) = Doublets[index]; - WriteRaw(output, RawReference(source, address, width), width); - WriteRaw(output, RawReference(target, address, width), width); + var width = Sections.Count > 0 ? Sections[0].Width : (byte)1; + output.Add((byte)(header | (WidthCode(width) << WidthShift))); + WriteLeb128(output, LinkCount); } - for (var index = 0; index < Sequences.Count; index++) + else { - var address = SequenceAddress(index); - var width = WidthAt(address); - var items = Sequences[index]; - var size = (ulong)items.Count; - if (size > UnsignedCapacity(width)) + output.Add((byte)(header | FlagExplicitLayout)); + WriteLeb128(output, (ulong)Sections.Count); + foreach (var section in Sections) { - throw LinoProtocolException.Unencodable( - $"sequence size {size} does not fit {width} byte(s) at link {address}"); + section.WriteHeader(output); } - WriteRaw(output, size, width); - foreach (var item in items) + } + foreach (var section in Sections) + { + foreach (var link in section.Links) { - WriteRaw(output, RawReference(item, address, width), width); + if (!section.Arity.IsFixed) + { + WriteLeb128(output, (ulong)link.Length - section.Arity.Min); + } + foreach (var reference in link) + { + WriteRaw(output, RawReference(reference, section.Width), section.Width); + } } } return output.ToArray(); } + private ulong RawReference(PacketReference reference, byte width) + { + if (ReferenceWidth(reference, ExternalReferences) > width) + { + throw LinoProtocolException.Unencodable($"{reference} does not fit {width} byte(s)"); + } + return reference.IsExternal ? EncodeExternal(reference.Value, width) : reference.Value; + } + /// Parses a complete packet; trailing bytes are an error. public static LinksPacket FromBytes(byte[] bytes, DecodeLimits? limits = null) { @@ -333,62 +468,90 @@ public static LinksPacket FromBytes(byte[] bytes, DecodeLimits? limits = null) { throw LinoProtocolException.Malformed($"unsupported binary header byte 0x{header:X2}"); } - var packet = new LinksPacket - { - ExternalReferences = (header & FlagExternalReferences) != 0, - SequencesSection = (header & FlagSequences) != 0, - MinWidth = Widths[(header >> WidthShift) & 0b11], - }; - var doubletCount = ReadLeb128(reader); - var sequenceCount = packet.SequencesSection ? ReadLeb128(reader) : 0; - if (doubletCount > limits.MaxLinks || sequenceCount > limits.MaxLinks - doubletCount) + var packet = new LinksPacket((header & FlagExternalReferences) != 0); + var counts = new List(); + if ((header & FlagExplicitLayout) == 0) { - throw LinoProtocolException.Limit( - $"packet declares {doubletCount} + {sequenceCount} links, limit is {limits.MaxLinks}"); - } - for (var index = 0; index < (int)doubletCount; index++) - { - var address = packet.DoubletAddress(index); - var width = packet.WidthAt(address); - var source = packet.ReadReference(reader, address, width); - var target = packet.ReadReference(reader, address, width); - packet.Doublets.Add((source, target)); + var width = WidthFromCode((ulong)((header & WidthBits) >> WidthShift)); + var count = ReadLeb128(reader); + if (count > 0) + { + packet.Sections.Add(new Section { Gap = CompactGap, Arity = ArityRange.Doublets, Width = width }); + counts.Add(count); + } } - var itemsLeft = limits.MaxSequenceItems; - for (var index = 0; index < (int)sequenceCount; index++) + else { - var address = packet.SequenceAddress(index); - var width = packet.WidthAt(address); - var size = ReadRaw(reader, width); - if (size > itemsLeft) + if ((header & WidthBits) != 0) + { + throw LinoProtocolException.Malformed("the explicit layout keeps the header width bits clear"); + } + var sectionCount = ReadLeb128(reader); + if (sectionCount > limits.MaxLinks) { - throw LinoProtocolException.Limit($"sequence items exceed the limit of {limits.MaxSequenceItems}"); + throw TooManyLinks(limits); } - itemsLeft -= size; - var items = new List((int)Math.Min(size, 4096)); - for (ulong item = 0; item < size; item++) + var nextAddress = 1UL; + for (ulong index = 0; index < sectionCount; index++) { - items.Add(packet.ReadReference(reader, address, width)); + var (section, count) = Section.ReadHeader(reader); + if (section.Gap > ulong.MaxValue - nextAddress || count > ulong.MaxValue - nextAddress - section.Gap) + { + throw LinoProtocolException.Malformed("addresses overflow 64 bits"); + } + nextAddress += section.Gap + count; + packet.Sections.Add(section); + counts.Add(count); } - packet.Sequences.Add(items); } - return packet; - } - - private PacketReference ReadReference(LinoStreamReader reader, ulong address, byte width) - { - var raw = ReadRaw(reader, width); - if (ExternalReferences && TryDecodeExternal(raw, width, out var value)) + var total = 0UL; + foreach (var count in counts) { - return PacketReference.External(value); + if (count > limits.MaxLinks - total) + { + throw TooManyLinks(limits); + } + total += count; } - if (raw >= address) + var referencesLeft = limits.MaxReferences; + for (var index = 0; index < packet.Sections.Count; index++) { - throw LinoProtocolException.Malformed($"link {address} refers to {raw}, which is not an earlier link"); + var section = packet.Sections[index]; + section.Links.Capacity = (int)Math.Min(counts[index], 4096); + for (ulong link = 0; link < counts[index]; link++) + { + var length = section.Arity.Min; + if (!section.Arity.IsFixed) + { + var extra = ReadLeb128(reader); + if (extra > ulong.MaxValue - length || !section.Arity.Contains(extra + length)) + { + throw LinoProtocolException.Malformed($"link length outside the section arity {section.Arity}"); + } + length += extra; + } + if (length > referencesLeft) + { + throw LinoProtocolException.Limit($"references exceed the limit of {limits.MaxReferences}"); + } + referencesLeft -= length; + var references = new List((int)Math.Min(length, 4096)); + for (ulong item = 0; item < length; item++) + { + var raw = ReadRaw(reader, section.Width); + references.Add(packet.ExternalReferences && TryDecodeExternal(raw, section.Width, out var value) + ? PacketReference.External(value) + : PacketReference.Internal(raw)); + } + section.Links.Add(references.ToArray()); + } } - return PacketReference.Internal(raw); + return packet; } + private static LinoProtocolException TooManyLinks(DecodeLimits limits) => + LinoProtocolException.Limit($"packet declares more than {limits.MaxLinks} links"); + private static void WriteRaw(List output, ulong value, byte width) { for (var index = 0; index < width; index++) diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoMapping.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoMapping.cs index 1fccb72..3ba5f52 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoMapping.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoMapping.cs @@ -9,31 +9,65 @@ namespace Foundation.Data.Doublets.Cli.Protocol; /// Optional features of the binary LiNo protocol. Every feature is off by /// default; each one can be switched on independently, like stacking a decorator. ///
-/// -/// Send numbers and code points as Hybrid external references instead of -/// in-band unary links. Halves the internal address range of each width. -/// -/// -/// Use the variable-length sequence section for lists, strings and links with -/// ids instead of cons chains of doublets. -/// -/// -/// Let the reference width grow with the address instead of using the width of -/// the largest address for every reference. -/// -public readonly record struct BinaryLinoOptions( - bool ExternalReferences = false, - bool Sequences = false, - bool ProgressiveWidths = false) +public readonly record struct BinaryLinoOptions { + // Doublets are stored as null so that default(BinaryLinoOptions) means doublets. + private readonly ArityRange? _arity; + + /// + /// Send numbers and code points as Hybrid external references instead of + /// in-band unary links. Halves the internal address range of each width. + /// + public bool ExternalReferences { get; init; } + + /// + /// The link lengths the encoder may use. The default, exactly 2, sends + /// only doublets; 2..3 adds triplets and 1.. any length. + /// The range must include 2. + /// + public ArityRange Arity + { + get => _arity ?? ArityRange.Doublets; + init => _arity = value == ArityRange.Doublets ? null : value; + } + + /// + /// Give every section of the packet the narrowest width its links need + /// instead of one width for the whole packet. + /// + public bool PackedWidths { get; init; } + /// Enables or disables external references. public BinaryLinoOptions WithExternalReferences(bool enabled = true) => this with { ExternalReferences = enabled }; - /// Enables or disables the sequence section. - public BinaryLinoOptions WithSequences(bool enabled = true) => this with { Sequences = enabled }; + /// Sets the link lengths the encoder may use. + public BinaryLinoOptions WithArity(ArityRange arity) => this with { Arity = arity }; + + /// Enables or disables packed widths. + public BinaryLinoOptions WithPackedWidths(bool enabled = true) => this with { PackedWidths = enabled }; - /// Enables or disables progressive reference widths. - public BinaryLinoOptions WithProgressiveWidths(bool enabled = true) => this with { ProgressiveWidths = enabled }; + /// The options a peer most likely used to write , so a reply can be written in the same style. + public static BinaryLinoOptions OfPacket(LinksPacket packet) + { + ArgumentNullException.ThrowIfNull(packet); + ulong shortest = 2; + ulong longest = 2; + foreach (var (_, link) in packet.Links()) + { + shortest = Math.Min(shortest, (ulong)link.Length); + longest = Math.Max(longest, (ulong)link.Length); + } + return new BinaryLinoOptions + { + ExternalReferences = packet.ExternalReferences, + Arity = ArityRange.Between(shortest, longest), + PackedWidths = packet.Sections.Select(section => section.Width).Distinct().Skip(1).Any(), + }; + } + + /// Lists every option, for test and error messages. + public override string ToString() => + $"BinaryLinoOptions {{ ExternalReferences = {ExternalReferences}, Arity = {Arity}, PackedWidths = {PackedWidths} }}"; } /// @@ -50,10 +84,14 @@ public readonly record struct BinaryLinoOptions( /// A link with an id is (Identified id values…). /// The document is a list of its top-level links, stored last (the root). /// -/// Typed values and lists are a sequence [marker, elements…] when the -/// sequence section is enabled and the doublet (marker chain) otherwise, -/// where chain is the nil-terminated cons list (e1 (e2 (… (en 0)))). -/// Identical sub-links are emitted once and shared. +/// A list or typed value with any number of values other than two is a +/// single link of that many references when +/// allows it ([marker, elements…] for typed values, the bare elements +/// for lists), and otherwise the doublet (marker chain), where chain is +/// the nil-terminated cons list (e1 (e2 (… (en 0)))). Identical +/// sub-links are emitted once and shared. Links are numbered so that every +/// link only refers to earlier ones: doublets of plain doublets first, then +/// the rest in creation order. /// public static class LinoMapping { @@ -61,19 +99,14 @@ public static class LinoMapping public static LinksPacket EncodeDocument(IReadOnlyList document, BinaryLinoOptions options = default) { ArgumentNullException.ThrowIfNull(document); + if (!options.Arity.Contains(2)) + { + throw LinoProtocolException.Unencodable($"arity {options.Arity} does not include doublets (2)"); + } var encoder = new Encoder(options); if (document.Count > 0) { - var items = document.Select(encoder.Encode).ToList(); - if (options.Sequences) - { - encoder.Sequences.Add(items); - } - else - { - var chain = encoder.Chain(items); - encoder.Doublets.Add((Node.Internal(LinksPacket.List), chain)); - } + encoder.List(document.Select(encoder.Encode).ToList()); } return encoder.Finish(); } @@ -83,16 +116,16 @@ public static IReadOnlyList DecodeDocument(LinksPacket packet, DecodeL { ArgumentNullException.ThrowIfNull(packet); limits ??= DecodeLimits.Default; - if (packet.LastAddress is not { } root) + var decoder = new Decoder(packet, limits); + if (decoder.LinkCount == 0) { return Array.Empty(); } - var decoder = new Decoder(packet, limits); - var view = decoder.View(PacketReference.Internal(root)); - IReadOnlyList items = view switch + var root = decoder.View(PacketReference.Internal(LinksPacket.FirstLinkAddress + (ulong)decoder.LinkCount - 1)); + IReadOnlyList items = root switch { - { Kind: ViewKind.Sequence } when !StartsWithMarker(view.Items!) => view.Items!, - { Kind: ViewKind.Doublet, Source: { IsExternal: false, Value: LinksPacket.List } } => decoder.Chain(view.Target), + { Kind: ViewKind.Link, Items: [{ IsExternal: false, Value: LinksPacket.List }, var chain] } => decoder.Chain(chain), + { Kind: ViewKind.Link } when !StartsWithMarker(root.Items!) => root.Items!, _ => throw LinoProtocolException.Malformed("the root link is not a list"), }; var budget = limits.MaxNodes; @@ -109,17 +142,19 @@ public static bool TryParseCanonicalNumber(string text, out ulong value) return canonical && ulong.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out value); } - private static bool IsMarker(ulong address) => address is >= LinksPacket.One and < LinksPacket.FirstLinkAddress; + private static bool IsMarker(PacketReference reference) => + !reference.IsExternal && reference.Value is >= LinksPacket.One and < LinksPacket.FirstLinkAddress; - private static bool StartsWithMarker(IReadOnlyList items) => - items.Count > 0 && !items[0].IsExternal && IsMarker(items[0].Value); + private static bool StartsWithMarker(IReadOnlyList items) => items.Count > 0 && IsMarker(items[0]); private enum NodeKind { Internal, External, + /// A doublet of plain doublets, markers and externals. Doublet, - Sequence, + /// Any other link; numbered after every doublet. + Tuple, } private readonly record struct Node(NodeKind Kind, ulong Value) @@ -150,73 +185,74 @@ public int GetHashCode(List obj) private sealed class Encoder { private readonly BinaryLinoOptions _options; - private readonly Dictionary<(Node, Node), int> _doubletIndex = new(); - private readonly Dictionary, int> _sequenceIndex = new(NodeListComparer.Instance); + private readonly List> _doublets = new(); + private readonly List> _tuples = new(); + private readonly Dictionary, Node> _created = new(NodeListComparer.Instance); private readonly List _powers = new() { Node.Internal(LinksPacket.One) }; public Encoder(BinaryLinoOptions options) => _options = options; - public List<(Node Source, Node Target)> Doublets { get; } = new(); - - public List> Sequences { get; } = new(); - - private Node Doublet(Node source, Node target) + private Node Link(List items) { - if (_doubletIndex.TryGetValue((source, target), out var existing)) + if (_created.TryGetValue(items, out var existing)) { - return new Node(NodeKind.Doublet, (ulong)existing); + return existing; } - var index = Doublets.Count; - Doublets.Add((source, target)); - _doubletIndex[(source, target)] = index; - return new Node(NodeKind.Doublet, (ulong)index); - } - - private Node Sequence(List items) - { - if (_sequenceIndex.TryGetValue(items, out var existing)) + Node node; + if (items.Count == 2 && items.All(item => item.Kind != NodeKind.Tuple)) + { + _doublets.Add(items); + node = new Node(NodeKind.Doublet, (ulong)_doublets.Count - 1); + } + else { - return new Node(NodeKind.Sequence, (ulong)existing); + _tuples.Add(items); + node = new Node(NodeKind.Tuple, (ulong)_tuples.Count - 1); } - var index = Sequences.Count; - Sequences.Add(items); - _sequenceIndex[items] = index; - return new Node(NodeKind.Sequence, (ulong)index); + _created[items] = node; + return node; } - // Fixed doublets may only refer to fixed doublets, so a pair holding - // a sequence becomes a two-element sequence. - private Node Pair(Node first, Node second) => - first.Kind == NodeKind.Sequence || second.Kind == NodeKind.Sequence - ? Sequence(new List { first, second }) - : Doublet(first, second); + private Node Pair(Node first, Node second) => Link(new List { first, second }); - public Node Chain(IReadOnlyList items) + private Node Chain(IReadOnlyList items) { var tail = Node.Internal(LinksPacket.NullAddress); for (var index = items.Count - 1; index >= 0; index--) { - tail = Doublet(items[index], tail); + tail = Pair(items[index], tail); } return tail; } + // One link of `items` when the arity allows it, except that two items + // are always a doublet. + private bool FitsOneLink(int items) => items != 2 && _options.Arity.Contains((ulong)items); + private Node Typed(ulong marker, List elements) { - if (_options.Sequences) + if (FitsOneLink(elements.Count + 1)) { elements.Insert(0, Node.Internal(marker)); - return Sequence(elements); + return Link(elements); } - return Doublet(Node.Internal(marker), Chain(elements)); + return Pair(Node.Internal(marker), Chain(elements)); } + public Node List(List elements) => elements.Count switch + { + 0 => Node.Internal(LinksPacket.NullAddress), + 2 => Pair(elements[0], elements[1]), + _ when FitsOneLink(elements.Count) => Link(elements), + _ => Typed(LinksPacket.List, elements), + }; + private Node Power(int exponent) { while (_powers.Count <= exponent) { var previous = _powers[^1]; - _powers.Add(Doublet(previous, previous)); + _powers.Add(Pair(previous, previous)); } return _powers[exponent]; } @@ -238,24 +274,20 @@ private Node Unary(ulong value) var sum = powers[^1]; for (var index = powers.Count - 2; index >= 0; index--) { - sum = Doublet(powers[index], sum); + sum = Pair(powers[index], sum); } return sum; } - private Node Scalar(ulong value) => - _options.ExternalReferences && value <= LinksPacket.ExternalCapacity(8) ? Node.External(value) : Unary(value); + private bool IsExternal(ulong value) => _options.ExternalReferences && value <= LinksPacket.ExternalCapacity(8); + + private Node Scalar(ulong value) => IsExternal(value) ? Node.External(value) : Unary(value); private Node Reference(string text) { if (TryParseCanonicalNumber(text, out var value)) { - if (_options.ExternalReferences && value <= LinksPacket.ExternalCapacity(8)) - { - return Node.External(value); - } - var unary = Unary(value); - return Doublet(Node.Internal(LinksPacket.Number), unary); + return IsExternal(value) ? Node.External(value) : Pair(Node.Internal(LinksPacket.Number), Unary(value)); } var codePoints = text.EnumerateRunes().Select(rune => Scalar((ulong)rune.Value)).ToList(); return Typed(LinksPacket.String, codePoints); @@ -273,23 +305,12 @@ public Node Encode(LinoLink link) elements.AddRange(values.Select(Encode)); return Typed(LinksPacket.Identified, elements); } - switch (values.Count) - { - case 0: - return Node.Internal(LinksPacket.NullAddress); - case 2: - var first = Encode(values[0]); - var second = Encode(values[1]); - return Pair(first, second); - default: - var items = values.Select(Encode).ToList(); - return _options.Sequences ? Sequence(items) : Typed(LinksPacket.List, items); - } + return List(values.Select(Encode).ToList()); } public LinksPacket Finish() { - var doubletCount = (ulong)Doublets.Count; + var doubletCount = (ulong)_doublets.Count; PacketReference Resolve(Node node) => node.Kind switch { NodeKind.Internal => PacketReference.Internal(node.Value), @@ -297,15 +318,11 @@ public LinksPacket Finish() NodeKind.Doublet => PacketReference.Internal(LinksPacket.FirstLinkAddress + node.Value), _ => PacketReference.Internal(LinksPacket.FirstLinkAddress + doubletCount + node.Value), }; - var packet = new LinksPacket - { - ExternalReferences = _options.ExternalReferences, - SequencesSection = _options.Sequences, - }; - packet.Doublets.AddRange(Doublets.Select(doublet => (Resolve(doublet.Source), Resolve(doublet.Target)))); - packet.Sequences.AddRange(Sequences.Select(items => items.Select(Resolve).ToList())); - packet.MinWidth = packet.RequiredMinWidth(!_options.ProgressiveWidths); - return packet; + var links = _doublets + .Concat(_tuples) + .Select((items, index) => (LinksPacket.FirstLinkAddress + (ulong)index, items.Select(Resolve).ToArray())) + .ToList(); + return LinksPacket.Pack(_options.ExternalReferences, links, _options.PackedWidths); } } @@ -314,119 +331,97 @@ private enum ViewKind Null, Marker, External, - Doublet, - Sequence, + Link, } - private readonly record struct View( - ViewKind Kind, - ulong Value = 0, - PacketReference Source = default, - PacketReference Target = default, - IReadOnlyList? Items = null); + private readonly record struct View(ViewKind Kind, ulong Value = 0, PacketReference[]? Items = null); private sealed class Decoder { - private readonly LinksPacket _packet; private readonly DecodeLimits _limits; - // _unary[i] is the number doublet i denotes, if it is a unary number. - private readonly ulong?[] _unary; + // _links[i] is the link at address FirstLinkAddress + i. + private readonly List _links = new(); + // _unary[i] is the number link i denotes, if it is a unary number. + private readonly List _unary = new(); public Decoder(LinksPacket packet, DecodeLimits limits) { - _packet = packet; _limits = limits; - // Links only refer backwards, so one forward pass evaluates every - // unary number without recursion. - _unary = new ulong?[packet.Doublets.Count]; - for (var index = 0; index < packet.Doublets.Count; index++) + foreach (var (address, link) in packet.Links()) { - var (source, target) = packet.Doublets[index]; - if (UnaryValue(source) is { } sourceValue - && UnaryValue(target) is { } targetValue - && sourceValue <= ulong.MaxValue - targetValue) + var expected = LinksPacket.FirstLinkAddress + (ulong)_links.Count; + if (address != expected) + { + throw LinoProtocolException.Malformed( + $"a LiNo packet stores its links contiguously from address {LinksPacket.FirstLinkAddress}, " + + $"found link {address} where {expected} belongs"); + } + foreach (var reference in link) + { + if (!reference.IsExternal && reference.Value >= address) + { + throw LinoProtocolException.Malformed( + $"link {address} refers to {reference}, which is not an earlier link"); + } + } + // Links only refer backwards, so one forward pass evaluates every + // unary number without recursion. + ulong? value = null; + if (link.Length == 2 + && UnaryValue(link[0]) is { } source + && UnaryValue(link[1]) is { } target + && source <= ulong.MaxValue - target) { - _unary[index] = sourceValue + targetValue; + value = source + target; } + _unary.Add(value); + _links.Add(link); } } - private ulong? UnaryValue(PacketReference reference) - { - if (reference.IsExternal) - { - return null; - } - return reference.Value switch - { - LinksPacket.NullAddress => 0, - LinksPacket.One => 1, - >= LinksPacket.FirstLinkAddress when reference.Value - LinksPacket.FirstLinkAddress < (ulong)_unary.Length - => _unary[reference.Value - LinksPacket.FirstLinkAddress], - _ => null, - }; - } + public int LinkCount => _links.Count; - public View View(PacketReference reference) + private ulong? UnaryValue(PacketReference reference) => reference switch { - if (reference.IsExternal) - { - return new View(ViewKind.External, reference.Value); - } - if (reference.Value == LinksPacket.NullAddress) - { - return new View(ViewKind.Null); - } - if (reference.Value < LinksPacket.FirstLinkAddress) - { - return new View(ViewKind.Marker, reference.Value); - } - var address = reference.Value - LinksPacket.FirstLinkAddress; - var doublets = (ulong)_packet.Doublets.Count; - if (address < doublets) - { - var (source, target) = _packet.Doublets[(int)address]; - return new View(ViewKind.Doublet, Source: source, Target: target); - } - if (address - doublets < (ulong)_packet.Sequences.Count) - { - return new View(ViewKind.Sequence, Items: _packet.Sequences[(int)(address - doublets)]); - } - throw LinoProtocolException.Malformed($"dangling reference {address}"); - } + { IsExternal: true } => null, + { Value: LinksPacket.NullAddress } => 0, + { Value: LinksPacket.One } => 1, + { Value: >= LinksPacket.FirstLinkAddress } => _unary[(int)(reference.Value - LinksPacket.FirstLinkAddress)], + _ => null, + }; - private ulong Number(PacketReference reference) + public View View(PacketReference reference) => reference switch { - if (reference.IsExternal) - { - return reference.Value; - } - return UnaryValue(reference) ?? throw LinoProtocolException.Malformed("expected a unary number"); - } + { IsExternal: true } => new View(ViewKind.External, reference.Value), + { Value: LinksPacket.NullAddress } => new View(ViewKind.Null), + { Value: < LinksPacket.FirstLinkAddress } => new View(ViewKind.Marker, reference.Value), + _ => new View(ViewKind.Link, Items: _links[(int)(reference.Value - LinksPacket.FirstLinkAddress)]), + }; - // The elements of a typed value given in doublet form: a cons chain, - // optionally ending in a sequence holding the remaining elements. + private ulong Number(PacketReference reference) => + reference.IsExternal + ? reference.Value + : UnaryValue(reference) ?? throw LinoProtocolException.Malformed("expected a unary number"); + + /// The elements of the cons list (e1 (e2 (… (en 0)))). public List Chain(PacketReference tail) { var elements = new List(); while (true) { var view = View(tail); - switch (view.Kind) + switch (view) { - case ViewKind.Null: + case { Kind: ViewKind.Null }: return elements; - case ViewKind.Doublet: + case { Kind: ViewKind.Link, Items: [var head, var next] }: if (elements.Count >= _limits.MaxNodes) { throw LinoProtocolException.Limit("chain too long"); } - elements.Add(view.Source); - tail = view.Target; + elements.Add(head); + tail = next; break; - case ViewKind.Sequence: - elements.AddRange(view.Items!); - return elements; default: throw LinoProtocolException.Malformed("broken element chain"); } @@ -496,24 +491,22 @@ public LinoLink Decode(PacketReference reference, int depth, ref long budget) } budget--; var view = View(reference); - switch (view.Kind) + switch (view) { - case ViewKind.Null: + case { Kind: ViewKind.Null }: return LinoFormat.Link(null, new List()); - case ViewKind.External: + case { Kind: ViewKind.External }: return LinoFormat.Reference(view.Value.ToString(CultureInfo.InvariantCulture)); - case ViewKind.Marker: + case { Kind: ViewKind.Marker }: throw LinoProtocolException.Malformed($"marker {view.Value} used as a value"); - case ViewKind.Doublet when !view.Source.IsExternal && view.Source.Value == LinksPacket.Number: - return Typed(LinksPacket.Number, new[] { view.Target }, depth, ref budget); - case ViewKind.Doublet when !view.Source.IsExternal && IsMarker(view.Source.Value): - return Typed(view.Source.Value, Chain(view.Target), depth, ref budget); - case ViewKind.Doublet: - return List(new[] { view.Source, view.Target }, 0, depth, ref budget); + case { Items: [{ IsExternal: false, Value: LinksPacket.Number }, var value] }: + return Typed(LinksPacket.Number, new[] { value }, depth, ref budget); + case { Items: [var marker, var chain] } when IsMarker(marker): + return Typed(marker.Value, Chain(chain), depth, ref budget); default: var items = view.Items!; return StartsWithMarker(items) - ? Typed(items[0].Value, items.Skip(1).ToList(), depth, ref budget) + ? Typed(items[0].Value, items[1..], depth, ref budget) : List(items, 0, depth, ref budget); } } diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs index ec7e29e..27869a0 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs @@ -206,10 +206,7 @@ public static (IReadOnlyList Document, MessageFormat Format)? ReadAnyD { return null; } - var progressive = packet.LastAddress is { } last - && packet.MinWidth < LinksPacket.AddressTier(last, packet.ExternalReferences); - var options = new BinaryLinoOptions(packet.ExternalReferences, packet.SequencesSection, progressive); - return (LinoMapping.DecodeDocument(packet, limits), MessageFormat.Binary(options)); + return (LinoMapping.DecodeDocument(packet, limits), MessageFormat.Binary(BinaryLinoOptions.OfPacket(packet))); } } diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/SectionPlanner.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/SectionPlanner.cs new file mode 100644 index 0000000..850c5de --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/SectionPlanner.cs @@ -0,0 +1,128 @@ +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// Splits links into s with a linear dynamic program. +/// +/// After link k, costs[s] is the fewest estimated bytes for links +/// 0..=k with link k in a section of state s. A link either continues the +/// section of the previous link (same state, no hole between them and, for a +/// fixed arity, the same length) or opens a new section after the cheapest +/// previous state, paying for a section header. The tie-breaking matches the +/// Rust SectionPlanner, so both ports produce identical bytes. +/// +internal sealed class SectionPlanner +{ + /// Estimated bytes of a section header (shape and count), used to weigh a split. + private const ulong SectionHeaderEstimate = 2; + /// Estimated extra bytes of a variable-arity section: its extra_arity field and each length prefix. + private const ulong VariableArityEstimate = 1; + private const ulong Unreachable = ulong.MaxValue / 4; + /// Planner states: a width code (0..4) times fixed (0) or variable (1) arity. + private static readonly int States = LinksPacket.Widths.Count * 2; + + private readonly IReadOnlyList<(ulong Address, PacketReference[] References)> _links; + private readonly byte[] _needs; + + public SectionPlanner(bool externalReferences, IReadOnlyList<(ulong Address, PacketReference[] References)> links) + { + _links = links; + _needs = new byte[links.Count]; + var previousAddress = 0UL; + for (var index = 0; index < links.Count; index++) + { + var (address, references) = links[index]; + if (address <= previousAddress) + { + throw LinoProtocolException.Unencodable( + $"link addresses must ascend from 1, got {address} after {previousAddress}"); + } + if (references.Length == 0) + { + throw LinoProtocolException.Unencodable($"link {address} has no references"); + } + previousAddress = address; + byte need = 1; + foreach (var reference in references) + { + need = Math.Max(need, LinksPacket.ReferenceWidth(reference, externalReferences)); + } + _needs[index] = need; + } + } + + private static int FirstCheapest(ulong[] costs) + { + var best = 0; + for (var state = 1; state < costs.Length; state++) + { + if (costs[state] < costs[best]) + { + best = state; + } + } + return best; + } + + /// + /// The sections as (link count, width) pairs; with + /// every width may be used, otherwise only + /// the widest one needed. + /// + public List<(int Count, byte Width)> Plan(bool packedWidths) + { + var sections = new List<(int Count, byte Width)>(); + if (_links.Count == 0) + { + return sections; + } + var widest = _needs.Max(); + var opensSection = new byte[_links.Count]; + var previousBest = new byte[_links.Count]; + var costs = Enumerable.Repeat(Unreachable, States).ToArray(); + for (var index = 0; index < _links.Count; index++) + { + var (address, link) = _links[index]; + var best = FirstCheapest(costs); + previousBest[index] = (byte)best; + var cheapestBefore = index == 0 ? 0 : costs[best]; + var continues = index > 0 && _links[index - 1].Address + 1 == address; + var sameLength = index > 0 && _links[index - 1].References.Length == link.Length; + var next = Enumerable.Repeat(Unreachable, States).ToArray(); + for (var state = 0; state < States; state++) + { + var width = LinksPacket.Widths[state / 2]; + var variable = state % 2 == 1; + if (!(packedWidths || width == widest) || width < _needs[index]) + { + continue; + } + var variableEstimate = variable ? VariableArityEstimate : 0; + var body = (ulong)link.Length * width + variableEstimate; + var openingCost = cheapestBefore + SectionHeaderEstimate + variableEstimate; + var continuingCost = continues && (variable || sameLength) ? costs[state] : Unreachable; + if (continuingCost <= openingCost) + { + next[state] = continuingCost + body; + } + else + { + next[state] = openingCost + body; + opensSection[index] |= (byte)(1 << state); + } + } + costs = next; + } + var current = FirstCheapest(costs); + var end = _links.Count; + for (var index = _links.Count - 1; index >= 0; index--) + { + if ((opensSection[index] & (1 << current)) != 0) + { + sections.Add((end - index, LinksPacket.Widths[current / 2])); + end = index; + current = previousBest[index]; + } + } + sections.Reverse(); + return sections; + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj b/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj index 28f0cf9..db2cdad 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj @@ -21,7 +21,8 @@ - + + diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs new file mode 100644 index 0000000..f85fb55 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs @@ -0,0 +1,431 @@ +using Foundation.Data.Doublets.Cli.Protocol; +using Platform.Data; + +using LinoLink = Link.Foundation.Links.Notation.Link; + +namespace Foundation.Data.Doublets.Cli.Tests.Protocol; + +/// +/// The binary links notation: packets, sections, arities, widths and the +/// LiNo mapping on top of them. Mirrors rust/tests/protocol_packet_tests.rs. +/// +public sealed class BinaryLinksNotationTests +{ + internal static string Hex(byte[] bytes) => string.Join(" ", bytes.Select(value => value.ToString("x2"))); + + internal static byte[] Unhex(string text) => + text.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(value => Convert.ToByte(value, 16)).ToArray(); + + private static IReadOnlyList Parse(string text) => LinoFormat.ParseDocument(text); + + private static bool ReferencesOption(string field) => field switch + { + "plain" => false, + "external" => true, + _ => throw new ArgumentException($"unknown references option {field}"), + }; + + private static bool WidthsOption(string field) => field switch + { + "uniform" => false, + "packed" => true, + _ => throw new ArgumentException($"unknown widths option {field}"), + }; + + /// address:reference reference… separated by ;, #v being external. + private static List<(ulong Address, PacketReference[] References)> ParseLinks(string text) => + text.Split(';').Select(link => + { + var parts = link.Split(':', 2); + var references = parts[1].Split(' ').Select(reference => reference.StartsWith('#') + ? PacketReference.External(ulong.Parse(reference[1..])) + : PacketReference.Internal(ulong.Parse(reference))).ToArray(); + return (ulong.Parse(parts[0]), references); + }).ToList(); + + /// The golden vectors shared with the Rust test suite: both ports must write and read exactly these bytes. + private static IEnumerable GoldenVectors() => + File.ReadAllLines(Path.Combine(AppContext.BaseDirectory, "Protocol", "binary-links-notation-vectors.txt")) + .Where(line => line.Length > 0 && !line.StartsWith('#')) + .Select(line => line.Split('\t')); + + private static string Describe(IEnumerable<(ulong Address, PacketReference[] References)> links) => + string.Join(";", links.Select(link => $"{link.Address}:{string.Join(" ", link.References)}")); + + private static string Describe(LinksPacket packet) => + $"{packet.ExternalReferences} " + string.Join(" | ", packet.Sections.Select(section => + $"gap {section.Gap} arity {section.Arity} width {section.Width}: " + + string.Join(";", section.Links.Select(link => string.Join(" ", link))))); + + [Fact] + public void GoldenDocumentsAreStable() + { + var checkedVectors = 0; + foreach (var fields in GoldenVectors().Where(fields => fields[0] == "document")) + { + Assert.Equal(6, fields.Length); + var text = fields[1].Replace("\\n", "\n"); + var options = new BinaryLinoOptions + { + ExternalReferences = ReferencesOption(fields[2]), + Arity = ArityRange.Parse(fields[3]), + PackedWidths = WidthsOption(fields[4]), + }; + var document = Parse(text); + var binary = new BinaryLinoProtocol(options); + Assert.True(fields[5] == Hex(binary.Encode(document)), $"{text} with {options}"); + Assert.Equal(document, binary.Decode(Unhex(fields[5]))); + checkedVectors++; + } + Assert.True(checkedVectors >= 80, $"only {checkedVectors} document vectors"); + } + + [Fact] + public void GoldenPacketsAreStable() + { + var checkedVectors = 0; + foreach (var fields in GoldenVectors().Where(fields => fields[0] == "links")) + { + Assert.Equal(5, fields.Length); + var links = ParseLinks(fields[1]); + var packet = LinksPacket.Pack(ReferencesOption(fields[2]), links, WidthsOption(fields[3])); + Assert.True(fields[4] == Hex(packet.ToBytes()), fields[1]); + var read = LinksPacket.FromBytes(Unhex(fields[4])); + Assert.Equal(Describe(packet), Describe(read)); + Assert.Equal(Describe(links), Describe(read.Links())); + checkedVectors++; + } + Assert.True(checkedVectors >= 10, $"only {checkedVectors} links vectors"); + } + + [Fact] + public void WidthTiersFollowTheNumberOfLinks() + { + Assert.Equal(1, LinksPacket.AddressTier(0, false)); + Assert.Equal(1, LinksPacket.AddressTier(255, false)); + Assert.Equal(2, LinksPacket.AddressTier(256, false)); + Assert.Equal(2, LinksPacket.AddressTier(65_535, false)); + Assert.Equal(4, LinksPacket.AddressTier(65_536, false)); + Assert.Equal(4, LinksPacket.AddressTier(uint.MaxValue, false)); + Assert.Equal(8, LinksPacket.AddressTier((ulong)uint.MaxValue + 1, false)); + // External references take the top bit, halving every range. + Assert.Equal(1, LinksPacket.AddressTier(127, true)); + Assert.Equal(2, LinksPacket.AddressTier(128, true)); + Assert.Equal(2, LinksPacket.AddressTier(32_767, true)); + Assert.Equal(4, LinksPacket.AddressTier(32_768, true)); + Assert.Equal(ulong.MaxValue, LinksPacket.InternalCapacity(8, false)); + Assert.Equal((ulong)long.MaxValue, LinksPacket.InternalCapacity(8, true)); + Assert.Equal(127UL, LinksPacket.ExternalCapacity(1)); + } + + [Fact] + public void ExternalReferencesMatchPlatformDataHybrid() + { + foreach (var width in LinksPacket.Widths) + { + foreach (var value in new ulong[] { 0, 1, 2, 100, LinksPacket.ExternalCapacity(width) }) + { + var raw = LinksPacket.EncodeExternal(value, width); + Assert.True(LinksPacket.TryDecodeExternal(raw, width, out var decoded)); + Assert.Equal(value, decoded); + } + Assert.False(LinksPacket.TryDecodeExternal(LinksPacket.InternalCapacity(width, true), width, out _)); + } + Assert.Equal(0xFFUL, LinksPacket.EncodeExternal(1, 1)); + Assert.Equal(0x80UL, LinksPacket.EncodeExternal(0, 1)); + foreach (var value in new uint[] { 0, 1, 5, 1000, int.MaxValue }) + { + Assert.Equal((ulong)new Hybrid(value, isExternal: true).Value, LinksPacket.EncodeExternal(value, 4)); + } + } + + private static string ManyLinks(int count) => + string.Join("\n", Enumerable.Range(0, count).Select(index => $"(n{index} m{index})")); + + private static byte[] SectionWidths(LinksPacket packet) => packet.Sections.Select(section => section.Width).ToArray(); + + [Fact] + public void UniformWidthsGrowPast256AddressesAndPackedWidthsStaySmall() + { + Assert.Equal(new byte[] { 1 }, SectionWidths(LinoMapping.EncodeDocument(Parse(ManyLinks(3))))); + + var document = Parse(ManyLinks(400)); + var uniform = LinoMapping.EncodeDocument(document); + Assert.Equal(new byte[] { 2 }, SectionWidths(uniform)); + var uniformBytes = uniform.ToBytes(); + const int header = 1 + 2; // header byte + two-byte LEB128 count + Assert.Equal(header + (long)uniform.LinkCount * 2 * 2, uniformBytes.Length); + + var packed = LinoMapping.EncodeDocument(document, new BinaryLinoOptions().WithPackedWidths()); + Assert.Equal(1, SectionWidths(packed)[0]); + Assert.Contains((byte)2, SectionWidths(packed)); + var packedBytes = packed.ToBytes(); + Assert.True(packedBytes.Length < uniformBytes.Length); + foreach (var bytes in new[] { uniformBytes, packedBytes }) + { + Assert.Equal(document, LinoMapping.DecodeDocument(LinksPacket.FromBytes(bytes))); + } + } + + [Fact] + public void PackedWidthsNeverTakeMoreBytesThanUniformOnes() + { + var corpus = LinoProtocolCodecTests.Corpus.Append(ManyLinks(200)).Append("(1000 1)"); + foreach (var text in corpus) + { + var document = Parse(text); + foreach (var options in LinoProtocolCodecTests.AllOptions().Where(options => options.PackedWidths)) + { + var packed = LinoMapping.EncodeDocument(document, options).ToBytes(); + var uniform = LinoMapping.EncodeDocument(document, options.WithPackedWidths(false)).ToBytes(); + Assert.True(packed.Length <= uniform.Length, $"{text} with {options}"); + } + } + } + + [Fact] + public void LargeExternalValuesWidenOnlyTheirSection() + { + var options = new BinaryLinoOptions().WithExternalReferences().WithPackedWidths(); + Assert.Equal(new byte[] { 1 }, SectionWidths(LinoMapping.EncodeDocument(Parse("(100 1)"), options))); + Assert.Equal(new byte[] { 4, 1 }, SectionWidths(LinoMapping.EncodeDocument(Parse("(70000 1)"), options))); + // Beyond 63 bits the number falls back to in-band unary links. + Assert.True(LinoMapping.EncodeDocument(Parse("18446744073709551615"), options).LinkCount > 60); + } + + private static int[] LinkLengths(IReadOnlyList document, BinaryLinoOptions options) + { + var packet = LinoMapping.EncodeDocument(document, options); + Assert.Equal(document, new BinaryLinoProtocol().Decode(packet.ToBytes())); + return packet.Links().Select(link => link.References.Length).ToArray(); + } + + [Fact] + public void ArityOptionsChooseDoubletsTripletsOrAnyLength() + { + var document = Parse("(a b c)\n(d e)"); + var external = new BinaryLinoOptions().WithExternalReferences(); + Assert.All(LinkLengths(document, external), length => Assert.Equal(2, length)); + // `(a b c)` is one triplet; the strings `(String code point)` stay doublets. + var triplets = LinkLengths(document, external.WithArity(ArityRange.Between(2, 3))); + Assert.Single(triplets, length => length == 3); + Assert.All(triplets, length => Assert.InRange(length, 2, 3)); + Assert.Equal(triplets, LinkLengths(document, external.WithArity(ArityRange.AtLeast(1)))); + // Only any length turns a one-item list, here the root, into one link. + Assert.Equal(new[] { 2, 2, 2, 2 }, LinkLengths(Parse("a"), external)); + Assert.Equal(new[] { 2, 2, 1 }, LinkLengths(Parse("a"), external.WithArity(ArityRange.AtLeast(1)))); + // Long lists only become single links when the range allows their length. + var longList = Parse("(a b c d e f g h)"); + foreach (var (arity, singleLink) in new[] + { + (ArityRange.Between(2, 3), false), + (ArityRange.Between(2, 8), true), + (ArityRange.AtLeast(2), true), + }) + { + Assert.True(LinkLengths(longList, external.WithArity(arity)).Max() == 8 == singleLink, arity.ToString()); + } + } + + [Fact] + public void AritiesWithoutDoubletsAreUnencodable() + { + foreach (var arity in new[] { ArityRange.Exactly(3), ArityRange.Exactly(1), ArityRange.AtLeast(3) }) + { + var error = Assert.Throws( + () => LinoMapping.EncodeDocument(Parse("(a b)"), new BinaryLinoOptions().WithArity(arity))); + Assert.Equal(LinoProtocolErrorKind.Unencodable, error.Kind); + } + } + + [Fact] + public void ArityRangesParseAndDisplay() + { + foreach (var (text, range) in new[] + { + ("2", ArityRange.Doublets), + ("2..3", ArityRange.Between(2, 3)), + ("1..", ArityRange.AtLeast(1)), + ("3..3", ArityRange.Exactly(3)), + ("+2", ArityRange.Doublets), + }) + { + Assert.Equal(range, ArityRange.Parse(text)); + } + Assert.Equal("3", ArityRange.Between(3, 3).ToString()); + Assert.Equal("2..3", ArityRange.Between(2, 3).ToString()); + Assert.Equal("1..", ArityRange.AtLeast(1).ToString()); + Assert.Equal(ArityRange.Doublets, new BinaryLinoOptions().Arity); + Assert.Equal(default, new BinaryLinoOptions().WithArity(ArityRange.Doublets)); + foreach (var invalid in new[] { "", "0", "0..2", "3..2", "a", "1..b", "..3", "-1", " 2", "2 ", "++2", null }) + { + Assert.False(ArityRange.TryParse(invalid, out _), invalid); + } + Assert.False(ArityRange.TryParse("3..2", out _, out var error)); + Assert.Equal("arity range 3..2 is empty", error); + Assert.False(ArityRange.TryParse("x", out _, out error)); + Assert.Equal("invalid arity 'x': expected n, min..max or min..", error); + Assert.Equal("arity must be at least 1", Assert.Throws(() => ArityRange.Parse("0")).Message); + Assert.True(ArityRange.AtLeast(2).Contains(1_000)); + Assert.False(ArityRange.Between(2, 3).Contains(4)); + Assert.False(ArityRange.Between(2, 3).Contains(1)); + Assert.True(ArityRange.Exactly(2).IsFixed && !ArityRange.AtLeast(2).IsFixed); + } + + [Fact] + public void ReplyOptionsFollowTheReceivedPacket() + { + var document = Parse("(a b c) 70000"); + var options = new BinaryLinoOptions() + .WithExternalReferences() + .WithArity(ArityRange.Between(2, 3)) + .WithPackedWidths(); + Assert.Equal(options, BinaryLinoOptions.OfPacket(LinoMapping.EncodeDocument(document, options))); + Assert.Equal(default, BinaryLinoOptions.OfPacket(LinoMapping.EncodeDocument(document))); + Assert.Equal(default, BinaryLinoOptions.OfPacket(new LinksPacket())); + Assert.Equal("BinaryLinoOptions { ExternalReferences = True, Arity = 2..3, PackedWidths = True }", options.ToString()); + } + + private static Section DoubletSection(ulong gap, ArityRange arity, byte width, params (ulong Source, ulong Target)[] links) + { + var section = new Section { Gap = gap, Arity = arity, Width = width }; + section.Links.AddRange(links.Select(link => + new[] { PacketReference.Internal(link.Source), PacketReference.Internal(link.Target) })); + return section; + } + + [Fact] + public void HandBuiltPacketsDecode() + { + // One doublet (One One) is 2^1 in unary, so `(Number 7)` is the number 2. + var links = new (ulong, ulong)[] { (1, 1), (2, 6), (7, 0), (4, 8) }; + var compact = new LinksPacket(); + compact.Sections.Add(DoubletSection(5, ArityRange.Doublets, 1, links)); + var bytes = compact.ToBytes(); + Assert.Equal("10 04 01 01 02 06 07 00 04 08", Hex(bytes)); + Assert.Equal("2", LinoFormat.FormatDocument(new BinaryLinoProtocol().Decode(bytes))); + + // The same links as a variable section of width 2 use the explicit layout. + var explicitLayout = new LinksPacket(); + explicitLayout.Sections.Add(DoubletSection(5, ArityRange.AtLeast(1), 2, links)); + bytes = explicitLayout.ToBytes(); + Assert.Equal("12 01 1d 05 00 04 01 01 00 01 00 01 02 00 06 00 01 07 00 00 00 01 04 00 08 00", Hex(bytes)); + Assert.Equal("2", LinoFormat.FormatDocument(new BinaryLinoProtocol().Decode(bytes))); + Assert.Equal("10 00", Hex(new LinksPacket().ToBytes())); + } + + private static PacketReference[] Internals(params ulong[] addresses) => + addresses.Select(PacketReference.Internal).ToArray(); + + [Fact] + public void PackingRejectsLinksItCannotLayOut() + { + void Unencodable(bool externalReferences, params (ulong, PacketReference[])[] links) + { + var error = Assert.Throws(() => LinksPacket.Pack(externalReferences, links, true)); + Assert.Equal(LinoProtocolErrorKind.Unencodable, error.Kind); + } + Unencodable(false, (0, Internals(1))); // address 0 is null + Unencodable(false, (2, Internals(1)), (1, Internals(1))); // descending + Unencodable(false, (2, Internals(1)), (2, Internals(1))); // repeated + Unencodable(false, (1, Array.Empty())); // no references + Unencodable(false, (1, new[] { PacketReference.External(1) })); + Unencodable(true, (1, new[] { PacketReference.External(1UL << 63) })); + Unencodable(true, (1, Internals(1UL << 63))); // the top bit marks externals + Assert.NotNull(LinksPacket.Pack(false, new[] { (1UL, Internals(ulong.MaxValue)) }, true)); + } + + [Fact] + public void WritingRejectsSectionsThatDoNotHoldTheirLinks() + { + static void Unencodable(Section section) + { + var packet = new LinksPacket(); + packet.Sections.Add(section); + var error = Assert.Throws(() => packet.ToBytes()); + Assert.Equal(LinoProtocolErrorKind.Unencodable, error.Kind); + } + Unencodable(DoubletSection(0, ArityRange.Exactly(3), 1, (1, 1))); + Unencodable(DoubletSection(0, ArityRange.Exactly(0), 1)); + Unencodable(DoubletSection(0, ArityRange.Between(3, 2), 1)); + Unencodable(DoubletSection(0, ArityRange.Exactly(1UL << 62), 1)); + Unencodable(DoubletSection(0, ArityRange.Doublets, 3, (1, 1))); + Unencodable(DoubletSection(0, ArityRange.Doublets, 1, (256, 0))); + Unencodable(DoubletSection(ulong.MaxValue, ArityRange.Doublets, 1, (1, 1))); + } + + private static LinoProtocolErrorKind DecodeError(byte[] bytes, DecodeLimits? limits = null) + { + var protocol = new BinaryLinoProtocol { Limits = limits ?? DecodeLimits.Default }; + return Assert.Throws(() => protocol.Decode(bytes)).Kind; + } + + [Theory] + [InlineData("")] + [InlineData("20 00")] // unknown version nibble + [InlineData("10 01 06 00")] // refers to itself + [InlineData("10 01 07 00")] // refers forward + [InlineData("10 02 00")] // truncated + [InlineData("10 00 00")] // trailing byte + [InlineData("10 ff ff ff ff ff ff ff ff ff 7f")] // LEB128 overflow + [InlineData("10 01 01 01")] // root is not a list + [InlineData("10 02 00 02 04 06")] // marker 2 used as a value + [InlineData("10 01 04 02")] // a bare marker in a list chain + [InlineData("16 00")] // the explicit layout keeps the width bits clear + [InlineData("12 01 04 05 00")] // arity 0 + [InlineData("12 01 18 01 01 05 00")] // length 6 in arity 1..2 + [InlineData("12 01 f8 ff ff ff ff ff ff ff ff 0f ff ff ff ff ff ff ff ff ff 01 00")] // arity overflows + [InlineData("12 02 04 ff ff ff ff ff ff ff ff ff 01 01 20 01")] // addresses overflow + [InlineData("12 01 24 06 01 01 01")] // LiNo links start at 6, not 7 + [InlineData("12 02 24 05 01 24 01 01 01 01 04 06")] // a hole + public void MalformedPacketsAreRejected(string hex) + { + Assert.Equal(LinoProtocolErrorKind.Malformed, DecodeError(Unhex(hex))); + } + + private static List<(ulong Address, PacketReference[] References)> FromAddress6(params PacketReference[][] links) => + links.Select((references, index) => (6UL + (ulong)index, references)).ToList(); + + [Fact] + public void InvalidCodePointsAreRejected() + { + // (String (0x110000)) is not a valid code point. + var links = FromAddress6( + new[] { PacketReference.External(0x11_0000), PacketReference.Null }, + Internals(3, 6), + new[] { PacketReference.Internal(7), PacketReference.Null }, + Internals(4, 8)); + Assert.Equal(LinoProtocolErrorKind.Malformed, DecodeError(LinksPacket.Pack(true, links, false).ToBytes())); + } + + [Fact] + public void HostilePacketsHitLimits() + { + void Limited(string hex, DecodeLimits limits) => + Assert.True(LinoProtocolErrorKind.LimitExceeded == DecodeError(Unhex(hex), limits), hex); + var fewLinks = new DecodeLimits { MaxLinks = 8 }; + Limited("10 09", fewLinks); + Limited("12 09", fewLinks); // more sections than links allowed + Limited("12 02 20 05 20 05", fewLinks); // 10 in all + var fewReferences = new DecodeLimits { MaxReferences = 5 }; + Limited("10 03 01 01 01 01 01 01", fewReferences); + Limited("12 01 18 00 01 05", fewReferences); // one link of 6 + + // A doubling chain `d(k) = (d(k-1) d(k-1))` expands to 2^k nodes. + var chain = new List { new[] { PacketReference.Null, PacketReference.Null } }; + for (ulong address = 6; address < 60; address++) + { + chain.Add(Internals(address, address)); + } + var last = 5 + (ulong)chain.Count; + chain.Add(new[] { PacketReference.Internal(last), PacketReference.Null }); + chain.Add(Internals(4, last + 1)); + var bomb = LinksPacket.Pack(false, FromAddress6(chain.ToArray()), false); + Assert.Equal(LinoProtocolErrorKind.LimitExceeded, DecodeError(bomb.ToBytes(), new DecodeLimits { MaxNodes = 1 << 12 })); + + var deep = string.Concat(Enumerable.Repeat("(y ", 40)) + "x" + new string(')', 40); + var bytes = new BinaryLinoProtocol().Encode(Parse(deep)); + Assert.Equal(LinoProtocolErrorKind.LimitExceeded, DecodeError(bytes, new DecodeLimits { MaxDepth = 10 })); + Assert.Equal(ulong.MaxValue, DecodeLimits.Unlimited.MaxLinks); + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs index e0fa9cc..7420496 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliTcpIntegrationTests.cs @@ -60,11 +60,11 @@ public async Task ClientsQueryAServedDatabaseOverBothProtocols() Assert.Equal("() ((2: 2 2))\n", await server.StdoutAsync("--protocol", "binary", "() ((2 2))")); Assert.Equal( "((1: 1 1)) ((1: 1 2))\n", - await server.StdoutAsync("--external-references", "--sequences", "((1: 1 1)) ((1: 1 2))")); + await server.StdoutAsync("--external-references", "--arity", "1..", "((1: 1 1)) ((1: 1 2))")); Assert.Equal("(1: 1 2)\n(2: 2 2)\n", await server.StdoutAsync()); Assert.Equal( "((1: 1 2)) ((1: 1 2))\n", - await server.StdoutAsync("--progressive-widths", "--query", "((1: 1 2)) ((1: 1 2))")); + await server.StdoutAsync("--packed-widths", "--arity", " 2..3 ", "--query", "((1: 1 2)) ((1: 1 2))")); // Deleting 2 also deletes 1, which refers to it, exactly like the Rust port. Assert.Equal("((2: 2 2)) ()\n((1: 1 2)) ()\n", await server.StdoutAsync("--protocol", "binary", "((2: 2 2)) ()")); Assert.Equal("", await server.StdoutAsync("--protocol", "text")); @@ -89,7 +89,10 @@ public async Task ServedChangesArePersisted() [Theory] [InlineData("--serve", "127.0.0.1:0", "--connect", "127.0.0.1:1")] - [InlineData("--connect", "127.0.0.1:1", "--protocol", "text", "--sequences")] + [InlineData("--connect", "127.0.0.1:1", "--protocol", "text", "--arity", "2..3")] + [InlineData("--connect", "127.0.0.1:1", "--protocol", "text", "--packed-widths")] + [InlineData("--connect", "127.0.0.1:1", "--arity", "0")] + [InlineData("--connect", "127.0.0.1:1", "--arity", "3..2")] [InlineData("--connect", "127.0.0.1:1", "--protocol", "udp")] [InlineData("--serve", "127.0.0.1:0", "--protocol", "udp")] [InlineData("--serve", "127.0.0.1:0", "() ((1 1))")] @@ -100,11 +103,19 @@ public async Task InvalidCombinationsAreRejected(params string[] arguments) Assert.False(string.IsNullOrWhiteSpace(stderr)); } + [Fact] + public async Task InvalidAritiesAreExplained() + { + var (exitCode, _, stderr) = await Clink.RunAsync("--connect", "127.0.0.1:1", "--arity", "2..x", "()"); + Assert.NotEqual(0, exitCode); + Assert.Contains("invalid value for '--arity': invalid arity '2..x': expected n, min..max or min..", stderr, StringComparison.Ordinal); + } + [Fact] public async Task HelpListsTheTcpOptions() { var (_, stdout, _) = await Clink.RunAsync("--help"); - foreach (var option in new[] { "--serve", "--connect", "--protocol", "--external-references", "--sequences", "--progressive-widths" }) + foreach (var option in new[] { "--serve", "--connect", "--protocol", "--external-references", "--arity", "--packed-widths" }) { Assert.Contains(option, stdout, StringComparison.Ordinal); } diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs index 9921f82..17aeec0 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs @@ -39,7 +39,7 @@ public void TextAndBinaryRepliesAreIdenticalDocuments() { using var server = new RunningServer(new LinksServerOptions { AutoCreateMissingReferences = true }); using var text = server.Client(new TextLinoProtocol()); - using var binary = server.Client(new BinaryLinoProtocol(new BinaryLinoOptions().WithExternalReferences().WithSequences())); + using var binary = server.Client(new BinaryLinoProtocol(new BinaryLinoOptions().WithExternalReferences().WithArity(ArityRange.AtLeast(1)))); text.Query("() ((child: father mother))"); binary.Query("() (('two words': 'it''s' \"x\"))"); Assert.Equal(text.Query(""), binary.Query("")); diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs index a3e3ad1..fed293f 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs @@ -1,5 +1,4 @@ using Foundation.Data.Doublets.Cli.Protocol; -using Platform.Data; using LinoLink = Link.Foundation.Links.Notation.Link; @@ -8,7 +7,7 @@ namespace Foundation.Data.Doublets.Cli.Tests.Protocol; /// Binary and text LiNo protocol codecs (issue #105). public sealed class LinoProtocolCodecTests { - private static readonly string[] Corpus = + internal static readonly string[] Corpus = { "() ((1 1))", "((1: 1 1)) ((1: 1 2))", @@ -39,49 +38,27 @@ public sealed class LinoProtocolCodecTests "(type: type type)", }; - /// Golden vectors shared with the Rust test suite: both ports must produce exactly these bytes. - public static TheoryData GoldenVectors => new() - { - { "() ((1 1))", false, false, "10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b" }, - { "() ((1 1))", false, true, "12 02 03 02 01 06 06 01 07 02 00 08 01 09" }, - { "() ((1 1))", true, false, "11 06 ff ff 06 00 04 07 00 08 09 00 04 0a" }, - { "() ((1 1))", true, true, "13 01 03 ff ff 01 06 02 00 07 01 08" }, - { "hi", true, false, "11 05 97 00 98 06 03 07 08 00 04 09" }, - { "hi", true, true, "13 00 02 03 03 98 97 01 06" }, - { "", false, false, "10 00" }, - { "", true, true, "13 00 00" }, - }; - internal static IEnumerable AllOptions() { foreach (var externalReferences in new[] { false, true }) { - foreach (var sequences in new[] { false, true }) + foreach (var arity in new[] { ArityRange.Doublets, ArityRange.Between(2, 3), ArityRange.AtLeast(1) }) { - foreach (var progressiveWidths in new[] { false, true }) + foreach (var packedWidths in new[] { false, true }) { - yield return new BinaryLinoOptions(externalReferences, sequences, progressiveWidths); + yield return new BinaryLinoOptions + { + ExternalReferences = externalReferences, + Arity = arity, + PackedWidths = packedWidths, + }; } } } } - private static string Hex(byte[] bytes) => string.Join(" ", bytes.Select(value => value.ToString("x2"))); - - private static byte[] Unhex(string text) => - text.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(value => Convert.ToByte(value, 16)).ToArray(); - private static IReadOnlyList Parse(string text) => LinoFormat.ParseDocument(text); - [Theory] - [MemberData(nameof(GoldenVectors))] - public void GoldenVectorsAreStable(string text, bool externalReferences, bool sequences, string expected) - { - var binary = new BinaryLinoProtocol(new BinaryLinoOptions(externalReferences, sequences)); - Assert.Equal(expected, Hex(binary.Encode(Parse(text)))); - Assert.Equal(Parse(text), new BinaryLinoProtocol().Decode(Unhex(expected))); - } - [Fact] public void EveryOptionSetRoundTripsTheCorpus() { @@ -176,155 +153,6 @@ public void EveryShortReferenceOverDelimitersRoundTrips() } } - [Fact] - public void WidthTiersFollowTheNumberOfLinks() - { - Assert.Equal(1, LinksPacket.AddressTier(0, false)); - Assert.Equal(1, LinksPacket.AddressTier(255, false)); - Assert.Equal(2, LinksPacket.AddressTier(256, false)); - Assert.Equal(2, LinksPacket.AddressTier(65_535, false)); - Assert.Equal(4, LinksPacket.AddressTier(65_536, false)); - Assert.Equal(4, LinksPacket.AddressTier(uint.MaxValue, false)); - Assert.Equal(8, LinksPacket.AddressTier((ulong)uint.MaxValue + 1, false)); - // External references take the top bit, halving every range. - Assert.Equal(1, LinksPacket.AddressTier(127, true)); - Assert.Equal(2, LinksPacket.AddressTier(128, true)); - Assert.Equal(2, LinksPacket.AddressTier(32_767, true)); - Assert.Equal(4, LinksPacket.AddressTier(32_768, true)); - Assert.Equal(ulong.MaxValue, LinksPacket.InternalCapacity(8, false)); - Assert.Equal((ulong)long.MaxValue, LinksPacket.InternalCapacity(8, true)); - Assert.Equal(127UL, LinksPacket.ExternalCapacity(1)); - } - - [Fact] - public void ExternalReferencesMatchPlatformDataHybrid() - { - foreach (var width in LinksPacket.Widths) - { - foreach (var value in new ulong[] { 0, 1, 2, 100, LinksPacket.ExternalCapacity(width) }) - { - var raw = LinksPacket.EncodeExternal(value, width); - Assert.True(LinksPacket.TryDecodeExternal(raw, width, out var decoded)); - Assert.Equal(value, decoded); - } - Assert.False(LinksPacket.TryDecodeExternal(LinksPacket.InternalCapacity(width, true), width, out _)); - } - Assert.Equal(0xFFUL, LinksPacket.EncodeExternal(1, 1)); - Assert.Equal(0x80UL, LinksPacket.EncodeExternal(0, 1)); - foreach (var value in new uint[] { 0, 1, 5, 1000, int.MaxValue }) - { - Assert.Equal((ulong)new Hybrid(value, isExternal: true).Value, LinksPacket.EncodeExternal(value, 4)); - } - } - - private static string ManyLinks(int count) => - string.Join("\n", Enumerable.Range(0, count).Select(index => $"(n{index} m{index})")); - - [Fact] - public void UniformWidthGrowsPast256Addresses() - { - var small = LinoMapping.EncodeDocument(Parse(ManyLinks(3))); - Assert.True(small.LastAddress <= 255); - Assert.Equal(1, small.MinWidth); - - var document = Parse(ManyLinks(400)); - var uniform = LinoMapping.EncodeDocument(document); - Assert.True(uniform.LastAddress > 255); - Assert.Equal(2, uniform.MinWidth); - var uniformBytes = uniform.ToBytes(); - const int header = 1 + 2; // header byte + two-byte LEB128 count - Assert.Equal(header + uniform.Doublets.Count * 4, uniformBytes.Length); - - var progressive = LinoMapping.EncodeDocument(document, new BinaryLinoOptions().WithProgressiveWidths()); - Assert.Equal(1, progressive.MinWidth); - var progressiveBytes = progressive.ToBytes(); - Assert.True(progressiveBytes.Length < uniformBytes.Length); - foreach (var bytes in new[] { uniformBytes, progressiveBytes }) - { - Assert.Equal(document, LinoMapping.DecodeDocument(LinksPacket.FromBytes(bytes))); - } - } - - [Fact] - public void LargeExternalValuesRaiseTheMinimumWidth() - { - var options = new BinaryLinoOptions(ExternalReferences: true, ProgressiveWidths: true); - Assert.Equal(2, LinoMapping.EncodeDocument(Parse("(1000 1)"), options).MinWidth); - Assert.Equal(1, LinoMapping.EncodeDocument(Parse("(100 1)"), options).MinWidth); - // Beyond 63 bits the number falls back to in-band unary links. - Assert.True(LinoMapping.EncodeDocument(Parse("18446744073709551615"), options).Doublets.Count > 60); - } - - [Fact] - public void HandBuiltPacketsDecode() - { - // One doublet (One One) is 2^1 in unary, so `(Number 7)` is the number 2. - var packet = new LinksPacket(); - packet.Doublets.Add((PacketReference.Internal(1), PacketReference.Internal(1))); - packet.Doublets.Add((PacketReference.Internal(2), PacketReference.Internal(6))); - packet.Doublets.Add((PacketReference.Internal(7), PacketReference.Null)); - packet.Doublets.Add((PacketReference.Internal(4), PacketReference.Internal(8))); - var bytes = packet.ToBytes(); - Assert.Equal("10 04 01 01 02 06 07 00 04 08", Hex(bytes)); - Assert.Equal("2", LinoFormat.FormatDocument(new BinaryLinoProtocol().Decode(bytes))); - } - - private static LinoProtocolErrorKind DecodeError(byte[] bytes, DecodeLimits? limits = null) - { - var protocol = new BinaryLinoProtocol { Limits = limits ?? DecodeLimits.Default }; - return Assert.Throws(() => protocol.Decode(bytes)).Kind; - } - - [Theory] - [InlineData("")] - [InlineData("20 00")] // unknown version nibble - [InlineData("10 01 06 00")] // refers to itself - [InlineData("10 01 07 00")] // refers forward - [InlineData("10 02 00")] // truncated - [InlineData("10 00 00")] // trailing byte - [InlineData("10 ff ff ff ff ff ff ff ff ff 7f")] // LEB128 overflow - [InlineData("10 01 01 01")] // root is not a list - [InlineData("10 02 00 02 04 06")] // marker 2 used as a value - [InlineData("10 01 04 02")] // a bare marker in a list chain - public void MalformedPacketsAreRejected(string hex) - { - Assert.Equal(LinoProtocolErrorKind.Malformed, DecodeError(Unhex(hex))); - } - - [Fact] - public void InvalidCodePointsAreRejected() - { - // (String (0x110000)) is not a valid code point. - var packet = new LinksPacket { ExternalReferences = true, MinWidth = 4 }; - packet.Doublets.Add((PacketReference.External(0x11_0000), PacketReference.Null)); - packet.Doublets.Add((PacketReference.Internal(3), PacketReference.Internal(6))); - packet.Doublets.Add((PacketReference.Internal(7), PacketReference.Null)); - packet.Doublets.Add((PacketReference.Internal(4), PacketReference.Internal(8))); - Assert.Equal(LinoProtocolErrorKind.Malformed, DecodeError(packet.ToBytes())); - } - - [Fact] - public void HostilePacketsHitLimits() - { - Assert.Equal(LinoProtocolErrorKind.LimitExceeded, DecodeError(Unhex("10 09"), new DecodeLimits { MaxLinks = 8 })); - - // A doubling chain `d(k) = (d(k-1) d(k-1))` expands to 2^k nodes. - var bomb = new LinksPacket(); - bomb.Doublets.Add((PacketReference.Null, PacketReference.Null)); - for (ulong address = 6; address < 60; address++) - { - bomb.Doublets.Add((PacketReference.Internal(address), PacketReference.Internal(address))); - } - var last = 5 + (ulong)bomb.Doublets.Count; - bomb.Doublets.Add((PacketReference.Internal(last), PacketReference.Null)); - bomb.Doublets.Add((PacketReference.Internal(4), PacketReference.Internal(last + 1))); - Assert.Equal(LinoProtocolErrorKind.LimitExceeded, DecodeError(bomb.ToBytes(), new DecodeLimits { MaxNodes = 1 << 12 })); - - var deep = string.Concat(Enumerable.Repeat("(y ", 40)) + "x" + new string(')', 40); - var bytes = new BinaryLinoProtocol().Encode(Parse(deep)); - Assert.Equal(LinoProtocolErrorKind.LimitExceeded, DecodeError(bytes, new DecodeLimits { MaxDepth = 10 })); - } - [Fact] public void TextMessagesAreDotStuffed() { @@ -361,7 +189,7 @@ public void TextMessagesAcceptCrlfAndRejectTruncation() public void ProtocolsAreDetectedPerMessage() { var document = Parse("() ((1 1))"); - var options = new BinaryLinoOptions().WithExternalReferences().WithSequences(); + var options = new BinaryLinoOptions().WithExternalReferences(); using var wire = new MemoryStream(); new TextLinoProtocol().WriteDocument(wire, document); new BinaryLinoProtocol(options).WriteDocument(wire, document); diff --git a/csharp/Foundation.Data.Doublets.Cli/Program.cs b/csharp/Foundation.Data.Doublets.Cli/Program.cs index 632eeeb..039de4d 100644 --- a/csharp/Foundation.Data.Doublets.Cli/Program.cs +++ b/csharp/Foundation.Data.Doublets.Cli/Program.cs @@ -199,15 +199,14 @@ DefaultValueFactory = _ => false }; -var sequencesOption = new Option("--sequences") +var arityOption = new Option("--arity") { - Description = "Binary protocol: send lists as a variable-length sequence section (implies --protocol binary)", - DefaultValueFactory = _ => false + Description = "Binary protocol: link lengths to use, 'n', 'min..max' or 'min..' (default: 2, doublets only; e.g. 2..3 adds triplets, 1.. any length; implies --protocol binary)" }; -var progressiveWidthsOption = new Option("--progressive-widths") +var packedWidthsOption = new Option("--packed-widths") { - Description = "Binary protocol: grow reference widths with the address (implies --protocol binary)", + Description = "Binary protocol: give each section of links the narrowest reference width it needs instead of one width for the whole packet (implies --protocol binary)", DefaultValueFactory = _ => false }; @@ -246,8 +245,8 @@ rootCommand.Options.Add(connectOption); rootCommand.Options.Add(protocolOption); rootCommand.Options.Add(externalReferencesOption); -rootCommand.Options.Add(sequencesOption); -rootCommand.Options.Add(progressiveWidthsOption); +rootCommand.Options.Add(arityOption); +rootCommand.Options.Add(packedWidthsOption); rootCommand.SetAction( parseResult => @@ -285,10 +284,19 @@ var serveAddress = parseResult.GetValue(serveOption); var connectAddress = parseResult.GetValue(connectOption); var protocolName = parseResult.GetValue(protocolOption)?.Trim(); - var binaryOptions = new BinaryLinoOptions( - parseResult.GetValue(externalReferencesOption), - parseResult.GetValue(sequencesOption), - parseResult.GetValue(progressiveWidthsOption)); + var arity = ArityRange.Doublets; + if (parseResult.GetValue(arityOption) is { } arityText + && !ArityRange.TryParse(arityText.Trim(), out arity, out var arityError)) + { + Console.Error.WriteLine($"invalid value for '--arity': {arityError}"); + return 1; + } + var binaryOptions = new BinaryLinoOptions + { + ExternalReferences = parseResult.GetValue(externalReferencesOption), + Arity = arity, + PackedWidths = parseResult.GetValue(packedWidthsOption), + }; var triggerCommandCount = new[] { always, once, never }.Count(value => value); if (triggerCommandCount > 1) From 4ef87ff118d0816314cab3d59ed098d841a3daf1 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:11:19 +0000 Subject: [PATCH 29/44] docs: specify the section-based binary links notation Add docs/protocol/binary-links-notation.md as the normative description of the format both ports implement, and replace every mention of the removed --sequences and --progressive-widths options with --arity and --packed-widths. The interop example now covers every new option. --- README.md | 19 +- csharp/README.md | 2 +- docs/HOW-IT-WORKS.md | 24 +- docs/REQUIREMENTS.md | 2 +- docs/case-studies/issue-105/README.md | 275 +++++++--------- docs/case-studies/issue-105/research-notes.md | 5 + .../binary-links-notation-vectors.txt | 2 +- docs/protocol/binary-links-notation.md | 300 ++++++++++++++++++ examples/tcp/README.md | 8 +- examples/tcp/run-interop.sh | 6 +- rust/README.md | 2 +- 11 files changed, 452 insertions(+), 193 deletions(-) create mode 100644 docs/protocol/binary-links-notation.md diff --git a/README.md b/README.md index 8aa5af8..fc0ca89 100644 --- a/README.md +++ b/README.md @@ -438,13 +438,18 @@ There are two interchangeable protocols: - **text**: UTF-8 LiNo. Each message ends with a line holding a single `.`, so you can also talk to the server with `nc`. -- **binary**: `--protocol binary`. Each packet carries its link count, and - references are 8, 16, 32 or 64 bits wide depending on that count. Three - optional decorators can each be switched on independently: +- **binary**: `--protocol binary`. Each packet is + [binary links notation](docs/protocol/binary-links-notation.md): it counts + its links first, and references are 8, 16, 32 or 64 bits wide depending on + the largest one. Three optional decorators can each be switched on + independently: - `--external-references`: numbers and characters travel as Hybrid external references. - - `--sequences`: a variable-length sequence section. - - `--progressive-widths`: the reference width grows with the address. + - `--arity `: the link lengths the encoder may use. The default, `2`, + sends only doublets. `2..3` adds triplets, and `1..` allows links of any + length. + - `--packed-widths`: each section of the packet gets the narrowest width its + links need. The server detects the protocol of each message and replies in the same one. C# and Rust servers and clients work with each other: see @@ -656,8 +661,8 @@ Both the C# NuGet tool and the Rust CLI support every option below. | `--connect` | string | _None_ | _None_ | Send the query to a `clink --serve` server at `host:port` and print the reply | | `--protocol` | string | detected | _None_ | `text` or `binary` (with `--serve`, also `any`, which is the default) | | `--external-references` | bool | `false` | _None_ | Binary protocol: send numbers and characters as external references | -| `--sequences` | bool | `false` | _None_ | Binary protocol: use the variable-length sequence section | -| `--progressive-widths` | bool | `false` | _None_ | Binary protocol: grow the reference width with the address | +| `--arity` | string | `2` | _None_ | Binary protocol: link lengths to use, `n`, `min..max` or `min..` | +| `--packed-widths` | bool | `false` | _None_ | Binary protocol: give each section the narrowest reference width | The query can be passed as the first positional argument or through `--query`, `--apply`, or `--do`. In the Rust CLI, `--query` takes precedence when both diff --git a/csharp/README.md b/csharp/README.md index b3ffd21..95bba18 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -81,7 +81,7 @@ library API for explicit batches. End-to-end demo scripts live in ```bash clink --db data.links --serve 127.0.0.1:7777 # serve a database clink --connect 127.0.0.1:7777 '() ((1 1))' # text protocol -clink --connect 127.0.0.1:7777 --protocol binary --sequences # binary protocol, list all links +clink --connect 127.0.0.1:7777 --protocol binary --arity 1.. # binary protocol, list all links ``` In the library, `LinksServer` serves any `INamedTypesLinks`, and diff --git a/docs/HOW-IT-WORKS.md b/docs/HOW-IT-WORKS.md index aabf9f8..ee61342 100644 --- a/docs/HOW-IT-WORKS.md +++ b/docs/HOW-IT-WORKS.md @@ -304,19 +304,23 @@ the reply. - The protocol is detected per message, and the reply uses the same one: - **Text**: UTF-8 LiNo. A line holding a single `.` ends a message, and leading dots are doubled. - - **Binary**: a packet whose first byte is `0x10 | flags`. It continues - with the number of fixed doublets and sequences, then the doublets - `source target` and the sequences `size ref…`. Addresses are implicit, - and a reference is 1, 2, 4 or 8 bytes wide, depending on the highest - address. + - **Binary**: a packet of + [binary links notation](protocol/binary-links-notation.md) whose first + byte is `0x10 | flags`. It continues with the number of links, or with + section headers, and then the links themselves. Addresses are implicit, + and a reference is 1, 2, 4 or 8 bytes wide, depending on the largest + one. - Numbers and strings are encoded as links by default: unary numbers and code-point lists under marker points. Three options change the encoding: - `--external-references` sends them as Hybrid external references instead. - - `--sequences` replaces cons chains with variable-length records. - - `--progressive-widths` lets early links use narrower references. - -The full wire format, golden vectors and design notes are in -[case-studies/issue-105](case-studies/issue-105/README.md). + - `--arity 2..3` or `--arity 1..` turns lists into single links of three or + any number of references, instead of cons chains of doublets. + - `--packed-widths` lets each section of a packet use the narrowest + references its links need. + +The wire format and its golden vectors are specified in +[protocol/binary-links-notation.md](protocol/binary-links-notation.md). The +design notes are in [case-studies/issue-105](case-studies/issue-105/README.md). ## Browser Runtime diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index afe66d2..0b5ec1c 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -50,7 +50,7 @@ the core CLI behavior and is also used by the WebAssembly browser workbench. | Document and test crash safety. | #98 | Durability is stated per backend: a file-mapped store survives a process crash with no explicit save and needs `flush()` only to survive a machine crash, while in-memory stores need `flush()`/`save()` for anything. Tests crash a file-backed store mid-rebuild and reopen it; recovery replays committed-but-unapplied transitions, rolls back uncommitted ones, and is idempotent. `transactions::FileTransitionLog` discards a tail torn by a crash. | | Support multi-process access to one database. | #98 | Both implementations take advisory locks on a `.lock` sidecar (shared for readers, exclusive for writers, blocking and non-blocking) and expose a `StorageRevision` fingerprint that answers "has anyone else written since I last looked?" without reparsing. File-mapped stores are mutated in place, so the inode never changes and other processes' mappings stay valid. | | Keep dependencies current across every language. | #98 | Rust, C#, and JavaScript dependencies were refreshed together, keeping shared libraries on the same version across languages (notably `links-notation` 0.16.1 in both Rust and C#). | -| LiNo substitution operations over TCP. | #105 | C# and Rust expose `--serve` / `--connect` and the `LinksServer` / `LinksClient` library types. Requests are substitution queries covering CRUD. There are two interchangeable protocols, a dot-terminated UTF-8 text protocol and a binary links packet:
  • the binary header carries the link count;
  • references are 8, 16, 32 or 64 bits wide depending on that count;
  • three optional decorators, external (Hybrid) references, a variable-length sequence section and progressive widths, can each be switched on independently.
Both ports produce identical bytes, and Rust and C# servers and clients work with each other. `links-notation` was upgraded to 0.22.0 in both languages, because older versions parsed nested input in exponential time. | +| LiNo substitution operations over TCP. | #105 | C# and Rust expose `--serve` / `--connect` and the `LinksServer` / `LinksClient` library types. Requests are substitution queries covering CRUD. There are two interchangeable protocols, a dot-terminated UTF-8 text protocol and a binary links packet:
  • the binary packet is [binary links notation](protocol/binary-links-notation.md), which counts its links before the links themselves;
  • references are 8, 16, 32 or 64 bits wide depending on the largest one;
  • three optional decorators, external (Hybrid) references, a configurable arity range (doublets by default, any length with `1..`) and packed per-section widths, can each be switched on independently.
Both ports produce identical bytes, and Rust and C# servers and clients work with each other. `links-notation` was upgraded to 0.22.0 in both languages, because older versions parsed nested input in exponential time. | | Separate code by implementation language. | #63, #64, #77, #79 | C# code and release helpers live under `csharp/`; Rust code, release helpers, and the WebAssembly wrapper crate live under `rust/`; the browser app and JavaScript lockfile live under `js/`. | | Provide Rust parity for core behavior. | #63, #67, #68 | Rust mirrors query processing, names, import/export, structure formatting, and Unicode sequence support. | | Run in a browser. | #12, #52, #69, #70 | The Rust query processor is wrapped with `wasm-bindgen` and surfaced through a React/Vite workbench. | diff --git a/docs/case-studies/issue-105/README.md b/docs/case-studies/issue-105/README.md index 32ca473..437c1fd 100644 --- a/docs/case-studies/issue-105/README.md +++ b/docs/case-studies/issue-105/README.md @@ -30,12 +30,12 @@ decorator, in both the C# and the Rust port. |---|---|---| | R1 | "lightweight protocols to transfer links notation representing substitution operations to allow all CRUD operations over TCP/IP" | `LinksServer` / `LinksClient` in both ports. A request is a substitution query and the empty request reads everything (§4.4). `clink --serve` / `--connect` (§4.6). `RemoteLinks` serves the links interface itself (§4.5) | | R2 | "UTF-8 based text only links notation" | `TextLinoProtocol` (§4.1) | -| R3 | "binary version of links notation, that will have number of links in a message/packet" | `BinaryLinoProtocol` and `LinksPacket`. The header carries N (doublets) and M (sequences) as LEB128 (§4.2) | -| R4 | "0-256 links have 8 bit size. 257-65536 links have 16 bits size and so on" (8/16/32/64) | The width is the tier of the largest address: 1, 2, 4 or 8 bytes. `--progressive-widths` grows the width per address instead (§4.2.3) | +| R3 | "binary version of links notation, that will have number of links in a message/packet" | `BinaryLinoProtocol` and `LinksPacket`. The header counts links (compact layout) or sections, each with its own count, as LEB128 (§4.2) | +| R4 | "0-256 links have 8 bit size. 257-65536 links have 16 bits size and so on" (8/16/32/64) | The width is the tier of the largest reference: 1, 2, 4 or 8 bytes. `--packed-widths` gives each section the narrowest width it needs instead (§4.2) | | R5 | The binary protocol is "implemented as another decorator, for both C# and Rust" and can be "used in the same manner as unicode version … easily switchable" | Both protocols implement `LinoProtocol` (Rust trait) / `ILinoProtocol` (C# interface). `LinoConnection` decorates any byte stream with either one, so switching means swapping one value. The server detects the protocol of each message and answers in kind (§4.3) | -| R6 | "representation of numbers as defined in links themselves (see how we define numbers in our linksplatform projects)" | Default mapping: a number is `(Number unary(n))`, built from powers of two as in `Platform.Data.Doublets.Numbers.Unary` (§4.2.4) | -| R7 | "Or … use external references to directly support encoding of unicode sequences": "0-128 links have 8 bit size. 128-32768 … 16 bits" | `--external-references` / `BinaryLinoOptions.ExternalReferences`. Numbers and code points become `Platform.Data.Hybrid` external references, and the internal range per width is halved (§4.2.5) | -| R8 | "at the end we can have improved format for storing variable length sequences … size, reference_1, reference_2 … Each variable link sequences is addressable itself by last link + 1" | `--sequences` / `BinaryLinoOptions.Sequences`. A section of `size ref…` records follows the fixed doublets, and sequence *k* has address `6 + N + k` (§4.2.6) | +| R6 | "representation of numbers as defined in links themselves (see how we define numbers in our linksplatform projects)" | Default mapping: a number is `(Number unary(n))`, built from powers of two as in `Platform.Data.Doublets.Numbers.Unary` (§4.2) | +| R7 | "Or … use external references to directly support encoding of unicode sequences": "0-128 links have 8 bit size. 128-32768 … 16 bits" | `--external-references` / `BinaryLinoOptions.ExternalReferences`. Numbers and code points become `Platform.Data.Hybrid` external references, and the internal range per width is halved (§4.2) | +| R8 | "at the end we can have improved format for storing variable length sequences … size, reference_1, reference_2 … Each variable link sequences is addressable itself by last link + 1" | `--arity` / `BinaryLinoOptions.Arity`, an `ArityRange` such as `2..3` or `1..`. A list of any allowed length is one link. A variable-arity section prefixes each link with its length, and every link keeps the address after the previous one (§4.2) | | R9 | "all these protocols options are optional, and come as decorators" | Every option is off by default and can be switched on independently, with `With…` builders. `AcceptedProtocols` lets a server restrict the protocol (§4.3) | | R10 | "Reuse all the best experience … from other repositories of link-assistant, link-foundation, and linksplatform" | §3 and `research-notes.md`. Hybrid encoding, unary numbers, the linksql CRUD table and the RFC 9457-like error shape were reused | | R11 | "collect data … to `./docs/case-studies/issue-{id}` … search online … list of each and all requirements … propose possible solutions and solution plans … check known existing components/libraries" | This folder | @@ -48,17 +48,24 @@ decorator, in both the C# and the Rust port. Hybrid top bit, 8-bit internal references reach `0..=127`. The issue's boundaries are each off by one. We follow the arithmetic, which is also what `Platform.Data.Hybrid` does. -- **"number of links in a message".** The header stores the number of fixed - doublets N and, when the sequence section is on, the number of sequences M. - The width follows from the largest address, `5 + N + M`, not from N alone. - Addresses `1..=5` are reserved marker points (§4.2.2), so a packet always - knows its own maximum width before it reads any reference. +- **"number of links in a message".** The compact layout stores the number + of links N. The explicit layout stores the number of sections and a count + for each. Either way, every count and width comes before the first + reference, so a reader knows each width before it reads a reference. The + widths follow from the largest reference in a section, not from N. + Addresses `1..=5` are reserved marker points (§4.2). +- **"variable length sequences".** The issue puts sequences after the + doublets, as a separate kind of record. We generalised that: any section + may hold links of a fixed arity or of a variable one, so doublets, + triplets and longer links mix freely. The default arity stays `2`, the + doublets of linksplatform. The format accepts any arity from 1 up, and + `1..` sets no maximum. - **"decorator".** In C#, `NamedTypesDecorator`, `PinnedTypesDecorator` and the others wrap an `ILinks`. A protocol here wraps a byte stream - (`LinoConnection`), and each binary option is one flag on an options value - that changes the encoder. Neither option needs the other. This keeps the + (`LinoConnection`), and each binary option is one setting on an options + value that changes the encoder. No option needs another. This keeps the options composable without a separate wrapper type for every combination of - three flags. A reader still accepts every combination, because the header + three options. A reader still accepts every combination, because the header announces the options in use. ## 3. Prior art @@ -88,11 +95,11 @@ encodings above. |---|---|---| | CBOR, [RFC 8949](https://www.rfc-editor.org/rfc/rfc8949.html) | Integer arguments inline, or in 1/2/4/8 following bytes | The 1/2/4/8-byte tiers, chosen once per link rather than once per value | | MessagePack, PackStream (Bolt) | Width escalation through marker bytes (`0xcc..0xcf`, `INT_8..INT_64`) | Same as above | -| LEB128 / protobuf varints | 7 bits per byte, little-endian groups | Header counts N and M | +| LEB128 / protobuf varints | 7 bits per byte, little-endian groups | Counts, section headers and link lengths | | [PostgreSQL](https://www.postgresql.org/docs/current/protocol-overview.html), [MySQL](https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_packets.html) | Self-delimiting messages on a TCP stream | Every binary packet is self-delimiting: counts first, then fixed-size or size-prefixed records | | SMTP (RFC 5321 §4.5.2) | Messages end with a line holding a single `.`, and leading dots are doubled | Text framing. It can be typed by hand in `nc` or `telnet` | | Redis RESP | A text protocol that humans can type, where the first byte tells the type | Protocol detection by the first byte (§4.3) | -| LZW | Code width grows as the dictionary grows | `--progressive-widths` | +| LZW | Code width grows as the dictionary grows | `--packed-widths`: the width follows the references of each section | ### Libraries considered @@ -129,123 +136,69 @@ A session in `nc`: ### 4.2 Binary protocol -#### 4.2.1 Packet layout - -```text -byte 0 0x10 | flags high nibble 1 = format version 1 - bit 0 external references (Hybrid encoding) - bit 1 sequence section present - bits 2-3 log2 of the minimum width in bytes -LEB128 N number of fixed doublets -LEB128 M number of sequences (only when bit 1 is set) -N times source target one fixed doublet, refs width(a) bytes each -M times size ref_1 … ref_n one sequence, size and refs width(a) bytes each -``` - -All references are little-endian. A packet is self-delimiting, so it needs no -length prefix. - -#### 4.2.2 Addresses - -| Address | Meaning | -|---|---| -| `0` | null, the empty link `()` | -| `1` | `One`, the unary 1 | -| `2` | `Number` marker | -| `3` | `String` marker | -| `4` | `List` marker | -| `5` | `Identified` marker (a link with an id) | -| `6 … 5+N` | fixed doublets | -| `6+N … 5+N+M` | sequences ("last link + 1", R8) | - -Markers are never transmitted. A link may only refer to links before it, so a -decoder resolves everything in one pass. The last link is the document root. - -#### 4.2.3 Width tiers (R4) - -`tier(a)` is the smallest of 1, 2, 4 and 8 bytes that can hold address `a`. -Every reference of the link at address `a` uses -`width(a) = max(min_width, tier(a))` bytes. - -- **Default (uniform).** `min_width` is the tier of the highest address, so - one width is used for the whole packet: - - | Highest address | Plain | With external references | - |---|---|---| - | up to 255 / 127 | 8-bit | 8-bit | - | up to 65 535 / 32 767 | 16-bit | 16-bit | - | up to 2³²−1 / 2³¹−1 | 32-bit | 32-bit | - | larger | 64-bit | 64-bit | - - The first column gives the limit for plain references, the second for references with external references on. - -- **`--progressive-widths`.** `min_width` is 1, so the first links use 8-bit - references even in a large packet. The width only grows when the address - grows. Since a link at address `a` can only refer to addresses below `a`, - `tier(a)` always fits. - -#### 4.2.4 Numbers as links (R6, the default) - -| LiNo | Links | -|---|---| -| `()` | `0` | -| number `n` | `(Number unary(n))`. `unary(0) = 0`, `2^0 = One`, `2^k = (2^(k-1) 2^(k-1))`, and other values are right-nested sums of powers of two from the highest bit down | -| other reference | `(String code-point…)`, with each code point a unary number | -| link of two values, no id | a plain doublet | -| link of any other arity, no id | `(List e…)` | -| link with an id | `(Identified id value…)` | -| document | `(List top-level-link…)`, the root | - -Identical sub-links are emitted once, because links are content-addressed. - -#### 4.2.5 External references (R7, `--external-references`) - -Numbers and code points travel as Hybrid external references instead of -unary links. At width `w` bits: - -- value `v ≥ 1` is sent as `2^w − v` -- `0` is sent as `2^(w−1)` - -This is exactly `Platform.Data.Hybrid`. The top bit marks a reference as -external, so internal addresses lose one bit: 8-bit references reach `0..127`. - -#### 4.2.6 Sequence section (R8, `--sequences`) - -Without this option, lists, strings and identified links are cons chains of -doublets: `(marker (e1 (e2 (… (en 0)))))`. With it, they become one -variable-length record `size marker e1 … en`, and a plain list has no marker. - -Fixed doublets may only refer to earlier fixed doublets. So a two-value link -that holds a sequence becomes a two-element sequence. - -#### 4.2.7 Decoding untrusted input - -`DecodeLimits` caps the following: - -- the number of links: 2²² -- the total sequence items: 2²⁴ -- the expanded LiNo nodes -- the nesting depth -- the text message size: 64 MiB - -The decoder also rejects: - -- a forward reference or a reference to itself -- an unsupported header byte, meaning an unknown version -- a width too small for an address -- a truncated packet +The binary protocol writes each document as one packet of +[binary links notation](../../protocol/binary-links-notation.md). That document is the normative specification. It covers the +header, the sections, the widths, external references, packing and the LiNo +mapping. This section explains how the design meets the issue. + +- **Count first (R3).** A packet starts with one header byte. Next comes a + LEB128 count of links in the compact layout, or a count of sections and + their headers in the explicit layout. After that come the links. So a + packet is self-delimiting, and a reader knows every width before it reads a + reference. +- **Sections.** A section is a run of consecutive addresses. It has a gap + before it, an arity range (how many references each of its links holds) + and a width. The common case is a document made of doublets, which is one + section at address 6. It is written in the compact layout, so the header + costs two bytes. +- **Width tiers (R4).** A width is the narrowest of 1, 2, 4 and 8 bytes that + holds every reference of a section. By default every section uses the + widest width in the packet, so all references have one size, as in the + issue. `--packed-widths` gives each section its own width and splits + sections where that saves bytes. A linear dynamic program chooses the + split, and the result is never larger than the uniform layout. +- **Numbers as links (R6, the default).** Addresses `1` to `5` are the marker + points `One`, `Number`, `String`, `List` and `Identified`, and they are + never sent. A number is `(Number unary(n))`, built from powers of two as in + `Platform.Data.Doublets.Numbers.Unary`. A string is `(String code-point…)`. + The document is the list of its top-level links, stored last. +- **External references (R7, `--external-references`).** Numbers and code + points become `Platform.Data.Hybrid` values: `v ≥ 1` is `2^w − v` and `0` + is `2^(w−1)`. The top bit marks a value as external, so internal addresses + lose one bit: 8-bit references reach `0..=127`. +- **Variable-length links (R8, `--arity`).** The arity range is the set of + link lengths the encoder may use. The default, `2`, writes only doublets, + so a list becomes the cons chain `(marker (e1 (e2 (… (en 0)))))`. With + `--arity 2..3`, a list of three elements becomes one triplet. With + `--arity 1..`, any list becomes one link. A variable-arity section prefixes + each link with its length minus the section minimum. That is the issue's + `size, reference_1, reference_2 …` record. Every link keeps its own + address, one after the previous link, as the issue asks. +- **Untrusted input.** `DecodeLimits` caps links (2²²), references (2²⁴), + expanded LiNo nodes, nesting depth (1024) and the text message size + (64 MiB). The decoder rejects: + - unknown header versions; + - LEB128 overflow; + - lengths outside a section's arity; + - forward references; + - misplaced markers; + - invalid code points; + - truncated packets; + - trailing bytes. ### 4.3 Decorators and switching (R5, R9) ```rust let mut connection = LinoConnection::new(stream, BinaryLinoProtocol::with_options( - BinaryLinoOptions::default().with_external_references(true).with_sequences(true))); + BinaryLinoOptions::default() + .with_external_references(true) + .with_arity(ArityRange::at_least(1)))); // …or TextLinoProtocol::new(); the rest of the code is unchanged. ``` ```csharp using var client = LinksClient.Connect("127.0.0.1", 7777, - new BinaryLinoProtocol(new BinaryLinoOptions().WithExternalReferences().WithSequences())); + new BinaryLinoProtocol(new BinaryLinoOptions().WithExternalReferences().WithArity(ArityRange.AtLeast(1)))); var reply = client.QueryText("() ((1 1))"); ``` @@ -308,7 +261,7 @@ client of either port is proven to understand a server of the other. clink --db data.links --serve 127.0.0.1:7777 [--protocol text|binary] [--auto-create-missing-references] [--trace] clink --connect 127.0.0.1:7777 '() ((1 1))' # text clink --connect 127.0.0.1:7777 --protocol binary # list all links over binary -clink --connect 127.0.0.1:7777 --external-references --sequences --progressive-widths '((1: 1 1)) ((1: 1 2))' +clink --connect 127.0.0.1:7777 --external-references --arity 1.. --packed-widths '((1: 1 1)) ((1: 1 2))' ``` On the client, any binary option implies `--protocol binary`, and @@ -319,68 +272,58 @@ and `--connect` together is also an error. ### Golden vectors -The Rust and C# tests both assert these bytes. The vectors use the uniform -width. +[`docs/protocol/binary-links-notation-vectors.txt`](../../protocol/binary-links-notation-vectors.txt) +holds 84 documents and 12 raw packets. Every document is encoded under all +12 option sets: plain or external references, arity `2`, `2..3` or `1..`, and +uniform or packed widths. The Rust and C# tests both assert every byte, in +both directions. A few of the vectors: -| Document | ext | seq | Bytes | +| Document | ext | arity | Bytes | |---|---|---|---| -| `() ((1 1))` | – | – | `10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b` | -| `() ((1 1))` | – | ✓ | `12 02 03 02 01 06 06 01 07 02 00 08 01 09` | -| `() ((1 1))` | ✓ | – | `11 06 ff ff 06 00 04 07 00 08 09 00 04 0a` | -| `() ((1 1))` | ✓ | ✓ | `13 01 03 ff ff 01 06 02 00 07 01 08` | -| `hi` | ✓ | – | `11 05 97 00 98 06 03 07 08 00 04 09` | -| `hi` | ✓ | ✓ | `13 00 02 03 03 98 97 01 06` | -| (empty) | – | – | `10 00` | -| (empty) | ✓ | ✓ | `13 00 00` | - -How the first vector decodes: - -- header `0x10`, N = 7 -- `6 = (Number One)` = 1 -- `7 = (6 6)` = `(1 1)` -- `8 = (7 0)` -- `9 = (List 8)` = `((1 1))` -- `10 = (0 9)` = `() ((1 1))` -- `11 = (10 0)` -- `12 = (List 11)`, the document +| (empty) | – | any | `10 00` | +| `() ((1 1))` | – | `2` | `10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b` | +| `() ((1 1))` | – | `1..` | `12 04 24 05 02 10 01 20 01 10 01 02 01 06 06 07 00 08 09` | +| `() ((1 1))` | ✓ | `2` | `11 06 ff ff 06 00 04 07 00 08 09 00 04 0a` | +| `() ((1 1))` | ✓ | `1..` | `13 01 1c 05 01 04 01 ff ff 00 06 01 00 07 00 08` | +| `hi` | ✓ | `2` | `11 05 97 00 98 06 03 07 08 00 04 09` | +| `hi` | ✓ | `1..` | `13 02 34 05 01 10 01 03 98 97 06` | + +[The specification](../../protocol/binary-links-notation.md#72-example) +decodes the first two of these byte by byte. ### Sizes -Output of `cargo run --example lino_binary_dump -- ''`, in bytes. The -text size includes the `\n.\n` terminator. +The [sizes table](../../protocol/binary-links-notation.md#9-sizes) of the +specification compares text with every option set. In short: -| Document | text | plain | seq | ext | ext+seq | +| Document | text | plain | ext | ext, `1..` | ext, `1..`, packed | |---|---|---|---|---|---| -| `() ((1 1))` | 13 | 16 | 14 | 14 | 12 | -| `((1: 1 1)) ((1: 1 2))` | 24 | 38 | 28 | 32 | 22 | -| `() ((child: father mother))` | 30 | 132 | 118 | 52 | 38 | -| `() ((1000 70000))` | 20 | 70 | 68 | 50 | 39 | +| `() ((1 1))` | 13 | 16 | 14 | 16 | 16 | +| `((1: 1 1)) ((1: 1 2))` | 24 | 38 | 32 | 25 | 25 | +| `() ((child: father mother))` | 30 | 132 | 52 | 41 | 41 | +| `() ((1000 70000))` | 20 | 70 | 50 | 34 | 23 | The pure-links mapping (R6) is not meant to be smaller than text. It transmits a document as nothing but links, which a links store can import -without any parser. External references and sequences are the options that -make the packet compact, and with both on, the binary form is the smallest -for numeric links. Text stays smaller for documents made of names or large -numbers, because: - -- each name costs a sequence with a marker and one reference per code point; -- each number needs a whole external reference at the packet's width. - -The binary protocol's advantage is that it needs no parsing and has a fixed -structure, not that it is smaller. +without any parser. The options that make a packet compact are external +references, a wider arity and packed widths. Text stays smaller for documents +made of names, because each name costs a string link with a marker and one +reference per code point. The binary protocol's advantage is that it needs no +parsing and has a fixed structure, not that it is smaller. ### Automated tests - Rust: - - `rust/tests/protocol_packet_tests.rs`: codecs, the golden vectors, the corpus round trip under all 8 option sets, limits, malformed input, and deep nesting in linear time. + - `rust/tests/protocol_packet_tests.rs`: both codecs. For the binary links notation it covers the golden vectors, the corpus round trip under all 12 option sets, width tiers and Hybrid values. It also covers packing (packed widths are never larger than uniform ones), arity ranges, hand-built packets, malformed input and limits. For text it covers framing, protocol detection, and deep nesting in linear time. - `rust/tests/protocol_tcp_tests.rs`: CRUD over every protocol, mixed clients, errors, CRLF, malformed input, protocol restriction, concurrency and shutdown. - `rust/tests/cli_tcp_tests.rs`: `clink --serve` / `--connect` end to end. - `rust/tests/remote_links_tests.rs`: the same proof for the Rust `RemoteLinks`, including the shared conversation. - C#: - - `LinoProtocolCodecTests` (34 tests): the same corpus and golden vectors. - - `LinksServerTests` (10 tests): CRUD over every protocol, identical text and binary replies, errors, CRLF, malformed input, protocol restriction, concurrent clients, shutdown, and shutdown racing a new connection. + - `BinaryLinksNotationTests`: a port of `protocol_packet_tests.rs`, checked against the same golden vectors. + - `LinoProtocolCodecTests`: the same corpus under all 12 option sets, and the text framing. + - `LinksServerTests`: CRUD over every protocol, identical text and binary replies, errors, CRLF, malformed input, protocol restriction, concurrent clients, shutdown, and shutdown racing a new connection. - `RemoteLinksTests`: every `INamedTypesLinks` call, the query processor and the raw `ILinks` interface give the same answers on a local store and over every protocol. It also covers the shared conversation, malformed operations and replies, and lost connections. - - `CliTcpIntegrationTests` (8 tests): `clink --serve` / `--connect` end to end. + - `CliTcpIntegrationTests`: `clink --serve` / `--connect` end to end, including option validation. - Cross-language interop: [`examples/tcp/run-interop.sh`](../../../examples/tcp/run-interop.sh) runs a Rust server with a C# client, and a C# server with a Rust client. It covers text and every binary option, and both directions print identical results. ## 6. Findings along the way diff --git a/docs/case-studies/issue-105/research-notes.md b/docs/case-studies/issue-105/research-notes.md index b5ed594..01584ce 100644 --- a/docs/case-studies/issue-105/research-notes.md +++ b/docs/case-studies/issue-105/research-notes.md @@ -272,6 +272,11 @@ pub fn is_internal(&self) -> bool { self.value < Self::half() } PackStream integers use `TINY_INT` inline (-16..127), then markers `0xC8/0xC9/0xCA/0xCB` for INT_8/16/32/64. This is again width escalation by a marker. ### Design hints derived from the above + +These hints came before the design. The format that was built is specified in +[binary links notation](../../protocol/binary-links-notation.md): the separate +sequence section became sections of any arity. + - Binary frame idea: a magic/version handshake (Bolt), then per message `type byte + length` (PostgreSQL) or a varint length. - The header can carry `linkCount`, and the reference width follows from it: `count ≤ 2^(w-1)-1` (hybrid) or `≤ 2^w-1` (plain). This is equivalent to choosing a CBOR-like 1/2/4/8-byte argument once per message instead of once per value. diff --git a/docs/protocol/binary-links-notation-vectors.txt b/docs/protocol/binary-links-notation-vectors.txt index 5831901..e30c8ec 100644 --- a/docs/protocol/binary-links-notation-vectors.txt +++ b/docs/protocol/binary-links-notation-vectors.txt @@ -1,6 +1,6 @@ # Golden vectors of binary links notation, version 1. # -# Both ports check them in their test suites (csharp: LinoProtocolCodecTests, +# Both ports check them in their test suites (csharp: BinaryLinksNotationTests, # rust: protocol_packet_tests), so they write and read exactly these bytes. # Fields are separated by tabs; bytes are hexadecimal. # diff --git a/docs/protocol/binary-links-notation.md b/docs/protocol/binary-links-notation.md new file mode 100644 index 0000000..6fc55d5 --- /dev/null +++ b/docs/protocol/binary-links-notation.md @@ -0,0 +1,300 @@ +# Binary links notation, version 1 + +Binary links notation stores **links**, each a tuple of one or more +references, in a self-delimiting packet. Both ports implement it the same way: +`LinksPacket` in Rust is in `rust/src/protocol/packet.rs`, and `LinksPacket` +in C# is in `csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs`. +[`binary-links-notation-vectors.txt`](binary-links-notation-vectors.txt) +holds golden vectors. Both test suites check them, so both ports write and +read exactly these bytes. + +The packet layer knows nothing about LiNo. The binary LiNo protocol (§7) is +one user of it. Any other links data can use it as well, for example a dump +of a links store. + +## 1. Conventions + +- **LEB128** means unsigned LEB128: 7 bits per byte, least significant group + first, and the high bit set on every byte except the last. A value must fit + in 64 bits. +- A **reference** is `width` bytes, little-endian, where `width` is 1, 2, 4 + or 8. The width is set per section (§3). +- An **address** is a link's position. `0` is null, and links are numbered + from `1`. + +## 2. Header byte + +```text +bit 7 6 5 4 3 2 1 0 + 0 0 0 1 width explicit external + version (log2) layout references +``` + +- **Version** is the high nibble. Version 1 is `0x1_`. A text LiNo message + never starts with a byte in `0x10..=0x1F`, so the header also tells a + binary packet apart from a text message. `0x2_` to `0xF_` are reserved. +- **External references** (bit 0): the top bit of every reference marks an + external value (§5). +- **Explicit layout** (bit 1): section headers follow (§3.2). Otherwise the + packet uses the compact layout (§3.1). +- **Width** (bits 2–3): the log2 of the reference width in the compact + layout. The explicit layout must keep these bits clear. + +## 3. Layouts and sections + +A packet is a list of **sections**. A section has: + +- a **gap**: how many addresses are skipped before it; +- an **arity range** `min..=max`: how many references each of its links holds; +- a **width**; +- its links. + +The first section starts at address `1 + gap`, and every later section starts +at `previous end + gap`. So gaps leave holes in the address space, and a +packet can hold any ascending set of addresses. + +### 3.1 Compact layout + +```text +0x10 | external | width << 2 +LEB128 N the number of links, all doublets +``` + +The compact layout is exactly one section with gap 5, arity 2 and the header +width. That is the common case of a LiNo document made of doublets, and it +costs just two header bytes. An empty packet is `10 00`, or `11 00` with +external references. + +### 3.2 Explicit layout + +```text +0x12 | external +LEB128 S the number of sections, then S section headers: + LEB128 shape bits 0-1 log2 of the reference width + bit 2 a gap follows + bit 3 variable arity + bits 4+ min, at least 1 + LEB128 gap only when bit 2 is set; a gap of 0 is not written + LEB128 extra only when bit 3 is set: 0 = no maximum, else max - min + LEB128 count the number of links in the section +``` + +A section without the variable-arity bit has a fixed arity: every link holds +exactly `min` references. + +### 3.3 Links + +After the headers come the links, section by section and in address order. + +- A link in a **fixed-arity** section is `min` references. +- A link in a **variable-arity** section is `LEB128 (length - min)` followed + by `length` references. The length must lie in the section's range. + +Each reference takes the width of its section. + +### 3.4 Examples + +These packets come from the `links` vectors. A section is written as +`address:references`, and `#v` is the external value `v`. + +`1:1 1; 2:1 2 3; 3:2 2 2; 10:3 9` becomes +`12 03 20 01 30 02 24 06 01 01 01 01 02 03 02 02 02 03 09`: + +| Bytes | Meaning | +|---|---| +| `12` | version 1, explicit layout | +| `03` | three sections | +| `20 01` | shape: min 2, fixed, 1-byte references; 1 link (address 1) | +| `30 02` | min 3, fixed, 1 byte; 2 links (addresses 2 and 3) | +| `24 06 01` | min 2, gap 6, 1 byte; 1 link at `3 + 1 + 6 = 10` | +| `01 01` | link 1 | +| `01 02 03` `02 02 02` | links 2 and 3 | +| `03 09` | link 10 | + +`1:1; 2:1 1; 3:1 1 1; 4:1 1 1 1` becomes +`12 01 18 03 04 00 01 01 01 01 02 01 01 01 03 01 01 01 01`. The shape `18` is +min 1, variable arity and 1-byte references. `03` is the extra (max 4) and +`04` is the count. Every link starts with its length minus 1. + +## 4. Widths + +An internal reference `a` needs the narrowest width that holds `a`: + +| Width | Plain addresses | With external references | +|---|---|---| +| 1 byte | `0..=255` | `0..=127` | +| 2 bytes | `0..=65 535` | `0..=32 767` | +| 4 bytes | `0..=2³²−1` | `0..=2³¹−1` | +| 8 bytes | `0..=2⁶⁴−1` | `0..=2⁶³−1` | + +A section's width must hold every reference in it. A link never needs to be +wide because of its own address, only because of the references it holds. + +## 5. External references + +With bit 0 of the header set, a reference is either a link address or an +**external value**, exactly like `Platform.Data.Hybrid`. At a width of +`w` bits: + +- an external value `v ≥ 1` is stored as `2^w − v`, which is its + two's-complement negation; +- an external `0` is stored as `2^(w−1)`; +- any raw value of `2^(w−1)` or more is external, and anything below it is an + internal address. + +An external value up to `2^(w−1) − 1` fits `w` bits, so it can be at most +2⁶³ − 1. Without external references, every reference is an internal address. + +## 6. Packing + +`pack(external_references, links, packed_widths)` lays out +`(address, references)` pairs given in ascending address order: + +- A hole between two addresses always starts a new section. +- Links of one length can share a fixed-arity section. Links of differing + lengths need a variable-arity section or separate sections. +- **Uniform widths** (the default) give every section the width of the widest + reference in the packet, so all references have one size. +- **Packed widths** let each section take the narrowest width its own links + need, and split sections wherever that saves bytes. + +A linear dynamic program chooses the sections. Its state is the width and +whether the arity is fixed or variable, which makes eight states. A link +either continues the section of the previous link or opens a new one, at an +estimated cost of 2 bytes for a section header and 1 more byte for a +variable arity. With packed widths, `pack` also builds the uniform layout and +keeps the packed one only when it is strictly smaller. So **packed widths +never cost more than uniform ones.** Both ports break ties the same way, so +they produce identical bytes. + +A packet whose only section matches §3.1 is written in the compact layout. + +## 7. LiNo mapping + +The binary LiNo protocol maps a LiNo document to links and packs them. It +uses only links, the way linksplatform represents data. Addresses `1` to `5` +are fixed **marker points** that are never transmitted, and links start at +`6`: + +| Address | Meaning | +|---|---| +| `0` | null, the empty link `()` | +| `1` | `One`, the unary 1 | +| `2` | `Number`: `(Number unary)` is a non-negative integer | +| `3` | `String`: `(String code-points…)` is a Unicode string | +| `4` | `List`: `(List elements…)` is a list of links | +| `5` | `Identified`: `(Identified id values…)` is a link with an id | +| `6…` | the links of the document | + +| LiNo | Links | +|---|---| +| `()` | `0` | +| number `n` | `(Number unary(n))`. `unary(0)` is null, `2^0` is `One`, `2^k` is `(2^(k-1) 2^(k-1))`, and other numbers are right-nested sums of powers of two from the highest bit down. With external references, `n` is the external value `n` instead | +| any other reference | `(String code-point…)`. Each code point is a unary number, or an external value with external references | +| a link of two values, no id | a doublet | +| a link of any other number of values, no id | a list | +| a link with an id | `(Identified id value…)` | +| the document | the list of its top-level links, stored last as the root | + +A link of two values is always a doublet. A list or typed value of any +other length `n` becomes **one link** of `n` references when the arity range +contains `n`. That link is the bare elements for a list, or +`marker elements…` for a typed value. Otherwise it becomes the doublet +`(marker chain)`, where `chain` is the nil-terminated cons list +`(e1 (e2 (… (en 0))))`. Identical sub-links are written once and shared, +because links are content-addressed. Links are numbered so that each one +refers only to earlier ones: the doublets of plain doublets come first, then +the rest in creation order. + +### 7.1 Options + +Each option is off by default and can be switched on on its own: + +| Option | Rust | C# | CLI | Effect | +|---|---|---|---|---| +| external references | `with_external_references(true)` | `WithExternalReferences()` | `--external-references` | numbers and code points become external values (§5) | +| arity | `with_arity(ArityRange)` | `WithArity(ArityRange)` | `--arity 2..3` | the link lengths the encoder may use. The default, `2`, writes only doublets. `2..3` adds triplets, and `1..` allows any length. The range must contain 2 | +| packed widths | `with_packed_widths(true)` | `WithPackedWidths()` | `--packed-widths` | per-section widths (§6) | + +An arity is written `n`, `min..max` (inclusive) or `min..` (no maximum). + +A decoder needs none of these options. The header and the section headers +announce everything. A server answers in the style of the request: the +reply options come from the received packet. External references are taken +from the header. The arity is `min(2, shortest)..=max(2, longest)` over the +link lengths in the packet, and packed widths are on when the sections +differ in width. + +### 7.2 Example + +`() ((1 1))` with the defaults becomes +`10 07 02 01 06 06 07 00 04 08 00 09 0a 00 04 0b`, which is the compact layout +with 7 doublets of 1 byte each: + +| Address | Link | Meaning | +|---|---|---| +| 6 | `02 01` | `(Number One)`, the number 1 | +| 7 | `06 06` | `(1 1)` | +| 8 | `07 00` | the chain `((1 1))` | +| 9 | `04 08` | `(List 8)`, the list `((1 1))` | +| 10 | `00 09` | `() ((1 1))` | +| 11 | `0a 00` | the chain of top-level links | +| 12 | `04 0b` | `(List 11)`, the document | + +With `--arity 1..`, the two lists become single links and the packet becomes +`12 04 24 05 02 10 01 20 01 10 01 02 01 06 06 07 00 08 09`. That is four +fixed-arity sections: doublets 6 and 7 after the gap of 5, the one-element +list `8 = (7)`, the doublet `9 = (0 8)` and the document `10 = (9)`. + +## 8. Decoding untrusted input + +A packet must be read whole. Bytes after it, or a packet that ends early, +are an error. The decoder rejects: + +- a header outside `0x10..=0x1F`, or an explicit layout with width bits set; +- a LEB128 value over 64 bits; +- a section shape with min 0, or an arity range that overflows; +- a link length outside its section's arity; +- addresses that overflow 64 bits. + +`DecodeLimits` bounds the work a peer can cause: + +| Limit | Default | +|---|---| +| links in a packet | 2²² | +| references in all links | 2²⁴ | +| LiNo nodes a packet expands to | 2²² | +| LiNo nesting depth | 1024 | +| text message size | 64 MiB | + +The LiNo mapping also requires that: + +- links are contiguous from address 6; +- each link refers only to earlier links; +- markers appear only in marker position; +- numbers are well-formed unary values and code points are valid Unicode + scalar values; +- the root is a list. + +## 9. Sizes + +The table below lists the number of bytes each encoding of a document takes. +The values come from +`cargo run --example lino_binary_dump -- ''` in `rust/`. The +text size includes the `\n.\n` terminator. When the packed size differs from +the uniform size, it is given after a slash. + +| Document | text | plain 2 | plain 2..3 | plain 1.. | ext 2 | ext 2..3 | ext 1.. | +|---|---|---|---|---|---|---|---| +| `() ((1 1))` | 13 | 16 | 16 | 19 | 14 | 14 | 16 | +| `((1: 1 1)) ((1: 1 2))` | 24 | 38 | 38 | 33 | 32 | 32 | 25 | +| `() ((child: father mother))` | 30 | 132 | 132 | 123 | 52 | 52 | 41 | +| `() ((1000 70000))` | 20 | 70 | 70 | 73 | 50 / 25 | 50 / 25 | 34 / 23 | +| `(a b c d e f g h)` | 20 | 108 | 108 | 103 | 56 | 56 | 51 | + +The plain mapping is not meant to beat text. It sends a document as nothing +but links, which a links store can take in without a parser. External +references, a wider arity and packed widths are the options that make a +packet small. Packed widths matter when a few large values would otherwise +widen every reference: in `() ((1000 70000))`, only the link holding 70 000 +needs 4 bytes. diff --git a/examples/tcp/README.md b/examples/tcp/README.md index 31de366..e3b78ba 100644 --- a/examples/tcp/README.md +++ b/examples/tcp/README.md @@ -22,9 +22,10 @@ server: clink server listening on 127.0.0.1:40123 [text] () ((1: 1 1)) [--protocol binary] () ((2: 2 2)) [--external-references] () ((3: 3 3)) -[--sequences] () ((4: 4 4)) -[--progressive-widths] () ((5: 5 5)) -[--external-references --sequences --progressive-widths] () ((6: 6 6)) +[--arity 2..3] () ((4: 4 4)) +[--arity 1..] () ((5: 5 5)) +[--packed-widths] () ((6: 6 6)) +[--external-references --arity 1.. --packed-widths] () ((7: 7 7)) listing: (1: 1 1) (2: 2 2) @@ -32,6 +33,7 @@ listing: (4: 4 4) (5: 5 5) (6: 6 6) +(7: 7 7) ``` ## Doing it by hand diff --git a/examples/tcp/run-interop.sh b/examples/tcp/run-interop.sh index 47e2ddf..f22b26e 100755 --- a/examples/tcp/run-interop.sh +++ b/examples/tcp/run-interop.sh @@ -38,11 +38,11 @@ address="$(sed -n 's/^clink server listening on //p' "$work_dir/banner")" echo "server: $(head -n 1 "$work_dir/banner")" index=0 -for flags in "" "--protocol binary" "--external-references" "--sequences" \ - "--progressive-widths" "--external-references --sequences --progressive-widths"; do +for flags in "" "--protocol binary" "--external-references" "--arity 2..3" \ + "--arity 1.." "--packed-widths" "--external-references --arity 1.. --packed-widths"; do index=$((index + 1)) # shellcheck disable=SC2086 # flags are meant to split echo "[${flags:-text}] $("${client[@]}" --connect "$address" $flags "() (($index $index))")" done echo "listing:" -"${client[@]}" --connect "$address" --protocol binary --sequences +"${client[@]}" --connect "$address" --protocol binary --arity 1.. diff --git a/rust/README.md b/rust/README.md index 6e82e59..a28df51 100644 --- a/rust/README.md +++ b/rust/README.md @@ -64,7 +64,7 @@ End-to-end demo scripts live in ```bash clink --db data.links --serve 127.0.0.1:7777 # serve a database clink --connect 127.0.0.1:7777 '() ((1 1))' # text protocol -clink --connect 127.0.0.1:7777 --protocol binary --sequences # binary protocol, list all links +clink --connect 127.0.0.1:7777 --protocol binary --arity 1.. # binary protocol, list all links ``` In the library, `link_cli::protocol::LinksServer` serves any `NamedTypeLinks` store, and From 814d06d84518e765f7e7a64715368719831564c0 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:18:43 +0000 Subject: [PATCH 30/44] feat(rust): export and import a whole store in binary links notation --- ...20261005_150000_issue_105_store_archive.md | 5 + rust/src/cli.rs | 28 ++ rust/src/lino_database_input.rs | 2 +- rust/src/main.rs | 40 +- rust/src/protocol/archive.rs | 170 ++++++++ rust/src/protocol/mod.rs | 2 + rust/tests/store_archive_tests.rs | 364 ++++++++++++++++++ 7 files changed, 598 insertions(+), 13 deletions(-) create mode 100644 rust/changelog.d/20261005_150000_issue_105_store_archive.md create mode 100644 rust/src/protocol/archive.rs create mode 100644 rust/tests/store_archive_tests.rs diff --git a/rust/changelog.d/20261005_150000_issue_105_store_archive.md b/rust/changelog.d/20261005_150000_issue_105_store_archive.md new file mode 100644 index 0000000..94d50bb --- /dev/null +++ b/rust/changelog.d/20261005_150000_issue_105_store_archive.md @@ -0,0 +1,5 @@ +--- +bump: minor +--- + +Added `--export-binary ` (aliases `--binary-output`, `--binary-out`) and `--import-binary ` (aliases `--binary-input`, `--binary-in`), which write and read the whole store, names included, as a store archive in binary links notation: a links packet that keeps every address and hole, followed by a names packet. The library exposes the same as `protocol::export_store`, `protocol::import_store` and their `_file` variants. A binary archive is imported before the `--in` LiNo file and exported wherever `--out` is written. The archive bytes match the C# port. diff --git a/rust/src/cli.rs b/rust/src/cli.rs index 6c5c8cd..41cb0d8 100644 --- a/rust/src/cli.rs +++ b/rust/src/cli.rs @@ -20,6 +20,8 @@ pub struct Cli { pub after: bool, pub lino_input: Option, pub lino_output: Option, + pub binary_input: Option, + pub binary_output: Option, pub always: bool, pub once: bool, pub never: bool, @@ -61,6 +63,8 @@ impl Default for Cli { after: false, lino_input: None, lino_output: None, + binary_input: None, + binary_output: None, always: false, once: false, never: false, @@ -200,6 +204,19 @@ impl Cli { cli.lino_input = Some(value.to_string()); continue; } + if let Some(value) = inline_value( + &arg, + &["--export-binary", "--binary-output", "--binary-out"], + ) { + cli.binary_output = Some(value.to_string()); + continue; + } + if let Some(value) = + inline_value(&arg, &["--import-binary", "--binary-input", "--binary-in"]) + { + cli.binary_input = Some(value.to_string()); + continue; + } if let Some(value) = inline_value(&arg, &["--always"]) { cli.always = parse_bool("--always", value)?; continue; @@ -335,6 +352,12 @@ impl Cli { "--in" | "--lino-input" | "--import" => { cli.lino_input = Some(next_value(&mut args, &arg)?); } + "--export-binary" | "--binary-output" | "--binary-out" => { + cli.binary_output = Some(next_value(&mut args, &arg)?); + } + "--import-binary" | "--binary-input" | "--binary-in" => { + cli.binary_input = Some(next_value(&mut args, &arg)?); + } "--always" => { cli.always = next_bool_value(&mut args, true)?; } @@ -460,6 +483,11 @@ impl Cli { " Read and import a LiNo file into the database\n", " --out , --lino-output , --export \n", " Write the complete database as a LiNo file\n", + " --import-binary , --binary-input , --binary-in \n", + " Read and import a store archive in binary links notation\n", + " --export-binary , --binary-output , --binary-out \n", + " Write the complete database, names included, as a store archive in\n", + " binary links notation\n", " --always\n", " Store the query as an always-on persistent transformation trigger\n", " --once\n", diff --git a/rust/src/lino_database_input.rs b/rust/src/lino_database_input.rs index a293cc6..fc0f4d5 100644 --- a/rust/src/lino_database_input.rs +++ b/rust/src/lino_database_input.rs @@ -104,7 +104,7 @@ where storage.get_or_create_named(identifier) } -fn update_link(storage: &mut T, index: u32, source: u32, target: u32) -> Result<()> +pub(crate) fn update_link(storage: &mut T, index: u32, source: u32, target: u32) -> Result<()> where T: NamedTypeLinks, { diff --git a/rust/src/main.rs b/rust/src/main.rs index 408fb27..7a35a1e 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -7,8 +7,8 @@ use anyhow::{anyhow, bail, Result}; use link_cli::cli::{Cli, CliCommand}; use link_cli::import_lino_file; use link_cli::protocol::{ - AcceptedProtocols, BinaryLinoOptions, BinaryLinoProtocol, LinksClient, LinksServer, - LinoProtocol, ServerOptions, TextLinoProtocol, + export_store_file, import_store_file, AcceptedProtocols, BinaryLinoOptions, BinaryLinoProtocol, + LinksClient, LinksServer, LinoProtocol, ServerOptions, TextLinoProtocol, }; use link_cli::{ make_triggers_database_filename, CommitMode, LogRetentionPolicy, NamedTypeLinks, @@ -366,25 +366,19 @@ where storage.print_all_lino()?; } - if let Some(input_path) = &cli.lino_input { - import_lino_file(storage, input_path)?; - } + import_inputs(cli, storage)?; if let Some(link_id) = cli.structure { let structure_formatted = storage.format_structure(link_id)?; println!("{structure_formatted}"); - if let Some(output_path) = &cli.lino_output { - storage.write_lino_output(output_path)?; - } + export_outputs(cli, storage)?; return Ok(()); } let effective_query = cli.query.as_deref().or(cli.query_arg.as_deref()); if trigger_stage(cli, storage, effective_query)? { - if let Some(output_path) = &cli.lino_output { - storage.write_lino_output(output_path)?; - } + export_outputs(cli, storage)?; return Ok(()); } @@ -408,10 +402,32 @@ where storage.print_all_lino()?; } + export_outputs(cli, storage)?; + + Ok(()) +} + +/// Imports the store archive of `--import-binary`, then the LiNo file of +/// `--in`. +fn import_inputs(cli: &Cli, storage: &mut S) -> Result<()> { + if let Some(input_path) = &cli.binary_input { + import_store_file(storage, input_path)?; + } + if let Some(input_path) = &cli.lino_input { + import_lino_file(storage, input_path)?; + } + Ok(()) +} + +/// Writes the whole store to the LiNo file of `--out` and the store archive +/// of `--export-binary`. +fn export_outputs(cli: &Cli, storage: &mut S) -> Result<()> { if let Some(output_path) = &cli.lino_output { storage.write_lino_output(output_path)?; } - + if let Some(output_path) = &cli.binary_output { + export_store_file(storage, output_path)?; + } Ok(()) } diff --git a/rust/src/protocol/archive.rs b/rust/src/protocol/archive.rs new file mode 100644 index 0000000..4698448 --- /dev/null +++ b/rust/src/protocol/archive.rs @@ -0,0 +1,170 @@ +//! Store archive: a whole links store in binary links notation. +//! +//! An archive is two [`LinksPacket`]s, one after the other: +//! +//! 1. **Links**, without external references: the store link `(a: s t)` is +//! the doublet `s t` at address `a`. Addresses the store does not use are +//! holes, so every link keeps its address. +//! 2. **Names**, with external references: one link per named store link, +//! in address order, holding the external values +//! `address code-point…`. +//! +//! Both packets use packed widths, so a store of small addresses costs about +//! two bytes per link. [`import_store`] writes every link back at its own +//! address and restores the names, so exporting the imported store gives +//! the same bytes again. + +use super::error::{ProtocolError, ProtocolResult}; +use super::packet::{DecodeLimits, LinksPacket, Reference}; +use crate::lino_database_input::update_link; +use crate::named_type_links::NamedTypeLinks; +use anyhow::{Context, Result}; +use std::fs; +use std::io::{Read, Write}; +use std::path::Path; + +/// Writes every link and name of `storage` to `writer`. +pub fn export_store(storage: &mut S, writer: &mut dyn Write) -> Result<()> +where + S: NamedTypeLinks, +{ + let mut links = storage.all_links(); + links.sort_by_key(|link| link.index); + let doublets: Vec<(u64, Vec)> = links + .iter() + .map(|link| { + ( + u64::from(link.index), + vec![ + Reference::Internal(u64::from(link.source)), + Reference::Internal(u64::from(link.target)), + ], + ) + }) + .collect(); + let mut names = Vec::new(); + for link in &links { + if let Some(name) = storage.get_name(link.index)? { + let mut references = vec![Reference::External(u64::from(link.index))]; + references.extend( + name.chars() + .map(|code_point| Reference::External(u64::from(u32::from(code_point)))), + ); + names.push((names.len() as u64 + 1, references)); + } + } + LinksPacket::pack(false, &doublets, true)?.write_to(writer)?; + LinksPacket::pack(true, &names, true)?.write_to(writer)?; + Ok(()) +} + +/// Writes the archive of `storage` to the file at `path`. +pub fn export_store_file(storage: &mut S, path: P) -> Result<()> +where + S: NamedTypeLinks, + P: AsRef, +{ + let path = path.as_ref(); + let mut bytes = Vec::new(); + export_store(storage, &mut bytes)?; + fs::write(path, bytes) + .with_context(|| format!("Failed to write the store archive: {}", path.display())) +} + +/// Reads an archive from `reader` into `storage`: each link is written at +/// its own address, then every name is set. +pub fn import_store(storage: &mut S, reader: &mut dyn Read) -> Result<()> +where + S: NamedTypeLinks, +{ + let limits = DecodeLimits::unlimited(); + let links = read_packet(reader, &limits, "links")?; + let names = read_packet(reader, &limits, "names")?; + let mut trailing = [0u8; 1]; + if reader.read(&mut trailing)? != 0 { + return Err(ProtocolError::malformed("trailing bytes after the store archive").into()); + } + let doublets = links + .links() + .map(|(address, references)| match references { + [Reference::Internal(source), Reference::Internal(target)] => Ok(( + store_address(address)?, + store_address(*source)?, + store_address(*target)?, + )), + _ => Err(ProtocolError::malformed(format!( + "archive link {address} is not a doublet of link addresses" + ))), + }) + .collect::>>()?; + let named = names + .links() + .map(|(_, references)| decode_name(references)) + .collect::>>()?; + // Every address exists before any link refers to it. + for &(index, _, _) in &doublets { + if !storage.exists(index) { + storage.ensure_created(index); + } + } + for (index, source, target) in doublets { + update_link(storage, index, source, target)?; + } + for (index, name) in named { + storage.set_name(index, &name)?; + } + storage.save() +} + +/// Reads the archive in the file at `path` into `storage`. +pub fn import_store_file(storage: &mut S, path: P) -> Result<()> +where + S: NamedTypeLinks, + P: AsRef, +{ + let path = path.as_ref(); + let bytes = fs::read(path) + .with_context(|| format!("Failed to read the store archive: {}", path.display()))?; + import_store(storage, &mut bytes.as_slice()) +} + +fn read_packet( + reader: &mut dyn Read, + limits: &DecodeLimits, + part: &str, +) -> ProtocolResult { + LinksPacket::read_from(reader, limits)?.ok_or_else(|| { + ProtocolError::malformed(format!("the store archive ends before its {part}")) + }) +} + +fn store_address(address: u64) -> ProtocolResult { + u32::try_from(address).map_err(|_| { + ProtocolError::malformed(format!("address {address} does not fit a 32-bit store")) + }) +} + +fn decode_name(references: &[Reference]) -> ProtocolResult<(u32, String)> { + let values = references + .iter() + .map(|reference| match reference { + Reference::External(value) => Ok(*value), + Reference::Internal(_) => Err(ProtocolError::malformed( + "an archive name holds only external values", + )), + }) + .collect::>>()?; + let (&address, code_points) = values + .split_first() + .ok_or_else(|| ProtocolError::malformed("an archive name needs a link address"))?; + let name = code_points + .iter() + .map(|&code_point| { + u32::try_from(code_point) + .ok() + .and_then(char::from_u32) + .ok_or_else(|| ProtocolError::malformed(format!("invalid code point {code_point}"))) + }) + .collect::>()?; + Ok((store_address(address)?, name)) +} diff --git a/rust/src/protocol/mod.rs b/rust/src/protocol/mod.rs index a53a3e9..c504f68 100644 --- a/rust/src/protocol/mod.rs +++ b/rust/src/protocol/mod.rs @@ -34,6 +34,7 @@ //! assert_eq!(format_document(&document), "() ((1 1))"); //! ``` +pub mod archive; mod client; mod error; mod format; @@ -44,6 +45,7 @@ mod protocols; mod remote_links; mod server; +pub use archive::{export_store, export_store_file, import_store, import_store_file}; pub use client::LinksClient; pub use error::{ProtocolError, ProtocolResult}; pub use format::{format_document, format_link, format_reference, parse_document}; diff --git a/rust/tests/store_archive_tests.rs b/rust/tests/store_archive_tests.rs new file mode 100644 index 0000000..2abc0a3 --- /dev/null +++ b/rust/tests/store_archive_tests.rs @@ -0,0 +1,364 @@ +//! A whole store survives `--export-binary` and `--import-binary`: links keep +//! their addresses, holes stay holes, names come back, and the archive bytes +//! match the C# port. + +use link_cli::cli::{Cli, CliCommand}; +use link_cli::protocol::{ + export_store, import_store, LinksPacket, Reference, RemoteLinks, ServerOptions, + TextLinoProtocol, +}; +use link_cli::{Link, NamedTypeLinks, NamedTypesDecorator}; +use std::fs; +use std::path::Path; +use std::process::{Command, Output}; +use tempfile::{tempdir, TempDir}; + +mod common; + +use common::RunningServer; + +/// The archive of [`fill`], byte for byte; the C# `StoreArchiveTests` +/// assert the same bytes. +/// +/// - Links packet, 13 bytes: `12` explicit layout, `02` sections; `20 01` +/// one doublet of 1-byte references, `01 01`; `24 01 02` a section after a +/// gap of one address (the hole at 2) with two doublets, `03 03 01 03`. +/// - Names packet, 17 bytes: `13` explicit layout with external references, +/// `02` sections; `21 02` two links of two 2-byte references, +/// `ff ff ff 9f` (1, `a`) and `ff fd ff 17` (3, `é`); `30 01` one link of +/// three 1-byte references, `fc 9f 9e` (4, `a`, `b`). +const ARCHIVE: &str = "12 02 20 01 24 01 02 01 01 03 03 01 03 \ + 13 02 21 02 30 01 ff ff 9f ff fd ff 17 ff fc 9f 9e"; + +fn store(directory: &TempDir, name: &str) -> NamedTypesDecorator { + NamedTypesDecorator::new(directory.path().join(name), false).unwrap() +} + +/// Links `1: 1 1` named `a`, `3: 3 3` named `é` and `4: 1 3` named `ab`, +/// with a hole at 2. +fn fill(store: &mut impl NamedTypeLinks) { + for _ in 0..3 { + let id = store.create(0, 0); + store.update(id, id, id).unwrap(); + } + assert_eq!(store.create(1, 3), 4); + store.delete(2).unwrap(); + store.set_name(1, "a").unwrap(); + store.set_name(3, "é").unwrap(); + store.set_name(4, "ab").unwrap(); +} + +fn export(store: &mut impl NamedTypeLinks) -> Vec { + let mut bytes = Vec::new(); + export_store(store, &mut bytes).unwrap(); + bytes +} + +fn hex(bytes: &[u8]) -> String { + bytes + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::>() + .join(" ") +} + +fn bytes(hex: &str) -> Vec { + hex.split_whitespace() + .map(|byte| u8::from_str_radix(byte, 16).unwrap()) + .collect() +} + +fn names(store: &mut impl NamedTypeLinks) -> Vec<(u32, Option)> { + (1..=4) + .map(|id| (id, store.get_name(id).unwrap())) + .collect() +} + +fn links(store: &mut impl NamedTypeLinks) -> Vec { + let mut links = store.all_links(); + links.sort_by_key(|link| link.index); + links +} + +fn import(bytes: &[u8]) -> anyhow::Result { + let directory = tempdir().unwrap(); + let mut target = store(&directory, "target.links"); + import_store(&mut target, &mut &bytes[..])?; + Ok(target) +} + +fn import_error(bytes: &[u8]) -> String { + match import(bytes) { + Ok(_) => panic!("{} should not import", hex(bytes)), + Err(error) => error.to_string(), + } +} + +fn packet(external_references: bool, links: &[(u64, Vec)]) -> Vec { + LinksPacket::pack(external_references, links, true) + .unwrap() + .to_bytes() + .unwrap() +} + +fn doublet(address: u64, source: u64, target: u64) -> (u64, Vec) { + ( + address, + vec![Reference::Internal(source), Reference::Internal(target)], + ) +} + +fn no_names() -> Vec { + packet(true, &[]) +} + +#[test] +fn a_store_exports_to_the_golden_archive() { + let directory = tempdir().unwrap(); + let mut source = store(&directory, "source.links"); + fill(&mut source); + + assert_eq!(hex(&export(&mut source)), hex(&bytes(ARCHIVE))); +} + +#[test] +fn an_imported_archive_restores_links_holes_and_names() { + let directory = tempdir().unwrap(); + let mut source = store(&directory, "source.links"); + fill(&mut source); + let mut target = store(&directory, "target.links"); + + import_store(&mut target, &mut &bytes(ARCHIVE)[..]).unwrap(); + + assert_eq!(links(&mut target), links(&mut source)); + assert!(!target.exists(2), "the hole at 2 stays a hole"); + assert_eq!(names(&mut target), names(&mut source)); + assert_eq!(hex(&export(&mut target)), hex(&bytes(ARCHIVE))); +} + +#[test] +fn an_empty_store_is_two_empty_packets() { + let directory = tempdir().unwrap(); + let mut empty = store(&directory, "empty.links"); + + assert_eq!(hex(&export(&mut empty)), "10 00 11 00"); + let mut imported = import(&bytes("10 00 11 00")).unwrap(); + assert!(imported.all_links().is_empty()); +} + +#[test] +fn an_archive_round_trips_through_a_remote_store() { + let mut server = RunningServer::start(ServerOptions::default(), true); + let mut remote = RemoteLinks::new(server.client(TextLinoProtocol::new())); + + import_store(&mut remote, &mut &bytes(ARCHIVE)[..]).unwrap(); + + assert_eq!(hex(&export(&mut remote)), hex(&bytes(ARCHIVE))); + assert!(!remote.exists(2)); + server.stop(); +} + +#[test] +fn an_archive_imports_into_a_store_that_already_has_the_links() { + let directory = tempdir().unwrap(); + let mut target = store(&directory, "target.links"); + fill(&mut target); + + import_store(&mut target, &mut &bytes(ARCHIVE)[..]).unwrap(); + + assert_eq!(hex(&export(&mut target)), hex(&bytes(ARCHIVE))); +} + +#[test] +fn a_truncated_archive_is_rejected() { + assert!(import_error(&[]).contains("ends before its links")); + assert!(import_error(&bytes("10 00")).contains("ends before its names")); + assert!(import_error(&bytes("10 00 11")).contains("malformed")); +} + +#[test] +fn trailing_bytes_are_rejected() { + assert!(import_error(&bytes("10 00 11 00 10")).contains("trailing bytes")); +} + +#[test] +fn a_link_must_be_a_doublet_of_link_addresses() { + let triplet = ( + 1, + vec![ + Reference::Internal(1), + Reference::Internal(1), + Reference::Internal(1), + ], + ); + let external = (1, vec![Reference::External(1), Reference::Internal(1)]); + for link in [triplet, external] { + let mut archive = packet(true, &[link]); + archive.extend(no_names()); + assert!(import_error(&archive).contains("is not a doublet of link addresses")); + } +} + +#[test] +fn addresses_must_fit_a_32_bit_store() { + let too_large = u64::from(u32::MAX) + 1; + for link in [doublet(too_large, 1, 1), doublet(1, too_large, 1)] { + let mut archive = packet(false, &[link]); + archive.extend(no_names()); + assert!(import_error(&archive).contains("does not fit a 32-bit store")); + } +} + +#[test] +fn names_must_be_an_address_and_code_points() { + let cases = [ + (vec![Reference::Internal(1)], "only external values"), + ( + vec![Reference::External(1), Reference::External(0xD800)], + "invalid code point 55296", + ), + ( + vec![Reference::External(1), Reference::External(1 << 40)], + "invalid code point", + ), + ( + vec![Reference::External(1 << 40)], + "does not fit a 32-bit store", + ), + ]; + for (name, message) in cases { + let mut archive = packet(false, &[doublet(1, 1, 1)]); + archive.extend(packet(true, &[(1, name)])); + let error = import_error(&archive); + assert!(error.contains(message), "{error}"); + } +} + +#[test] +fn an_empty_name_is_kept() { + let mut archive = packet(false, &[doublet(1, 1, 1)]); + archive.extend(packet(true, &[(1, vec![Reference::External(1)])])); + + let mut imported = import(&archive).unwrap(); + + assert_eq!(imported.get_name(1).unwrap().as_deref(), Some("")); + assert_eq!(hex(&export(&mut imported)), hex(&archive)); +} + +#[test] +fn the_cli_parses_the_archive_options_and_their_aliases() { + for (export, import) in [ + ("--export-binary", "--import-binary"), + ("--binary-output", "--binary-input"), + ("--binary-out", "--binary-in"), + ] { + for arguments in [ + vec![ + export.to_string(), + "out.bin".into(), + import.into(), + "in.bin".into(), + ], + vec![format!("{export}=out.bin"), format!("{import}=in.bin")], + ] { + let arguments: Vec<&str> = std::iter::once("clink") + .chain(arguments.iter().map(String::as_str)) + .collect(); + let cli = match Cli::parse_from(&arguments).unwrap() { + CliCommand::Run(cli) => *cli, + other => panic!("expected run command, got {other:?}"), + }; + assert_eq!(cli.binary_output.as_deref(), Some("out.bin")); + assert_eq!(cli.binary_input.as_deref(), Some("in.bin")); + } + } +} + +fn clink(database: &Path, arguments: &[&str]) -> Output { + let output = Command::new(env!("CARGO_BIN_EXE_clink")) + .arg("--db") + .arg(database) + .args(arguments) + .output() + .unwrap(); + assert!( + output.status.success(), + "clink failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + output +} + +#[test] +fn the_cli_copies_a_store_through_an_archive() { + let directory = tempdir().unwrap(); + let archive = directory.path().join("store.bin"); + let source_lino = directory.path().join("source.lino"); + let target_lino = directory.path().join("target.lino"); + let archive_path = archive.to_str().unwrap(); + + clink( + &directory.path().join("source.links"), + &[ + "--auto-create-missing-references", + "() ((child: father mother) (2: 2 1))", + "--export-binary", + archive_path, + "--out", + source_lino.to_str().unwrap(), + ], + ); + clink( + &directory.path().join("target.links"), + &[ + "--import-binary", + archive_path, + "--out", + target_lino.to_str().unwrap(), + ], + ); + + let lino = fs::read_to_string(&target_lino).unwrap(); + assert_eq!(lino, fs::read_to_string(&source_lino).unwrap()); + assert!(lino.contains("(child: father mother)"), "{lino}"); +} + +#[test] +fn the_cli_imports_the_archive_before_the_lino_file_and_the_query() { + let directory = tempdir().unwrap(); + let archive = directory.path().join("store.bin"); + let lino = directory.path().join("more.lino"); + fs::write(&archive, bytes(ARCHIVE)).unwrap(); + fs::write(&lino, "(2: 3 4)\n").unwrap(); + + let output = clink( + &directory.path().join("target.links"), + &[ + "--import-binary", + archive.to_str().unwrap(), + "--in", + lino.to_str().unwrap(), + "((4: 1 3)) ((4: 4 1))", + "--after", + ], + ); + + assert_eq!( + String::from_utf8_lossy(&output.stdout), + "(a: a a)\n(2: é ab)\n(é: é é)\n(ab: ab a)\n" + ); +} + +#[test] +fn the_cli_reports_a_missing_archive() { + let directory = tempdir().unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_clink")) + .arg("--db") + .arg(directory.path().join("target.links")) + .args(["--import-binary", "missing.bin"]) + .output() + .unwrap(); + + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("Failed to read the store archive")); +} From eb134bb41f39a00b07b09060bccc940540b0fb61 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:23:23 +0000 Subject: [PATCH 31/44] feat(csharp): export and import a whole store in binary links notation --- csharp/.changeset/issue-105-store-archive.md | 5 + .../LinoDatabaseInput.cs | 2 +- .../Protocol/StoreArchive.cs | 146 ++++++++++++ .../Protocol/CliStoreArchiveTests.cs | 67 ++++++ .../Protocol/StoreArchiveTests.cs | 218 ++++++++++++++++++ .../Foundation.Data.Doublets.Cli/Program.cs | 67 +++++- 6 files changed, 499 insertions(+), 6 deletions(-) create mode 100644 csharp/.changeset/issue-105-store-archive.md create mode 100644 csharp/Foundation.Data.Doublets.Cli.Library/Protocol/StoreArchive.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliStoreArchiveTests.cs create mode 100644 csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/StoreArchiveTests.cs diff --git a/csharp/.changeset/issue-105-store-archive.md b/csharp/.changeset/issue-105-store-archive.md new file mode 100644 index 0000000..7134878 --- /dev/null +++ b/csharp/.changeset/issue-105-store-archive.md @@ -0,0 +1,5 @@ +--- +'Foundation.Data.Doublets.Cli': minor +--- + +Added `--export-binary ` (aliases `--binary-output`, `--binary-out`) and `--import-binary ` (aliases `--binary-input`, `--binary-in`), which write and read the whole store, names included, as a store archive in binary links notation: a links packet that keeps every address and hole, followed by a names packet. The library exposes the same as `StoreArchive.Export`, `StoreArchive.Import`, `StoreArchive.ExportToFile` and `StoreArchive.ImportFromFile`. A binary archive is imported before the `--in` LiNo file and exported wherever `--out` is written. The archive bytes match the Rust port. diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/LinoDatabaseInput.cs b/csharp/Foundation.Data.Doublets.Cli.Library/LinoDatabaseInput.cs index c6c15a7..d3cb70a 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/LinoDatabaseInput.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/LinoDatabaseInput.cs @@ -58,7 +58,7 @@ private static uint ResolveIndex(INamedTypesLinks links, ImportContext con return EnsureNamedPointLink(links, context, identifier); } - private static void UpdateLink(INamedTypesLinks links, uint index, uint source, uint target) + internal static void UpdateLink(INamedTypesLinks links, uint index, uint source, uint target) { if (!links.Exists(index)) { diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/StoreArchive.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/StoreArchive.cs new file mode 100644 index 0000000..5db4ea4 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/StoreArchive.cs @@ -0,0 +1,146 @@ +using System.Text; +using Platform.Data; +using Platform.Data.Doublets; + +using DoubletLink = Platform.Data.Doublets.Link; + +namespace Foundation.Data.Doublets.Cli.Protocol; + +/// +/// A whole links store in binary links notation: two s, +/// one after the other. +/// +/// +/// +/// Links, without external references: the store link +/// (a: s t) is the doublet s t at address a. Addresses the +/// store does not use are holes, so every link keeps its address. +/// Names, with external references: one link per named store link, +/// in address order, holding the external values address code-point…. +/// +/// Both packets use packed widths, so a store of small addresses costs about +/// two bytes per link. writes every link back at its own +/// address and restores the names, so exporting the imported store gives the +/// same bytes again. The Rust protocol::archive module writes the same bytes. +/// +public static class StoreArchive +{ + /// Writes every link and name of to . + public static void Export(INamedTypesLinks links, Stream output) + { + ArgumentNullException.ThrowIfNull(links); + ArgumentNullException.ThrowIfNull(output); + var any = links.Constants.Any; + var stored = links + .All(new DoubletLink(any, any, any)) + .Select(link => new DoubletLink(link)) + .OrderBy(link => link.Index) + .ToList(); + var doublets = stored + .Select(link => ((ulong)link.Index, new[] + { + PacketReference.Internal(link.Source), + PacketReference.Internal(link.Target), + })) + .ToList(); + var names = new List<(ulong Address, PacketReference[] References)>(); + foreach (var link in stored) + { + var name = links.GetName(link.Index); + if (name is not null) + { + var references = new List { PacketReference.External(link.Index) }; + references.AddRange(name.EnumerateRunes().Select(rune => PacketReference.External((ulong)rune.Value))); + names.Add(((ulong)names.Count + 1, references.ToArray())); + } + } + output.Write(LinksPacket.Pack(false, doublets, true).ToBytes()); + output.Write(LinksPacket.Pack(true, names, true).ToBytes()); + } + + /// Writes the archive of to the file at . + public static void ExportToFile(INamedTypesLinks links, string path) + { + using var output = new MemoryStream(); + Export(links, output); + File.WriteAllBytes(path, output.ToArray()); + } + + /// + /// Reads an archive from into : + /// each link is written at its own address, then every name is set. + /// + public static void Import(INamedTypesLinks links, Stream input) + { + ArgumentNullException.ThrowIfNull(links); + ArgumentNullException.ThrowIfNull(input); + var reader = new LinoStreamReader(input); + var linksPacket = ReadPacket(reader, "links"); + var namesPacket = ReadPacket(reader, "names"); + if (!reader.AtEnd) + { + throw LinoProtocolException.Malformed("trailing bytes after the store archive"); + } + var doublets = linksPacket.Links().Select(link => DecodeDoublet(link.Address, link.References)).ToList(); + var names = namesPacket.Links().Select(link => DecodeName(link.References)).ToList(); + // Every address exists before any link refers to it. + foreach (var (index, _, _) in doublets) + { + if (!links.Exists(index)) + { + LinksExtensions.EnsureCreated(links, index); + } + } + foreach (var (index, source, target) in doublets) + { + LinoDatabaseInput.UpdateLink(links, index, source, target); + } + foreach (var (index, name) in names) + { + links.SetName(index, name); + } + } + + /// Reads the archive in the file at into . + public static void ImportFromFile(INamedTypesLinks links, string path) + { + using var input = File.OpenRead(path); + Import(links, input); + } + + private static LinksPacket ReadPacket(LinoStreamReader reader, string part) => + LinksPacket.ReadFrom(reader, DecodeLimits.Unlimited) + ?? throw LinoProtocolException.Malformed($"the store archive ends before its {part}"); + + private static (uint Index, uint Source, uint Target) DecodeDoublet(ulong address, PacketReference[] references) + { + if (references is not [{ IsExternal: false } source, { IsExternal: false } target]) + { + throw LinoProtocolException.Malformed($"archive link {address} is not a doublet of link addresses"); + } + return (StoreAddress(address), StoreAddress(source.Value), StoreAddress(target.Value)); + } + + private static (uint Index, string Name) DecodeName(PacketReference[] references) + { + if (references.Any(reference => !reference.IsExternal)) + { + throw LinoProtocolException.Malformed("an archive name holds only external values"); + } + var name = new StringBuilder(); + foreach (var codePoint in references.Skip(1).Select(reference => reference.Value)) + { + if (codePoint > int.MaxValue || !Rune.IsValid((int)codePoint)) + { + throw LinoProtocolException.Malformed($"invalid code point {codePoint}"); + } + name.Append(new Rune((int)codePoint).ToString()); + } + return (StoreAddress(references[0].Value), name.ToString()); + } + + private static uint StoreAddress(ulong address) => + address <= uint.MaxValue + ? (uint)address + : throw LinoProtocolException.Malformed($"address {address} does not fit a 32-bit store"); +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliStoreArchiveTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliStoreArchiveTests.cs new file mode 100644 index 0000000..216ebd0 --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/CliStoreArchiveTests.cs @@ -0,0 +1,67 @@ +namespace Foundation.Data.Doublets.Cli.Tests.Protocol; + +/// --export-binary and --import-binary copy a whole store through a store archive. +public sealed class CliStoreArchiveTests : IDisposable +{ + private readonly string _directory = Directory.CreateTempSubdirectory("clink-archive-").FullName; + + public void Dispose() => Directory.Delete(_directory, recursive: true); + + private string PathOf(string name) => Path.Combine(_directory, name); + + [Theory] + [InlineData("--export-binary", "--import-binary")] + [InlineData("--binary-output", "--binary-input")] + [InlineData("--binary-out", "--binary-in")] + public async Task TheCliCopiesAStoreThroughAnArchive(string export, string import) + { + var archive = PathOf("store.bin"); + Clink.AssertSucceeded(await Clink.RunAsync( + "--db", PathOf("source.links"), + "--auto-create-missing-references", + "() ((child: father mother) (2: 2 1))", + export, archive, + "--out", PathOf("source.lino"))); + + Clink.AssertSucceeded(await Clink.RunAsync( + "--db", PathOf("target.links"), + import, archive, + "--out", PathOf("target.lino"))); + + var lino = File.ReadAllText(PathOf("target.lino")); + Assert.Equal(File.ReadAllText(PathOf("source.lino")), lino); + Assert.Contains("(child: father mother)", lino); + } + + [Fact] + public async Task TheCliImportsTheArchiveBeforeTheLinoFileAndTheQuery() + { + var archive = PathOf("store.bin"); + await File.WriteAllBytesAsync(archive, Convert.FromHexString( + "12 02 20 01 24 01 02 01 01 03 03 01 03 13 02 21 02 30 01 ff ff 9f ff fd ff 17 ff fc 9f 9e".Replace(" ", ""))); + await File.WriteAllTextAsync(PathOf("more.lino"), "(2: 3 4)\n"); + + var result = await Clink.RunAsync( + "--db", PathOf("target.links"), + "--import-binary", archive, + "--in", PathOf("more.lino"), + "((4: 1 3)) ((4: 4 1))", + "--after"); + + Clink.AssertSucceeded(result); + Assert.Equal("(a: a a)\n(2: é ab)\n(é: é é)\n(ab: ab a)\n", result.Stdout); + } + + [Fact] + public async Task TheCliReportsAMalformedArchive() + { + var archive = PathOf("broken.bin"); + await File.WriteAllBytesAsync(archive, new byte[] { 0x10, 0x00 }); + + var result = await Clink.RunAsync("--db", PathOf("target.links"), "--import-binary", archive); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("Error reading binary store archive", result.Stderr); + Assert.Contains("ends before its names", result.Stderr); + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/StoreArchiveTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/StoreArchiveTests.cs new file mode 100644 index 0000000..d5e5c8e --- /dev/null +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/StoreArchiveTests.cs @@ -0,0 +1,218 @@ +using Foundation.Data.Doublets.Cli.Protocol; +using Platform.Data; +using Platform.Data.Doublets; + +using DoubletLink = Platform.Data.Doublets.Link; + +namespace Foundation.Data.Doublets.Cli.Tests.Protocol; + +/// +/// A whole store survives --export-binary and --import-binary: links keep their addresses, holes stay +/// holes, names come back, and the archive bytes match the Rust port. +/// +public sealed class StoreArchiveTests +{ + /// + /// The archive of , byte for byte; the Rust store_archive_tests assert the same bytes. + /// + /// + /// Links packet, 13 bytes: 12 explicit layout, 02 sections; 20 01 one doublet of 1-byte + /// references, 01 01; 24 01 02 a section after a gap of one address (the hole at 2) with two + /// doublets, 03 03 01 03. Names packet, 17 bytes: 13 explicit layout with external references, + /// 02 sections; 21 02 two links of two 2-byte references, ff ff ff 9f (1, a) and + /// ff fd ff 17 (3, é); 30 01 one link of three 1-byte references, fc 9f 9e + /// (4, a, b). + /// + private const string Archive = + "12 02 20 01 24 01 02 01 01 03 03 01 03 13 02 21 02 30 01 ff ff 9f ff fd ff 17 ff fc 9f 9e"; + + private sealed class LocalStore : IDisposable + { + private readonly string _databaseFilename = Path.GetTempFileName(); + + public LocalStore() => Links = new NamedTypesDecorator(_databaseFilename); + + public NamedTypesDecorator Links { get; } + + public void Dispose() + { + Links.Dispose(); + File.Delete(_databaseFilename); + File.Delete(NamedTypesDecorator.MakeNamesDatabaseFilename(_databaseFilename)); + } + } + + /// Links 1: 1 1 named a, 3: 3 3 named é and 4: 1 3 named ab, with a hole at 2. + private static void Fill(INamedTypesLinks links) + { + for (var i = 0; i < 3; i++) + { + var id = links.Create(); + links.Update(id, id, id); + } + var last = links.Create(); + Assert.Equal(4u, last); + links.Update(last, 1u, 3u); + links.Delete(2u); + links.SetName(1, "a"); + links.SetName(3, "é"); + links.SetName(4, "ab"); + } + + private static byte[] Export(INamedTypesLinks links) + { + using var output = new MemoryStream(); + StoreArchive.Export(links, output); + return output.ToArray(); + } + + private static void Import(INamedTypesLinks links, byte[] archive) => + StoreArchive.Import(links, new MemoryStream(archive)); + + private static string Hex(byte[] bytes) => string.Join(" ", bytes.Select(value => value.ToString("x2"))); + + private static byte[] Bytes(string hex) => + hex.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(value => Convert.ToByte(value, 16)).ToArray(); + + private static List AllLinks(INamedTypesLinks links) => + links.All(new DoubletLink(links.Constants.Any, links.Constants.Any, links.Constants.Any)) + .Select(link => new DoubletLink(link)) + .OrderBy(link => link.Index) + .ToList(); + + private static List Names(INamedTypesLinks links) => + Enumerable.Range(1, 4).Select(id => links.GetName((uint)id)).ToList(); + + private static byte[] Packet(bool externalReferences, params (ulong Address, PacketReference[] References)[] links) => + LinksPacket.Pack(externalReferences, links, true).ToBytes(); + + private static (ulong, PacketReference[]) Doublet(ulong address, ulong source, ulong target) => + (address, new[] { PacketReference.Internal(source), PacketReference.Internal(target) }); + + private static string ImportError(byte[] archive) + { + using var store = new LocalStore(); + return Assert.Throws(() => Import(store.Links, archive)).Message; + } + + [Fact] + public void AStoreExportsToTheGoldenArchive() + { + using var source = new LocalStore(); + Fill(source.Links); + + Assert.Equal(Archive, Hex(Export(source.Links))); + } + + [Fact] + public void AnImportedArchiveRestoresLinksHolesAndNames() + { + using var source = new LocalStore(); + Fill(source.Links); + using var target = new LocalStore(); + + Import(target.Links, Bytes(Archive)); + + Assert.Equal(AllLinks(source.Links), AllLinks(target.Links)); + Assert.False(target.Links.Exists(2u), "the hole at 2 stays a hole"); + Assert.Equal(Names(source.Links), Names(target.Links)); + Assert.Equal(Archive, Hex(Export(target.Links))); + } + + [Fact] + public void AnEmptyStoreIsTwoEmptyPackets() + { + using var empty = new LocalStore(); + Assert.Equal("10 00 11 00", Hex(Export(empty.Links))); + + Import(empty.Links, Bytes("10 00 11 00")); + Assert.Empty(AllLinks(empty.Links)); + } + + [Fact] + public void AnArchiveRoundTripsThroughARemoteStore() + { + using var server = new RunningServer(); + using var remote = server.Remote(new TextLinoProtocol()); + + Import(remote, Bytes(Archive)); + + Assert.Equal(Archive, Hex(Export(remote))); + Assert.False(remote.Exists(2u)); + } + + [Fact] + public void AnArchiveImportsIntoAStoreThatAlreadyHasTheLinks() + { + using var target = new LocalStore(); + Fill(target.Links); + + Import(target.Links, Bytes(Archive)); + + Assert.Equal(Archive, Hex(Export(target.Links))); + } + + [Fact] + public void ATruncatedArchiveIsRejected() + { + Assert.Contains("ends before its links", ImportError(Array.Empty())); + Assert.Contains("ends before its names", ImportError(Bytes("10 00"))); + Assert.Contains("malformed", ImportError(Bytes("10 00 11"))); + } + + [Fact] + public void TrailingBytesAreRejected() => + Assert.Contains("trailing bytes", ImportError(Bytes("10 00 11 00 10"))); + + [Fact] + public void ALinkMustBeADoubletOfLinkAddresses() + { + var triplet = (1UL, new[] { PacketReference.Internal(1), PacketReference.Internal(1), PacketReference.Internal(1) }); + var external = (1UL, new[] { PacketReference.External(1), PacketReference.Internal(1) }); + foreach (var link in new[] { triplet, external }) + { + var archive = Packet(true, link).Concat(Packet(true)).ToArray(); + Assert.Contains("is not a doublet of link addresses", ImportError(archive)); + } + } + + [Fact] + public void AddressesMustFitA32BitStore() + { + var tooLarge = (ulong)uint.MaxValue + 1; + foreach (var link in new[] { Doublet(tooLarge, 1, 1), Doublet(1, tooLarge, 1) }) + { + var archive = Packet(false, link).Concat(Packet(true)).ToArray(); + Assert.Contains("does not fit a 32-bit store", ImportError(archive)); + } + } + + [Theory] + [InlineData(false, 0UL, "only external values")] + [InlineData(true, 0xD800UL, "invalid code point 55296")] + [InlineData(true, 1UL << 40, "invalid code point")] + [InlineData(true, null, "does not fit a 32-bit store")] + public void NamesMustBeAnAddressAndCodePoints(bool external, ulong? codePoint, string message) + { + var name = !external + ? new[] { PacketReference.Internal(1) } + : codePoint is { } value + ? new[] { PacketReference.External(1), PacketReference.External(value) } + : new[] { PacketReference.External(1UL << 40) }; + var archive = Packet(false, Doublet(1, 1, 1)).Concat(Packet(true, (1UL, name))).ToArray(); + + Assert.Contains(message, ImportError(archive)); + } + + [Fact] + public void AnEmptyNameIsKept() + { + var archive = Packet(false, Doublet(1, 1, 1)).Concat(Packet(true, (1UL, new[] { PacketReference.External(1) }))).ToArray(); + using var store = new LocalStore(); + + Import(store.Links, archive); + + Assert.Equal("", store.Links.GetName(1)); + Assert.Equal(Hex(archive), Hex(Export(store.Links))); + } +} diff --git a/csharp/Foundation.Data.Doublets.Cli/Program.cs b/csharp/Foundation.Data.Doublets.Cli/Program.cs index 039de4d..97859c7 100644 --- a/csharp/Foundation.Data.Doublets.Cli/Program.cs +++ b/csharp/Foundation.Data.Doublets.Cli/Program.cs @@ -108,6 +108,16 @@ Description = "Path to read and import a LiNo file into the database" }; +var binaryInputOption = new Option("--import-binary", "--binary-input", "--binary-in") +{ + Description = "Path to read and import a store archive in binary links notation" +}; + +var binaryOutputOption = new Option("--export-binary", "--binary-output", "--binary-out") +{ + Description = "Path to write the complete database, names included, as a store archive in binary links notation" +}; + var transactionsOption = new Option("--transactions") { Description = "Enable the transactions layer (default log path: .transitions.links)", @@ -228,6 +238,8 @@ rootCommand.Options.Add(embedTriggersOption); rootCommand.Options.Add(inputOption); rootCommand.Options.Add(outputOption); +rootCommand.Options.Add(binaryInputOption); +rootCommand.Options.Add(binaryOutputOption); rootCommand.Options.Add(transactionsOption); rootCommand.Options.Add(transactionsFileOption); rootCommand.Options.Add(commitModeOption); @@ -268,6 +280,8 @@ var embedTriggers = parseResult.GetValue(embedTriggersOption); var inputPath = parseResult.GetValue(inputOption); var outputPath = parseResult.GetValue(outputOption); + var binaryInputPath = parseResult.GetValue(binaryInputOption); + var binaryOutputPath = parseResult.GetValue(binaryOutputOption); var transactionsFlag = parseResult.GetValue(transactionsOption); var transactionsPathRaw = parseResult.GetValue(transactionsFileOption); var commitModeRaw = parseResult.GetValue(commitModeOption); @@ -620,7 +634,7 @@ int RunQueryPipeline() PrintAllLinks(decoratedLinks); } - if (!TryReadLinoInput(decoratedLinks, inputPath)) + if (!TryReadBinaryInput(decoratedLinks, binaryInputPath) || !TryReadLinoInput(decoratedLinks, inputPath)) { return 1; } @@ -639,7 +653,7 @@ int RunQueryPipeline() return 1; } - return TryWriteLinoOutput(decoratedLinks, outputPath) ? 0 : 1; + return TryWriteOutputs(decoratedLinks, outputPath, binaryOutputPath) ? 0 : 1; } var effectiveQuery = !string.IsNullOrWhiteSpace(queryOptionValue) ? queryOptionValue : queryArgumentValue; @@ -655,14 +669,14 @@ int RunQueryPipeline() var kind = always ? PersistentTransformationKind.Always : PersistentTransformationKind.Once; var trigger = persistentLinks.StoreTrigger(kind, effectiveQuery); Console.WriteLine($"{kind} persistent transformation trigger stored: {trigger}"); - return TryWriteLinoOutput(decoratedLinks, outputPath) ? 0 : 1; + return TryWriteOutputs(decoratedLinks, outputPath, binaryOutputPath) ? 0 : 1; } if (persistentLinks is not null && never) { var removed = persistentLinks.RemoveTriggers(effectiveQuery); Console.WriteLine($"Persistent transformation triggers removed: {removed}"); - return TryWriteLinoOutput(decoratedLinks, outputPath) ? 0 : 1; + return TryWriteOutputs(decoratedLinks, outputPath, binaryOutputPath) ? 0 : 1; } var changesList = new List<(DoubletLink Before, DoubletLink After)>(); @@ -728,7 +742,7 @@ int RunQueryPipeline() PrintAllLinks(decoratedLinks); } - return TryWriteLinoOutput(decoratedLinks, outputPath) ? 0 : 1; + return TryWriteOutputs(decoratedLinks, outputPath, binaryOutputPath) ? 0 : 1; } } ); @@ -785,6 +799,49 @@ static void PrintChange(INamedTypesLinks links, DoubletLink linkBefore, Do Console.WriteLine(LinoDatabaseOutput.FormatChange(links, linkBefore, linkAfter)); } +static bool TryWriteOutputs(INamedTypesLinks links, string? outputPath, string? binaryOutputPath) +{ + return TryWriteLinoOutput(links, outputPath) && TryWriteBinaryOutput(links, binaryOutputPath); +} + +static bool TryWriteBinaryOutput(INamedTypesLinks links, string? outputPath) +{ + if (string.IsNullOrWhiteSpace(outputPath)) + { + return true; + } + + try + { + StoreArchive.ExportToFile(links, outputPath); + return true; + } + catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException || ex is ArgumentException || ex is NotSupportedException || ex is LinoProtocolException) + { + Console.Error.WriteLine($"Error writing binary store archive '{outputPath}': {ex.Message}"); + return false; + } +} + +static bool TryReadBinaryInput(INamedTypesLinks links, string? inputPath) +{ + if (string.IsNullOrWhiteSpace(inputPath)) + { + return true; + } + + try + { + StoreArchive.ImportFromFile(links, inputPath); + return true; + } + catch (Exception ex) when (ex is IOException || ex is UnauthorizedAccessException || ex is ArgumentException || ex is NotSupportedException || ex is LinoProtocolException || ex is InvalidOperationException) + { + Console.Error.WriteLine($"Error reading binary store archive '{inputPath}': {ex.Message}"); + return false; + } +} + static bool TryWriteLinoOutput(INamedTypesLinks links, string? outputPath) { if (string.IsNullOrWhiteSpace(outputPath)) From 2d5ac41eced0376d7a1a73b0f7ad9835a1510196 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:27:24 +0000 Subject: [PATCH 32/44] docs: describe the binary store archive and add a cross-port example Adds section 10 of the binary links notation specification with the golden archive decoded byte by byte, the --import-binary/--export-binary options in the README, HOW-IT-WORKS, REQUIREMENTS and the issue-105 case study, and examples/archive/run-interop.sh, which copies a store from one port to the other through an archive. --- README.md | 21 ++++++++++ docs/HOW-IT-WORKS.md | 11 +++++ docs/REQUIREMENTS.md | 1 + docs/case-studies/issue-105/README.md | 22 ++++++++++ docs/protocol/binary-links-notation.md | 56 +++++++++++++++++++++++++- examples/archive/README.md | 31 ++++++++++++++ examples/archive/run-interop.sh | 37 +++++++++++++++++ 7 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 examples/archive/README.md create mode 100755 examples/archive/run-interop.sh diff --git a/README.md b/README.md index fc0ca89..3b74a17 100644 --- a/README.md +++ b/README.md @@ -351,6 +351,25 @@ When links do not have names, exported references are plain link numbers: (2: 1 2) ``` +## Export and import a binary store archive + +Use `--export-binary` (or `--binary-output`, `--binary-out`) to write the +complete database, names included, as a store archive in +[binary links notation](docs/protocol/binary-links-notation.md#10-store-archive). +Every link keeps its address, holes included. `--import-binary` (or +`--binary-input`, `--binary-in`) reads an archive back before `--in` and the +query run: + +```bash +clink --db family.links --export-binary family.bin +clink --db copy.links --import-binary family.bin --after +``` + +For the family above the archive takes 36 bytes, against 71 bytes of LiNo. +Both ports write and read the same bytes, so an archive written by the Rust +CLI can be read by the C# CLI and the other way around +([examples/archive](examples/archive/README.md)). + ## Persistent transformation triggers Store a query as a trigger with `--always` to apply it after later write operations: @@ -651,6 +670,8 @@ Both the C# NuGet tool and the Rust CLI support every option below. | `--after` | bool | `false` | `--links`, `-a` | Print the state of the database after applying changes | | `--in` | string | _None_ | `--import`, `--lino-input` | Read and import a LiNo file before query execution | | `--out` | string | _None_ | `--export`, `--lino-output` | Write the complete database as a LiNo file | +| `--import-binary` | string | _None_ | `--binary-input`, `--binary-in` | Read and import a store archive in binary links notation before `--in` | +| `--export-binary` | string | _None_ | `--binary-output`, `--binary-out` | Write the complete database, names included, as a binary store archive | | `--always` | bool | `false` | _None_ | Store the query as an always-on persistent transformation trigger | | `--once` | bool | `false` | _None_ | Store the query as a one-shot persistent transformation trigger | | `--never` | bool | `false` | _None_ | Remove stored persistent transformation triggers matching the query | diff --git a/docs/HOW-IT-WORKS.md b/docs/HOW-IT-WORKS.md index ee61342..cf9ceb0 100644 --- a/docs/HOW-IT-WORKS.md +++ b/docs/HOW-IT-WORKS.md @@ -154,6 +154,16 @@ Unnamed links are exported with numeric references: (2: 1 2) ``` +## Binary Store Archive + +`--export-binary` writes the complete database wherever `--out` would be +written, and `--import-binary` reads one back before `--in`. An archive is two +[binary links notation](protocol/binary-links-notation.md#10-store-archive) +packets: the links, each at its own address with unused addresses left as +holes, and then the names as code points. Import validates the whole archive +before it writes, then recreates every address, link and name, so exporting +the imported store gives the same bytes again. + ## Structure Formatting `--structure ` formats one link by recursively expanding the left branch. @@ -358,4 +368,5 @@ Common failures include: - Missing references without `--auto-create-missing-references`. - `--structure` requested for a link that does not exist. - Import lines that are not two-value link definitions. +- Store archives that are cut short or are not doublets of link addresses. - Multiple trigger commands such as `--always` and `--once` in one C# command. diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index 0b5ec1c..1c0e3a3 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -41,6 +41,7 @@ the core CLI behavior and is also used by the WebAssembly browser workbench. | Deduplicate identical links and sub-links. | #65, #66 | Creation reuses existing `(source target)` pairs instead of storing duplicates. | | Export the database as LiNo. | #8, #24, #44, #54 | `--out`, `--export`, and `--lino-output` write a complete LiNo file. | | Import a database from LiNo. | #25, #43 | `--in`, `--import`, and `--lino-input` read a LiNo file before query execution. | +| Export and import the whole store in binary links notation. | #105 | `--export-binary` (`--binary-output`, `--binary-out`) writes a [store archive](protocol/binary-links-notation.md#10-store-archive) of every link at its own address plus every name, and `--import-binary` (`--binary-input`, `--binary-in`) reads one before `--in`. Rust `export_store`/`import_store` and C# `StoreArchive.Export`/`Import` work on any named store, a remote one included, and both ports write identical bytes. | | Format a link structure. | #19, #48 | `--structure` recursively formats the left branch with indexes preserved. | | Store persistent transformations. | #3, #55 | C# supports `--always`, `--once`, `--never`, `--triggers`, `--triggers-file`, and `--embed-triggers`. | | Optional transactions layer. | #94 | C# and Rust expose `--transactions`, `--transactions-file`, `--commit-mode`, `--retention`, and `--log`. Each Create/Update/Delete is recorded as one or more reversible transitions in a doublets-store sidecar; explicit `BeginTransaction()` / `Commit()` / `Rollback()` APIs are available in both libraries. Three retention policies are supported: `infinite`, `sized:`, and `chunked::`. Crash recovery replays committed transitions on the next open. When no flag is passed, behaviour is identical to the bare CLI (no sidecar, no cost). | diff --git a/docs/case-studies/issue-105/README.md b/docs/case-studies/issue-105/README.md index 437c1fd..76d481d 100644 --- a/docs/case-studies/issue-105/README.md +++ b/docs/case-studies/issue-105/README.md @@ -268,6 +268,25 @@ On the client, any binary option implies `--protocol binary`, and `--protocol text` together with a binary option is an error. Using `--serve` and `--connect` together is also an error. +### 4.7 Store archive + +The review of PR #106 asked for a full import and export of a store in +binary links notation. A store archive is two packets: the links, each at its +own address with holes for unused ones, and then the names as external code +points. Both use packed widths, so a store of small addresses costs about two +bytes per link +([specification](../../protocol/binary-links-notation.md#10-store-archive)). + +```bash +clink --db family.links --export-binary family.bin # also --binary-output, --binary-out +clink --db copy.links --import-binary family.bin # also --binary-input, --binary-in +``` + +The archive works on the links interface only, so it exports and imports a +remote store (`RemoteLinks`) as well as a local one. Import validates the +whole archive before it writes, then recreates every address, link and name. +Exporting the imported store gives the same bytes again. + ## 5. Verification ### Golden vectors @@ -318,13 +337,16 @@ parsing and has a fixed structure, not that it is smaller. - `rust/tests/protocol_tcp_tests.rs`: CRUD over every protocol, mixed clients, errors, CRLF, malformed input, protocol restriction, concurrency and shutdown. - `rust/tests/cli_tcp_tests.rs`: `clink --serve` / `--connect` end to end. - `rust/tests/remote_links_tests.rs`: the same proof for the Rust `RemoteLinks`, including the shared conversation. + - `rust/tests/store_archive_tests.rs`: the golden archive, a round trip of links, holes and names, a remote store, malformed archives, and the CLI options. - C#: - `BinaryLinksNotationTests`: a port of `protocol_packet_tests.rs`, checked against the same golden vectors. - `LinoProtocolCodecTests`: the same corpus under all 12 option sets, and the text framing. - `LinksServerTests`: CRUD over every protocol, identical text and binary replies, errors, CRLF, malformed input, protocol restriction, concurrent clients, shutdown, and shutdown racing a new connection. - `RemoteLinksTests`: every `INamedTypesLinks` call, the query processor and the raw `ILinks` interface give the same answers on a local store and over every protocol. It also covers the shared conversation, malformed operations and replies, and lost connections. - `CliTcpIntegrationTests`: `clink --serve` / `--connect` end to end, including option validation. + - `StoreArchiveTests` and `CliStoreArchiveTests`: ports of `store_archive_tests.rs`, checked against the same golden archive. - Cross-language interop: [`examples/tcp/run-interop.sh`](../../../examples/tcp/run-interop.sh) runs a Rust server with a C# client, and a C# server with a Rust client. It covers text and every binary option, and both directions print identical results. +- Store archive interop: [`examples/archive/run-interop.sh`](../../../examples/archive/run-interop.sh) exports a store with one port and imports it with the other, in both directions, and checks that the stores are equal. ## 6. Findings along the way diff --git a/docs/protocol/binary-links-notation.md b/docs/protocol/binary-links-notation.md index 6fc55d5..f19111a 100644 --- a/docs/protocol/binary-links-notation.md +++ b/docs/protocol/binary-links-notation.md @@ -9,8 +9,8 @@ holds golden vectors. Both test suites check them, so both ports write and read exactly these bytes. The packet layer knows nothing about LiNo. The binary LiNo protocol (§7) is -one user of it. Any other links data can use it as well, for example a dump -of a links store. +one user of it, and the store archive (§10), a dump of a whole links store, +is another. ## 1. Conventions @@ -298,3 +298,55 @@ references, a wider arity and packed widths are the options that make a packet small. Packed widths matter when a few large values would otherwise widen every reference: in `() ((1000 70000))`, only the link holding 70 000 needs 4 bytes. + +## 10. Store archive + +A store archive holds a whole doublets store, names included, as two packets +written one after the other: + +1. **Links**, without external references and with packed widths. The store + link `(a: s t)` is the doublet `s t` at address `a`. Addresses the store + does not use are holes (section gaps), so every link keeps its address. +2. **Names**, with external references and with packed widths. One link per + named store link, in address order and from address 1, holds the external + values `address code-point…`. An empty name is a link of arity 1. + +An empty store is `10 00 11 00`. + +Importing reads both packets without decode limits and rejects: + +- an archive that is cut short or has bytes after its second packet; +- a link that is not a doublet of internal references; +- an address that does not fit a 32-bit store; +- a name that holds an internal reference or an invalid code point. + +Only then does it touch the store: it creates every missing address, writes +each link at its own address, and sets every name. So exporting the imported +store gives the same bytes again. + +The golden archive of `rust/tests/store_archive_tests.rs` and +`StoreArchiveTests.cs` holds `(a: a a)`, `(é: é é)` at address 3 and +`(ab: a é)` at address 4, with a hole at 2. It is +`12 02 20 01 24 01 02 01 01 03 03 01 03 13 02 21 02 30 01 ff ff 9f ff fd ff 17 ff fc 9f 9e`: + +| Bytes | Meaning | +|---|---| +| `12 02` | links packet: explicit layout, two sections | +| `20 01` | min 2, 1-byte references; 1 link (address 1) | +| `24 01 02` | min 2, gap 1, 1 byte; 2 links (addresses 3 and 4) | +| `01 01` `03 03` `01 03` | links 1, 3 and 4 | +| `13 02` | names packet: explicit layout, external references, two sections | +| `21 02` | min 2, 2-byte references; 2 links | +| `30 01` | min 3, 1-byte references; 1 link | +| `ff ff` `9f ff` | `#1 #97`: link 1 is `a` | +| `fd ff` `17 ff` | `#3 #233`: link 3 is `é` | +| `fc 9f 9e` | `#4 #97 #98`: link 4 is `ab` | + +The CLI writes an archive with `--export-binary` (or `--binary-output`, +`--binary-out`) wherever it writes `--out`, and reads one with +`--import-binary` (or `--binary-input`, `--binary-in`) before `--in` and the +query. The libraries expose the same operations as `export_store`, +`export_store_file`, `import_store` and `import_store_file` in Rust and +`StoreArchive.Export`, `ExportToFile`, `Import` and `ImportFromFile` in C#. +[`examples/archive`](../../examples/archive/README.md) copies a store from one +port to the other through an archive. diff --git a/examples/archive/README.md b/examples/archive/README.md new file mode 100644 index 0000000..4bf7adc --- /dev/null +++ b/examples/archive/README.md @@ -0,0 +1,31 @@ +# Store archive interop + +`run-interop.sh` builds both ports, fills a store with one of them, writes it +with `--export-binary` and reads it back with the other port's +`--import-binary`. The two stores must print the same LiNo, so the script ends +with an empty `diff`: + +```bash +./examples/archive/run-interop.sh # Rust exports, C# imports +./examples/archive/run-interop.sh csharp # C# exports, Rust imports +``` + +Both directions print the same 41-byte archive for a store whose LiNo export +is 80 bytes: + +``` +archive: 41 bytes, LiNo: 80 bytes + 12 02 20 03 24 01 01 01 01 02 02 01 02 05 05 13 + 02 70 02 60 01 ff 9a 9f 8c 98 9b 8e fe 93 91 8c + 98 9b 8e fd 9d 98 97 94 9c +(father: father father) +(mother: mother mother) +(child: father mother) +(5: 5 5) +``` + +The first 15 bytes are the links packet: links 1 to 3, then a gap of one +address (the hole at 4) and link 5. The rest is the names packet: `father` +and `mother` (seven references each), then `child` (six). The format is +described in +[docs/protocol/binary-links-notation.md](../../docs/protocol/binary-links-notation.md#10-store-archive). diff --git a/examples/archive/run-interop.sh b/examples/archive/run-interop.sh new file mode 100755 index 0000000..faf4bf3 --- /dev/null +++ b/examples/archive/run-interop.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Exports a store with one port as a binary store archive, imports it with the +# other, and checks that both stores print the same LiNo (issue #105). +# +# Usage: +# ./examples/archive/run-interop.sh # Rust exports, C# imports +# ./examples/archive/run-interop.sh csharp # C# exports, Rust imports + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT + +cargo build --manifest-path "$repo_root/rust/Cargo.toml" --quiet +dotnet build "$repo_root/csharp/Foundation.Data.Doublets.Cli" --nologo --verbosity quiet >/dev/null +rust_clink=("$repo_root/rust/target/debug/clink") +csharp_clink=(dotnet "$repo_root/csharp/Foundation.Data.Doublets.Cli/bin/Debug/net10.0/clink.dll") + +if [[ "${1:-rust}" == "csharp" ]]; then + exporter=("${csharp_clink[@]}") importer=("${rust_clink[@]}") +else + exporter=("${rust_clink[@]}") importer=("${csharp_clink[@]}") +fi + +# Named links, an unnamed one, and a hole at 4. +"${exporter[@]}" --db "$work_dir/source.links" --auto-create-missing-references \ + "() ((child: father mother) (4 4) (5 5))" >/dev/null +"${exporter[@]}" --db "$work_dir/source.links" "((4 4)) ()" \ + --export-binary "$work_dir/store.bin" --out "$work_dir/source.lino" >/dev/null +"${importer[@]}" --db "$work_dir/target.links" \ + --import-binary "$work_dir/store.bin" --out "$work_dir/target.lino" >/dev/null + +echo "archive: $(wc -c <"$work_dir/store.bin") bytes, LiNo: $(wc -c <"$work_dir/source.lino") bytes" +od -An -tx1 "$work_dir/store.bin" +diff "$work_dir/source.lino" "$work_dir/target.lino" +cat "$work_dir/target.lino" From f8571088e91e90b82e51dc29e09e177aba80898e Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:36:32 +0000 Subject: [PATCH 33/44] chore: update every dependency to its latest release - Rust: thiserror 2.0.21; wasm-bindgen 0.2.129, wasm-bindgen-test 0.3.79 and web-sys 0.3.106 in rust/wasm; both Cargo.lock files refreshed. - C#: System.CommandLine 2.0.12, Microsoft.NET.Test.Sdk 18.10.1, coverlet.collector 10.1.0. - js: lucide-react 1.52.0, react and react-dom 19.3.0, vite 8.3.2. - Workflows: actions/checkout v7, actions/cache v6, actions/setup-dotnet v6, actions/setup-node v7, and Node.js 24, the current LTS (Node.js 20 reached its end of life in April 2026). --- .github/workflows/csharp.yml | 50 +-- .github/workflows/docs.yml | 10 +- .github/workflows/links.yml | 2 +- .github/workflows/rust.yml | 30 +- .github/workflows/security.yml | 20 +- .github/workflows/wasm.yml | 8 +- .../issue-105-latest-dependencies.md | 5 + .../Foundation.Data.Doublets.Cli.Tests.csproj | 4 +- .../Foundation.Data.Doublets.Cli.csproj | 2 +- js/package-lock.json | 228 ++++++----- js/package.json | 8 +- rust/Cargo.lock | 360 +++--------------- rust/Cargo.toml | 2 +- ...05_180000_issue_105_latest_dependencies.md | 5 + rust/wasm/Cargo.lock | 134 ++++--- rust/wasm/Cargo.toml | 6 +- 16 files changed, 333 insertions(+), 541 deletions(-) create mode 100644 csharp/.changeset/issue-105-latest-dependencies.md create mode 100644 rust/changelog.d/20261005_180000_issue_105_latest_dependencies.md diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index c75e705..67786c5 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -71,14 +71,14 @@ jobs: any-code-changed: ${{ steps.changes.outputs.any-code-changed }} csharp-code-changed: ${{ steps.changes.outputs.csharp-code-changed }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: - node-version: '20.x' + node-version: '24.x' - name: Detect changes id: changes @@ -97,14 +97,14 @@ jobs: needs: [detect-changes] if: github.event_name == 'pull_request' && needs.detect-changes.outputs.csharp-code-changed == 'true' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: - node-version: '20.x' + node-version: '24.x' - name: Validate changeset working-directory: . @@ -146,17 +146,17 @@ jobs: needs.detect-changes.outputs.workflow-changed == 'true' ) steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup .NET - uses: actions/setup-dotnet@v5 + uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: - node-version: '20.x' + node-version: '24.x' - name: Run release script tests working-directory: . @@ -191,10 +191,10 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup .NET - uses: actions/setup-dotnet@v5 + uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' @@ -225,12 +225,12 @@ jobs: if: github.event_name == 'pull_request' needs: [detect-changes] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Setup .NET - uses: actions/setup-dotnet@v5 + uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' @@ -253,10 +253,10 @@ jobs: needs: [lint, test] if: ${{ !cancelled() && needs.lint.result == 'success' && needs.test.result == 'success' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup .NET - uses: actions/setup-dotnet@v5 + uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' @@ -291,19 +291,19 @@ jobs: contents: write packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Setup .NET - uses: actions/setup-dotnet@v5 + uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: - node-version: '20.x' + node-version: '24.x' - name: Check for changesets id: check_changesets @@ -479,20 +479,20 @@ jobs: contents: write packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} - name: Setup .NET - uses: actions/setup-dotnet@v5 + uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: - node-version: '20.x' + node-version: '24.x' - name: Version and commit id: version @@ -603,7 +603,7 @@ jobs: contents: write pull-requests: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6327eb7..ea7ab92 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -59,10 +59,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup .NET - uses: actions/setup-dotnet@v5 + uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' @@ -75,14 +75,14 @@ jobs: run: cargo install wasm-pack --version 0.14.0 --locked - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: - node-version: '20.x' + node-version: '24.x' cache: npm cache-dependency-path: js/package-lock.json - name: Cache cargo registry - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: | ~/.cargo/registry diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index e2d868e..6291945 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -40,7 +40,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Check links with lychee id: lychee diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4f1f876..26d058f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -66,7 +66,7 @@ jobs: any-code-changed: ${{ steps.changes.outputs.any-code-changed }} rust-code-changed: ${{ steps.changes.outputs.rust-code-changed }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 @@ -92,7 +92,7 @@ jobs: needs: [detect-changes] if: github.event_name == 'pull_request' && needs.detect-changes.outputs.rust-code-changed == 'true' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 @@ -125,7 +125,7 @@ jobs: needs.detect-changes.outputs.workflow-changed == 'true' ) steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup Rust uses: dtolnay/rust-toolchain@stable @@ -136,7 +136,7 @@ jobs: run: cargo install rust-script - name: Cache cargo registry - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: | ~/.cargo/registry @@ -172,9 +172,9 @@ jobs: cargo metadata --locked --format-version 1 --manifest-path rust/wasm/Cargo.toml > /dev/null - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: - node-version: '20.x' + node-version: '24.x' # Guards the workflow invariants restored in issue #96 (timeouts, least # privilege, no masked failures, no dead release modes). @@ -193,13 +193,13 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup Rust uses: dtolnay/rust-toolchain@stable - name: Cache cargo registry - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: | ~/.cargo/registry @@ -229,7 +229,7 @@ jobs: if: github.event_name == 'pull_request' needs: [detect-changes] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 @@ -239,7 +239,7 @@ jobs: components: rustfmt, clippy - name: Cache cargo registry - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: | ~/.cargo/registry @@ -266,13 +266,13 @@ jobs: needs: [lint, test] if: ${{ !cancelled() && needs.lint.result == 'success' && needs.test.result == 'success' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup Rust uses: dtolnay/rust-toolchain@stable - name: Cache cargo registry - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: | ~/.cargo/registry @@ -303,7 +303,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} @@ -402,7 +402,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} @@ -473,7 +473,7 @@ jobs: contents: write pull-requests: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 17acbc7..ffe2bfd 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -48,11 +48,11 @@ jobs: - language: actions build-mode: none steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup .NET if: matrix.language == 'csharp' - uses: actions/setup-dotnet@v5 + uses: actions/setup-dotnet@v6 with: dotnet-version: '10.0.x' @@ -79,7 +79,7 @@ jobs: contents: read pull-requests: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Review dependency changes uses: actions/dependency-review-action@v5 @@ -99,7 +99,7 @@ jobs: matrix: manifest: [rust, rust/wasm] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: taiki-e/install-action@v2 with: @@ -116,11 +116,11 @@ jobs: group: check-${{ github.workflow }}-${{ github.ref }}-npm-audit cancel-in-progress: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@v7 with: - node-version: '20.x' + node-version: '24.x' - name: Audit current lock working-directory: js @@ -141,11 +141,11 @@ jobs: group: check-${{ github.workflow }}-${{ github.ref }}-secrets-scan cancel-in-progress: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@v7 with: - node-version: '20.x' + node-version: '24.x' - name: Run secretlint run: npx --yes -p secretlint -p @secretlint/secretlint-rule-preset-recommend secretlint --secretlintignore .gitignore "**/*" diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 0c602f7..f2b6bc1 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -34,7 +34,7 @@ jobs: timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Setup Rust uses: dtolnay/rust-toolchain@stable @@ -45,14 +45,14 @@ jobs: run: cargo install wasm-pack --version 0.14.0 --locked - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: - node-version: '20.x' + node-version: '24.x' cache: npm cache-dependency-path: js/package-lock.json - name: Cache cargo registry - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: | ~/.cargo/registry diff --git a/csharp/.changeset/issue-105-latest-dependencies.md b/csharp/.changeset/issue-105-latest-dependencies.md new file mode 100644 index 0000000..2e1757a --- /dev/null +++ b/csharp/.changeset/issue-105-latest-dependencies.md @@ -0,0 +1,5 @@ +--- +'Foundation.Data.Doublets.Cli': patch +--- + +Updated `System.CommandLine` to 2.0.12, and the test dependencies `Microsoft.NET.Test.Sdk` to 18.10.1 and `coverlet.collector` to 10.1.0. A new `Dependencies` workflow fails while any dependency is behind its latest release, and Dependabot opens the update pull requests. diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj b/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj index db2cdad..1ffcb2b 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Foundation.Data.Doublets.Cli.Tests.csproj @@ -10,8 +10,8 @@ - - + + diff --git a/csharp/Foundation.Data.Doublets.Cli/Foundation.Data.Doublets.Cli.csproj b/csharp/Foundation.Data.Doublets.Cli/Foundation.Data.Doublets.Cli.csproj index 3bb312f..3175d89 100644 --- a/csharp/Foundation.Data.Doublets.Cli/Foundation.Data.Doublets.Cli.csproj +++ b/csharp/Foundation.Data.Doublets.Cli/Foundation.Data.Doublets.Cli.csproj @@ -25,7 +25,7 @@ - + diff --git a/js/package-lock.json b/js/package-lock.json index d407f6d..9c5eeaf 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -9,30 +9,47 @@ "version": "2.3.0", "dependencies": { "doublets-web": "^0.1.3", - "lucide-react": "^1.37.0", - "react": "^19.2.8", - "react-dom": "^19.2.8" + "lucide-react": "^1.52.0", + "react": "^19.3.0", + "react-dom": "^19.3.0" }, "devDependencies": { "@vitejs/plugin-react": "^6.1.1", - "vite": "^8.2.2", + "vite": "^8.3.2", "vite-plugin-wasm": "^3.6.0" } }, "node_modules/@oxc-project/types": { - "version": "0.144.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.144.0.tgz", - "integrity": "sha512-nuhZIOLuI6TFQ32I/WnUx+SCPY7SdSKwgnFHydAuoS1+Z4BRcaP+RRJmGzl9lw+0OFF7UmaESf7KQRXaNLHypg==", + "version": "0.152.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.152.0.tgz", + "integrity": "sha512-oM/5rLBm2tPkg0iBgkH/FOeR3PCDpY19GTgAZjMFM8h9WI9VW7cLgzp6nwtarYKmovavIQZ+Fe/RKX/8C8O/Rw==", "dev": true, "license": "MIT", "funding": { - "url": "https://github.com/sponsors/Boshen" + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.12.tgz", + "integrity": "sha512-dB/a1214qKfHMXCpgqR4OZT+jS4kTyEXbQGJPqzobt5EwH5rX080pxE37alt3RzvR1bf1Yz/yGqRfrYAxuPw0A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" } }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.4.tgz", - "integrity": "sha512-jHC2cnyKz5xU2fhECtFl8OZ83cYNt13GZQD+0uMJ/X3o+ijmd56okHhTUwxVSHPx1IRVIJEZ1/1pPzeLCU6XKA==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.12.tgz", + "integrity": "sha512-7KHFgQ5VJxIHcLlrwrc3Xbds7oTNQT7Pgi9gQCJKrd2VGab/UksIOYp6VD8MzCstGxOKMgNamPwUCfxPdP1OHg==", "cpu": [ "arm64" ], @@ -47,9 +64,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.4.tgz", - "integrity": "sha512-Dc5mPD8F5F/FS8i01syd7FTF6yB2fVthH/TRkjwJkzUK6EpoxHtqvZQP5Zwq80/5z19TWYHIg1KOHboCgVx/aQ==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.12.tgz", + "integrity": "sha512-3YIhqHD96nA5SaYNRBR16HnGv4oavZvXfD/ayHM+oYZ0WD/8lBAtf6zQua4kEyAvpqrluKXl0lnOBoiNby7x9w==", "cpu": [ "arm64" ], @@ -64,9 +81,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.4.tgz", - "integrity": "sha512-fpDm4oBo6SqLvWUYCmFhdde3U9KH2fRNNMeAnAPAIwxRL345xutL0EtEUcuoxsoazdJGv/MuDBQHlCDrtbvqOg==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.12.tgz", + "integrity": "sha512-UuuJ35MFw4gmFOrE9pEqIV+K3syIKveph+Qc1/ljHZVdoDW4pz/JHR/eMVom+TZGl/5OOvGJOWaOCVt3ZfqhxA==", "cpu": [ "x64" ], @@ -81,9 +98,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.4.tgz", - "integrity": "sha512-rSJoreDE/HoIzoaib6MTp5jQtCTdMHKIvItAKT/ImS6Y6Ww76oUaeMyp4Vc/fAgd/ehji068IxetHXAnqUwN9A==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.12.tgz", + "integrity": "sha512-uMvssit0a4W+/7D8CbHUvG719mH3R2jwXAlh/XcPvuHTE0g++LymF88DCGNX0HM2rBOn0xrzgXktIB6fLSJBTQ==", "cpu": [ "x64" ], @@ -98,9 +115,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.4.tgz", - "integrity": "sha512-/jm8OGHgn7oGaJu3i/qZI9spUGcJ+y/lk43ttQ/iO1tOd9NissG6o97bighBCiL+BKRngmcDuR6ikfwYdJmVuQ==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.12.tgz", + "integrity": "sha512-XcFu0R0xWnwzSf4IQgFH1rJIckPN1pLy2R+4r9IDB7Yfu/ys9cVqfa4pBrMHj7a3gl8mIR4nRNPg0e5IvEVs6g==", "cpu": [ "arm" ], @@ -115,9 +132,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.4.tgz", - "integrity": "sha512-tIP06BeD9EqvECBrPZ+sqdPlYrT+aYaAiu1wYziVx5elRK/ftm33JxVDy2bXGbr6J0CrtirCkR87/X5a2euEng==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.12.tgz", + "integrity": "sha512-260UrKgn8tz39ak+SMDOirKzr7V04M9dWPw5llW00SwBivCZoWcRBKV1d8cXnRkUmSZA3BdiUmBHWk7734Ulpw==", "cpu": [ "arm64" ], @@ -135,9 +152,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.4.tgz", - "integrity": "sha512-Ql1Q0EQqVThvn9VAVlwNzsUvbSFtCMGjLpRRi4pk5i7NZZ4n5ISiLMjHYtus4VQ2PvkSw24zyaCVsiS+sXPj1w==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.12.tgz", + "integrity": "sha512-5YK1I9SqDkbPgc1IA8BgDl34suqUS2q0KWnBrirm0E51YjOs6eo6dV6jbQfNE/argHRSvd0QUGgtpIoYx+WWpw==", "cpu": [ "arm64" ], @@ -155,9 +172,9 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.4.tgz", - "integrity": "sha512-GjbjXD4XXfN19D0LZNbmiCBUoDiRACsYHr0yaIbbn8aFsXjHZifcYqu/W5Er5X2X990WjHXFrxarn5chzItorQ==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.12.tgz", + "integrity": "sha512-Rkcrmp7eFRg74yL5fXEU91JEWbdEPLevWwGtXpmhbjlD1StScbWTmO94Bhly+Mo+ketKYkdmM1vNUKeWSlx8cQ==", "cpu": [ "ppc64" ], @@ -175,9 +192,9 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.4.tgz", - "integrity": "sha512-p5WR0NOwaRmJ/B1b6IjEFLLivwEsf3PrdBIhRbhTCQisbo2SvHHpG4ELB/+FgQNnB88LTOF86upmJmbvZdQ2lw==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.12.tgz", + "integrity": "sha512-qvK4DuAsQc2BSjlx+Xr+IzOIvvxbGZqxFwdWfG6F518Erj0GGISyQbJ6pIappnOxlNPzNHvo/L0BwB30GZ+zVw==", "cpu": [ "s390x" ], @@ -195,9 +212,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.4.tgz", - "integrity": "sha512-4/GyVjmhR+Tc6HLJvwc1sOhPqAZtySiSMesOZyX6JQ5XBxoTDEMKQzvo07NIK6nTon/SivlZqvhzvuVBNQhObQ==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.12.tgz", + "integrity": "sha512-Q9uLBO53Xd4QIq1WOycVQyPP1O4HhraEV2qqb3uTrnVw6QZih9duY4vNXOivL1xoUS1/z+W8eF4NMfl2a8Sdjw==", "cpu": [ "x64" ], @@ -215,9 +232,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.4.tgz", - "integrity": "sha512-l9eeLsCNvPpmSXUej0etw/J1eqV0Jj1D5G/xG6YTijmE6dkv6E2QezgWbTfQk63v952DPqrjOCoiqxq7Bw0YUQ==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.12.tgz", + "integrity": "sha512-3IBxWFMjbOZskDPKv8Lf9BCnahlKuHthWkYnyIxOH/QcJrFcS4EmcenthApkwr/5+nEqZlLzeYbxeMaX7A5u4g==", "cpu": [ "x64" ], @@ -235,9 +252,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.4.tgz", - "integrity": "sha512-e0F355MSTMm3+UOqtV3L24gFUp2N5m1f8L/7d56deik6va+AXdrt9F8LbzGpeWGWRbZEDq4m8NVnJDeBtf9DZg==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.12.tgz", + "integrity": "sha512-xtX61xg4LKPkPWilZU1ynKClz5Gj4bf74LML4r3eVLWumKnGjoEr1OSHQhMdbBDoYTi+yjrujvpZe2pUnqCrrA==", "cpu": [ "arm64" ], @@ -252,9 +269,9 @@ } }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.4.tgz", - "integrity": "sha512-AWLi0uBRYh6QlE7OKhiz+phZC0qwtij2QZmhmOdsLdFn64m7oMpooE9ICE3lhm9xMb4SpDo2WbHcxX1iFLFtqw==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.12.tgz", + "integrity": "sha512-At7fPB6PCaIjzgIhEZFxuT+BBFqiQibJDT4d3PhiR3f4E7bbMZF4aKblbFfEM3sETRDd1YiQx/+U/g/B/ou5Ew==", "cpu": [ "arm64" ], @@ -269,9 +286,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.4.tgz", - "integrity": "sha512-UwSDJOg3dqCAejWdxclJjCsh3Qq4vLYMDxmyHqo1btz3stK2VqgwNd3mm5tuIwzSlGIQ/1H9Hr+Zn09mrezNqQ==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.12.tgz", + "integrity": "sha512-WIw2haVKwjuYdXkHaoC0mF8Le71TuCBxjrdKqLbJGctbBABj+ClfmNvtbOnzpq3RokNo5+V1qhtSzJyXorsklQ==", "cpu": [ "x64" ], @@ -645,18 +662,24 @@ } }, "node_modules/lucide-react": { - "version": "1.37.0", - "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.37.0.tgz", - "integrity": "sha512-LPsB4rD1TD6wZu1djKOf9vUnS1jTNaHbolXebXDgiTdb6jeA1agIJhJsIybCmjKmQClcOaal1o1OaiYahEftyQ==", + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.52.0.tgz", + "integrity": "sha512-TUgPtl5ZI9WgxOcbu4ckaC5B3l1qxPQrCMgb6OfUB9owx/OvSJfcf93SMgUWAxRw5/1XDfh9uI5F5iNGXCcGDQ==", "license": "ISC", "peerDependencies": { + "@types/react": "*", "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } } }, "node_modules/nanoid": { - "version": "3.3.18", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", - "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", "dev": true, "funding": [ { @@ -680,9 +703,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "dev": true, "license": "MIT", "engines": { @@ -693,9 +716,9 @@ } }, "node_modules/postcss": { - "version": "8.5.26", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", - "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", "dev": true, "funding": [ { @@ -713,7 +736,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.17", + "nanoid": "^3.3.18", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -722,34 +745,34 @@ } }, "node_modules/react": { - "version": "19.2.8", - "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", - "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", + "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/react-dom": { - "version": "19.2.8", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", - "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", "license": "MIT", "dependencies": { - "scheduler": "^0.27.0" + "scheduler": "^0.28.0" }, "peerDependencies": { - "react": "^19.2.8" + "react": "^19.3.0" } }, "node_modules/rolldown": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.4.tgz", - "integrity": "sha512-rSr7irW0K7QRWzjdJXqZowkcRdDtjRduh43rBltnVKd0VFq839l1lJoDvGJb6gl7+4rTTCrPWu+YfujUL8Ug7w==", + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.12.tgz", + "integrity": "sha512-8wafseiaG80xmXSfqidUNqZcylTlhmPZZt+za2m+js2sFZ8dTNlhIOV2WcbIPx2hgwPBJpEUGFAMZ9bgBBLTSQ==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.144.0", + "@oxc-project/types": "=0.152.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -759,32 +782,33 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.2.4", - "@rolldown/binding-darwin-arm64": "1.2.4", - "@rolldown/binding-darwin-x64": "1.2.4", - "@rolldown/binding-freebsd-x64": "1.2.4", - "@rolldown/binding-linux-arm-gnueabihf": "1.2.4", - "@rolldown/binding-linux-arm64-gnu": "1.2.4", - "@rolldown/binding-linux-arm64-musl": "1.2.4", - "@rolldown/binding-linux-ppc64-gnu": "1.2.4", - "@rolldown/binding-linux-s390x-gnu": "1.2.4", - "@rolldown/binding-linux-x64-gnu": "1.2.4", - "@rolldown/binding-linux-x64-musl": "1.2.4", - "@rolldown/binding-openharmony-arm64": "1.2.4", - "@rolldown/binding-win32-arm64-msvc": "1.2.4", - "@rolldown/binding-win32-x64-msvc": "1.2.4" + "@rolldown/binding-android-arm-eabi": "1.2.12", + "@rolldown/binding-android-arm64": "1.2.12", + "@rolldown/binding-darwin-arm64": "1.2.12", + "@rolldown/binding-darwin-x64": "1.2.12", + "@rolldown/binding-freebsd-x64": "1.2.12", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.12", + "@rolldown/binding-linux-arm64-gnu": "1.2.12", + "@rolldown/binding-linux-arm64-musl": "1.2.12", + "@rolldown/binding-linux-ppc64-gnu": "1.2.12", + "@rolldown/binding-linux-s390x-gnu": "1.2.12", + "@rolldown/binding-linux-x64-gnu": "1.2.12", + "@rolldown/binding-linux-x64-musl": "1.2.12", + "@rolldown/binding-openharmony-arm64": "1.2.12", + "@rolldown/binding-win32-arm64-msvc": "1.2.12", + "@rolldown/binding-win32-x64-msvc": "1.2.12" } }, "node_modules/scheduler": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", - "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", "license": "MIT" }, "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -809,16 +833,16 @@ } }, "node_modules/vite": { - "version": "8.2.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", - "integrity": "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q==", + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.2.tgz", + "integrity": "sha512-SQr1x6W5vVSbROg7vsyXIaxK9b0G7zsT68acdWWRmnBUsgDieLCRG+Rep9WdZgcposvv/GSnr4GUUBqB3vXq6w==", "dev": true, "license": "MIT", "dependencies": { "lightningcss": "^1.33.0", - "picomatch": "^4.0.5", - "postcss": "^8.5.26", - "rolldown": "~1.2.4", + "picomatch": "^4.0.7", + "postcss": "^8.5.28", + "rolldown": "~1.2.11", "tinyglobby": "^0.2.17" }, "bin": { @@ -835,7 +859,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.4.0 || ^0.5.0", + "@vitejs/devtools": "^0.7.1", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", diff --git a/js/package.json b/js/package.json index 2e0d476..61cb45d 100644 --- a/js/package.json +++ b/js/package.json @@ -18,13 +18,13 @@ }, "dependencies": { "doublets-web": "^0.1.3", - "lucide-react": "^1.37.0", - "react": "^19.2.8", - "react-dom": "^19.2.8" + "lucide-react": "^1.52.0", + "react": "^19.3.0", + "react-dom": "^19.3.0" }, "devDependencies": { "@vitejs/plugin-react": "^6.1.1", - "vite": "^8.2.2", + "vite": "^8.3.2", "vite-plugin-wasm": "^3.6.0" } } diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 5b624da..8caaf39 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -66,9 +66,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "beef" @@ -78,21 +78,21 @@ checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "clap" -version = "4.6.1" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -100,9 +100,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -112,21 +112,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.6", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "colorchoice" @@ -169,7 +169,7 @@ dependencies = [ "platform-num", "platform-trees", "tap", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -187,12 +187,6 @@ version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7454e41ff9012c00d53cf7f475c5e3afa3b91b7c90568495495e8d9bf47a1055" -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - [[package]] name = "errno" version = "0.3.14" @@ -205,97 +199,44 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi", - "wasip2", - "wasip3", ] -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", -] - -[[package]] -name = "hashbrown" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" - [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.0", - "serde", - "serde_core", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - [[package]] name = "leak_slice" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecf3387da9fb41906394e1306ddd3cd26dd9b7177af11c19b45b364b743aed26" -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" -version = "0.2.186" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "link-cli" @@ -306,7 +247,7 @@ dependencies = [ "links-notation", "lino-arguments", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -327,7 +268,7 @@ checksum = "f30ea96250240a92d69d45579dbd199a713e7a79acfa53033d24628dad23cff5" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -356,17 +297,11 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" -[[package]] -name = "log" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" - [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" @@ -415,7 +350,7 @@ checksum = "6782bc71345465116de96d250a36dcf49336010a2320d958d12a5d4390186c90" dependencies = [ "beef", "platform-num", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -427,7 +362,7 @@ dependencies = [ "allocator-api2", "memmap2", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -448,30 +383,20 @@ dependencies = [ "platform-num", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.117", -] - [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -484,9 +409,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags", "errno", @@ -495,17 +420,11 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -513,35 +432,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "serde_json" -version = "1.0.149" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", + "syn 3.0.6", ] [[package]] @@ -552,9 +458,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -563,9 +469,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -602,11 +508,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -617,31 +523,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-xid" -version = "0.2.6" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "utf8parse" @@ -649,58 +549,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", -] - -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - [[package]] name = "windows-link" version = "0.2.1" @@ -715,103 +563,3 @@ checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ "windows-link", ] - -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - -[[package]] -name = "zmij" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 6a57ab4..f8909bd 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -24,7 +24,7 @@ name = "clink" path = "src/main.rs" [dependencies] -thiserror = "2.0.20" +thiserror = "2.0.21" anyhow = "1.0.104" # Issue 67 Rust basis crates: # Source: http://github.com/linksplatform/doublets-rs diff --git a/rust/changelog.d/20261005_180000_issue_105_latest_dependencies.md b/rust/changelog.d/20261005_180000_issue_105_latest_dependencies.md new file mode 100644 index 0000000..5423353 --- /dev/null +++ b/rust/changelog.d/20261005_180000_issue_105_latest_dependencies.md @@ -0,0 +1,5 @@ +--- +bump: patch +--- + +Updated `thiserror` to 2.0.21 and refreshed `Cargo.lock`. A new `Dependencies` workflow fails while any dependency or lock file entry is behind its latest release, and Dependabot opens the update pull requests. diff --git a/rust/wasm/Cargo.lock b/rust/wasm/Cargo.lock index 35c1b12..b9ef337 100644 --- a/rust/wasm/Cargo.lock +++ b/rust/wasm/Cargo.lock @@ -72,7 +72,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -89,9 +89,9 @@ checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bumpalo" @@ -107,9 +107,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.4.4" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "shlex", @@ -117,15 +117,15 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "clap" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -133,9 +133,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -145,21 +145,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "clink-wasm" @@ -226,7 +226,7 @@ dependencies = [ "platform-num", "platform-trees", "tap", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -262,9 +262,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "futures-core" @@ -321,9 +321,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -338,9 +338,9 @@ checksum = "ecf3387da9fb41906394e1306ddd3cd26dd9b7177af11c19b45b364b743aed26" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libm" @@ -356,7 +356,7 @@ dependencies = [ "doublets", "links-notation", "lino-arguments", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -377,7 +377,7 @@ checksum = "f30ea96250240a92d69d45579dbd199a713e7a79acfa53033d24628dad23cff5" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -423,9 +423,9 @@ dependencies = [ [[package]] name = "minicov" -version = "0.3.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3aa3aa12b448ac225b3102217d1ac5cc717908f02722926524b0599c933c7a0" +checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" dependencies = [ "cc", "walkdir", @@ -491,7 +491,7 @@ checksum = "6782bc71345465116de96d250a36dcf49336010a2320d958d12a5d4390186c90" dependencies = [ "beef", "platform-num", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -503,7 +503,7 @@ dependencies = [ "allocator-api2", "memmap2", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -550,9 +550,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags", "errno", @@ -603,7 +603,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -650,9 +650,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -689,11 +689,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -709,20 +709,29 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", +] + +[[package]] +name = "tokio" +version = "1.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "pin-project-lite", ] [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "utf8parse" @@ -742,9 +751,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -755,19 +764,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.77" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -775,31 +785,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-bindgen-test" -version = "0.3.77" +version = "0.3.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895a2607575412a4eda1df892084a375ea10dfeadc4d7d2ab87b854e4ddc7ba1" +checksum = "ae7499dfd45780a0a91d7ee6bb9ac51970a4479a41a89da443fdda5a39547d42" dependencies = [ "async-trait", "cast", @@ -819,26 +829,26 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.77" +version = "0.3.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288cb0ebe215033bf949ae1fd046726daa4c32a157f24b9dc6ac387a52aa759" +checksum = "3b84b5ac638bfb168196a1a461fcc8f46a294a18b1b6be52133b4e0db122cc9f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] name = "wasm-bindgen-test-shared" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff1c1b360982e93b6d8ea9c04836f71dba0817a16f91e229cf3a51bdd9d987" +checksum = "4f692aa943ccd88363733b77063f32cfed5bc6cbea8e6e8b251b302f881606fe" [[package]] name = "web-sys" -version = "0.3.104" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/rust/wasm/Cargo.toml b/rust/wasm/Cargo.toml index b6224ae..834b9e9 100644 --- a/rust/wasm/Cargo.toml +++ b/rust/wasm/Cargo.toml @@ -13,16 +13,16 @@ crate-type = ["cdylib", "rlib"] [dependencies] anyhow = "1.0.104" link-cli = { path = ".." } -wasm-bindgen = "0.2.127" +wasm-bindgen = "0.2.129" console_error_panic_hook = { version = "0.1.7", optional = true } serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" [dev-dependencies] -wasm-bindgen-test = "0.3.77" +wasm-bindgen-test = "0.3.79" [dependencies.web-sys] -version = "0.3.104" +version = "0.3.106" features = [ "console", ] From ff75869ff8b409d12c55b2770901eb6b648a24a4 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:36:32 +0000 Subject: [PATCH 34/44] ci: fail while any dependency is behind its latest release .github/scripts/check-latest-dependencies.mjs compares every Cargo, NuGet and npm dependency, the npm and Cargo lock files, every GitHub Action and the Node.js and .NET versions with their registries. The Dependencies workflow runs it on pull requests and daily, and Dependabot opens the update pull requests. --- .github/dependabot.yml | 32 +++ .github/scripts/check-latest-dependencies.mjs | 269 ++++++++++++++++++ .../check-latest-dependencies.test.mjs | 166 +++++++++++ .github/workflows/dependencies.yml | 42 +++ docs/ARCHITECTURE.md | 1 + 5 files changed, 510 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/scripts/check-latest-dependencies.mjs create mode 100644 .github/scripts/check-latest-dependencies.test.mjs create mode 100644 .github/workflows/dependencies.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9428acd --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,32 @@ +# Opens the pull requests that keep every dependency on its latest release. +# .github/workflows/dependencies.yml fails while anything is still behind. +version: 2 +updates: + - package-ecosystem: cargo + directories: ['/rust', '/rust/wasm'] + schedule: + interval: daily + groups: + cargo: + patterns: ['*'] + - package-ecosystem: nuget + directory: /csharp + schedule: + interval: daily + groups: + nuget: + patterns: ['*'] + - package-ecosystem: npm + directory: /js + schedule: + interval: daily + groups: + npm: + patterns: ['*'] + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily + groups: + actions: + patterns: ['*'] diff --git a/.github/scripts/check-latest-dependencies.mjs b/.github/scripts/check-latest-dependencies.mjs new file mode 100644 index 0000000..60e525d --- /dev/null +++ b/.github/scripts/check-latest-dependencies.mjs @@ -0,0 +1,269 @@ +#!/usr/bin/env node + +/** + * Fail when any dependency of the repository is behind its latest release. + * + * Checked, each against its own registry: + * - Cargo dependencies of rust/ and rust/wasm/ (crates.io), plus every + * package in their Cargo.lock files (`cargo update --dry-run`); + * - NuGet packages of every csharp/ project (nuget.org); + * - npm dependencies of js/, declared and locked (registry.npmjs.org); + * - GitHub Actions used by the workflows (major tags of the action's repo); + * - Node.js and .NET versions set up by the workflows (latest LTS) and tools + * installed with taiki-e/install-action (crates.io). + * + * A declared version is compared only as far as it is written: `v7` matches + * any 7.x release, `24.x` any 24.x, and `2.0.21` exactly 2.0.21. + * + * Usage: + * node .github/scripts/check-latest-dependencies.mjs [--skip-lockfiles] + * + * Exit codes: + * - 0: everything is on its latest version + * - 1: something is behind (listed in the output and the step summary) + */ + +import { execFileSync, spawnSync } from 'node:child_process'; +import { appendFileSync, readFileSync, readdirSync } from 'node:fs'; +import { dirname, join, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const userAgent = 'link-cli dependency check (https://github.com/link-foundation/link-cli)'; + +/** Splits a version into its numeric parts; `x` and anything after `-` or `+` end it. */ +export function versionParts(version) { + const parts = []; + for (const part of version.replace(/^v/, '').split(/[-+]/)[0].split('.')) { + if (!/^\d+$/.test(part)) break; + parts.push(Number(part)); + } + return parts; +} + +/** Compares two versions part by part, missing parts counting as 0. */ +export function compareVersions(a, b) { + const left = versionParts(a); + const right = versionParts(b); + for (let i = 0; i < Math.max(left.length, right.length); i++) { + const difference = (left[i] ?? 0) - (right[i] ?? 0); + if (difference !== 0) return Math.sign(difference); + } + return 0; +} + +/** True when `latest`, cut to as many parts as `declared` has, is newer than it. */ +export function isOutdated(declared, latest) { + const precision = versionParts(declared).length; + const cut = versionParts(latest).slice(0, precision).join('.'); + return compareVersions(cut, declared) > 0; +} + +/** The newest version without a pre-release suffix. */ +export function latestStable(versions) { + return versions + .filter((version) => /^v?\d+(\.\d+)*$/.test(version)) + .reduce((best, version) => (best === undefined || compareVersions(version, best) > 0 ? version : best), undefined); +} + +/** Registry dependencies of a Cargo.toml; path-only dependencies are skipped. */ +export function parseCargoManifest(text) { + const dependencies = []; + let table = null; // 'list' inside [dependencies], or the name of [dependencies.NAME] + for (const raw of text.split('\n')) { + const line = raw.replace(/#.*$/, '').trim(); + if (line === '') continue; + const header = /^\[(.+)\]$/.exec(line); + if (header) { + const path = header[1].trim(); + const inline = /(?:^|\.)(?:dev-|build-)?dependencies$/.exec(path); + const named = /(?:^|\.)(?:dev-|build-)?dependencies\.([A-Za-z0-9_-]+)$/.exec(path); + table = inline ? 'list' : named ? named[1] : null; + continue; + } + if (table === 'list') { + const plain = /^([A-Za-z0-9_-]+)\s*=\s*"([^"]+)"$/.exec(line); + const detailed = /^([A-Za-z0-9_-]+)\s*=\s*\{(.*)\}$/.exec(line); + if (plain) { + dependencies.push({ name: plain[1], version: plain[2] }); + } else if (detailed) { + const version = /\bversion\s*=\s*"([^"]+)"/.exec(detailed[2]); + if (version) dependencies.push({ name: detailed[1], version: version[1] }); + } + } else if (table) { + const version = /^version\s*=\s*"([^"]+)"$/.exec(line); + if (version) dependencies.push({ name: table, version: version[1] }); + } + } + return dependencies; +} + +/** `` entries of a project file. */ +export function parseCsproj(text) { + return [...text.matchAll(/ ({ name, version }), + ); +} + +/** Dependencies and devDependencies of a package.json, without range operators. */ +export function parsePackageJson(text) { + const manifest = JSON.parse(text); + return Object.entries({ ...manifest.dependencies, ...manifest.devDependencies }).map(([name, range]) => ({ + name, + version: range.replace(/^[\^~=]/, ''), + })); +} + +/** Locked versions of a package-lock.json's direct dependencies. */ +export function parsePackageLock(text, names) { + const packages = JSON.parse(text).packages ?? {}; + return names + .filter((name) => packages[`node_modules/${name}`]) + .map((name) => ({ name, version: packages[`node_modules/${name}`].version })); +} + +/** Actions pinned to a major tag (`owner/repo[/path]@vN`), keyed by repository. */ +export function parseWorkflowActions(text) { + const actions = new Map(); + for (const [, repository, version] of text.matchAll(/uses:\s*([\w.-]+\/[\w.-]+)(?:\/[\w./-]+)?@(v\d+)\s*$/gm)) { + actions.set(repository, version); + } + return [...actions].map(([name, version]) => ({ name, version })); +} + +/** Runtimes and tools a workflow installs: node-version, dotnet-version, `tool: name@version`. */ +export function parseWorkflowTools(text) { + const tools = []; + for (const [, kind, version] of text.matchAll(/^\s*(node|dotnet)-version:\s*['"]?([\w.]+)['"]?\s*$/gm)) { + tools.push({ kind, name: kind === 'node' ? 'Node.js' : '.NET', version }); + } + for (const [, list] of text.matchAll(/^\s*tool:\s*(.+)$/gm)) { + for (const entry of list.split(',')) { + const pinned = /^\s*([\w-]+)@([\d.]+)\s*$/.exec(entry); + if (pinned) tools.push({ kind: 'crate', name: pinned[1], version: pinned[2] }); + } + } + return tools; +} + +/** `Updating name vA -> vB` lines of `cargo update --dry-run`. */ +export function parseCargoUpdates(output) { + return [...output.matchAll(/^\s*Updating (\S+) v(\S+) -> v(\S+)/gm)].map(([, name, from, to]) => ({ + name, + version: from, + latest: to, + })); +} + +async function fetchJson(url) { + const response = await fetch(url, { headers: { 'User-Agent': userAgent, Accept: 'application/json' } }); + if (!response.ok) throw new Error(`${url} answered ${response.status}`); + return response.json(); +} + +const registries = { + crate: async (name) => (await fetchJson(`https://crates.io/api/v1/crates/${name}`)).crate.max_stable_version, + nuget: async (name) => + latestStable((await fetchJson(`https://api.nuget.org/v3-flatcontainer/${name.toLowerCase()}/index.json`)).versions), + npm: async (name) => (await fetchJson(`https://registry.npmjs.org/${name.replace('/', '%2F')}/latest`)).version, + action: async (name) => { + const tags = execFileSync('git', ['ls-remote', '--tags', `https://github.com/${name}`], { encoding: 'utf8' }) + .split('\n') + .map((line) => line.split('refs/tags/')[1]) + .filter(Boolean); + return `v${versionParts(latestStable(tags))[0]}`; + }, + node: async () => (await fetchJson('https://nodejs.org/dist/index.json')).find((release) => release.lts).version, + dotnet: async () => + latestStable( + (await fetchJson('https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/releases-index.json'))[ + 'releases-index' + ] + .filter((release) => release['release-type'] === 'lts' && release['support-phase'] === 'active') + .map((release) => release['channel-version']), + ), +}; + +function read(path) { + return readFileSync(join(repoRoot, path), 'utf8'); +} + +function findFiles(directory, suffix) { + return readdirSync(join(repoRoot, directory), { recursive: true }) + .filter((name) => name.endsWith(suffix) && !/(^|[\\/])(bin|obj|node_modules|target)[\\/]/.test(name)) + .map((name) => join(directory, name)); +} + +/** Everything to check, as `{ registry, name, version, source }`. */ +function collectDependencies() { + const entries = []; + for (const manifest of ['rust/Cargo.toml', 'rust/wasm/Cargo.toml']) { + for (const dependency of parseCargoManifest(read(manifest))) entries.push({ registry: 'crate', source: manifest, ...dependency }); + } + for (const project of findFiles('csharp', '.csproj')) { + for (const dependency of parseCsproj(read(project))) entries.push({ registry: 'nuget', source: project, ...dependency }); + } + const packageJson = parsePackageJson(read('js/package.json')); + for (const dependency of packageJson) entries.push({ registry: 'npm', source: 'js/package.json', ...dependency }); + const locked = parsePackageLock(read('js/package-lock.json'), packageJson.map(({ name }) => name)); + for (const dependency of locked) entries.push({ registry: 'npm', source: 'js/package-lock.json', ...dependency }); + for (const workflow of findFiles('.github/workflows', '.yml')) { + const text = read(workflow); + for (const action of parseWorkflowActions(text)) entries.push({ registry: 'action', source: workflow, ...action }); + for (const { kind, ...tool } of parseWorkflowTools(text)) entries.push({ registry: kind, source: workflow, ...tool }); + } + const seen = new Set(); + return entries.filter(({ registry, source, name, version }) => { + const key = [registry, source, name, version].join(' '); + return !seen.has(key) && seen.add(key); + }); +} + +function checkLockfiles() { + const outdated = []; + for (const directory of ['rust', 'rust/wasm']) { + // cargo reports the updates on stderr. + const { stderr } = spawnSync('cargo', ['update', '--dry-run'], { cwd: join(repoRoot, directory), encoding: 'utf8' }); + for (const update of parseCargoUpdates(stderr ?? '')) outdated.push({ source: `${directory}/Cargo.lock`, ...update }); + } + return outdated; +} + +async function main() { + const entries = collectDependencies(); + const cache = new Map(); + const latest = (registry, name) => { + const key = `${registry}:${name}`; + if (!cache.has(key)) cache.set(key, registries[registry](name)); + return cache.get(key); + }; + const outdated = []; + const failures = []; + await Promise.all( + entries.map(async (entry) => { + try { + entry.latest = await latest(entry.registry, entry.name); + if (isOutdated(entry.version, entry.latest)) outdated.push(entry); + } catch (error) { + failures.push(`${entry.source}: ${entry.name}: ${error.message}`); + } + }), + ); + if (!process.argv.includes('--skip-lockfiles')) outdated.push(...checkLockfiles()); + + console.log(`Checked ${entries.length} declared versions${process.argv.includes('--skip-lockfiles') ? '' : ' and the Cargo.lock files'}.`); + const lines = outdated + .sort((a, b) => a.source.localeCompare(b.source) || a.name.localeCompare(b.name)) + .map(({ source, name, version, latest: newest }) => `| ${relative(repoRoot, join(repoRoot, source))} | ${name} | ${version} | ${newest} |`); + if (lines.length > 0) { + const table = ['| File | Dependency | Used | Latest |', '|---|---|---|---|', ...lines].join('\n'); + console.log(`\n${lines.length} dependencies are behind their latest release:\n\n${table}`); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## Outdated dependencies\n\n${table}\n`); + } + for (const failure of failures) console.error(`Could not check ${failure}`); + process.exitCode = lines.length > 0 || failures.length > 0 ? 1 : 0; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + await main(); +} diff --git a/.github/scripts/check-latest-dependencies.test.mjs b/.github/scripts/check-latest-dependencies.test.mjs new file mode 100644 index 0000000..ec4bd7a --- /dev/null +++ b/.github/scripts/check-latest-dependencies.test.mjs @@ -0,0 +1,166 @@ +// Tests for the dependency freshness check. They use no network: the parsers +// and the version comparison decide whether a declared version is behind, so +// a parser that misses an entry would be a silent false negative. +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + compareVersions, + isOutdated, + latestStable, + parseCargoManifest, + parseCargoUpdates, + parseCsproj, + parsePackageJson, + parsePackageLock, + parseWorkflowActions, + parseWorkflowTools, + versionParts, +} from './check-latest-dependencies.mjs'; + +test('versions compare numerically, not as text', () => { + assert.deepEqual(versionParts('v24.21.0'), [24, 21, 0]); + assert.deepEqual(versionParts('10.0.x'), [10, 0]); + assert.deepEqual(versionParts('1.2.3-beta.1'), [1, 2, 3]); + assert.equal(compareVersions('0.2.129', '0.2.13'), 1); + assert.equal(compareVersions('2.0', '2.0.0'), 0); + assert.equal(compareVersions('18.9.0', '18.10.1'), -1); +}); + +test('a declared version is compared only as far as it is written', () => { + assert.equal(isOutdated('v7', 'v7'), false); + assert.equal(isOutdated('v6', 'v7'), true); + assert.equal(isOutdated('24.x', 'v24.21.0'), false); + assert.equal(isOutdated('20.x', 'v24.21.0'), true); + assert.equal(isOutdated('10.0.x', '10.0'), false); + assert.equal(isOutdated('2.0.20', '2.0.21'), true); + assert.equal(isOutdated('2.0.21', '2.0.21'), false); + // A version ahead of the registry (for example a yanked release) is not behind. + assert.equal(isOutdated('3.0.0', '2.9.9'), false); +}); + +test('pre-releases are never the latest version', () => { + assert.equal(latestStable(['1.0.0', '1.1.0-rc.1', '1.0.1']), '1.0.1'); + assert.equal(latestStable(['v6', 'v6.1.0', 'v7.0.0-beta', 'v7-preview']), 'v6.1.0'); + assert.equal(latestStable([]), undefined); +}); + +test('every Cargo dependency form is found, path-only ones are skipped', () => { + const manifest = ` +[package] +version = "9.9.9" + +[dependencies] +thiserror = "2.0.21" # a comment +link-cli = { path = ".." } +serde = { version = "1.0.229", features = ["derive"] } + +[dev-dependencies] +tempfile = "3.27.0" + +[target.'cfg(unix)'.dependencies] +libc = "0.2.180" + +[dependencies.web-sys] +version = "0.3.106" +features = [ + "console", +] + +[features] +default = ["x"] +`; + assert.deepEqual(parseCargoManifest(manifest), [ + { name: 'thiserror', version: '2.0.21' }, + { name: 'serde', version: '1.0.229' }, + { name: 'tempfile', version: '3.27.0' }, + { name: 'libc', version: '0.2.180' }, + { name: 'web-sys', version: '0.3.106' }, + ]); +}); + +test('NuGet package references are found', () => { + const project = ` + + + all + + + `; + assert.deepEqual(parseCsproj(project), [ + { name: 'xunit', version: '2.9.3' }, + { name: 'coverlet.collector', version: '10.1.0' }, + ]); +}); + +test('npm ranges lose their operator, and locked versions come from the lock file', () => { + const manifest = JSON.stringify({ + dependencies: { react: '^19.3.0' }, + devDependencies: { '@vitejs/plugin-react': '~6.1.1', vite: '8.3.2' }, + }); + const dependencies = parsePackageJson(manifest); + assert.deepEqual(dependencies, [ + { name: 'react', version: '19.3.0' }, + { name: '@vitejs/plugin-react', version: '6.1.1' }, + { name: 'vite', version: '8.3.2' }, + ]); + const lock = JSON.stringify({ + packages: { + '': {}, + 'node_modules/react': { version: '19.2.8' }, + 'node_modules/vite': { version: '8.3.2' }, + 'node_modules/nested/node_modules/react': { version: '1.0.0' }, + }, + }); + assert.deepEqual(parsePackageLock(lock, dependencies.map(({ name }) => name)), [ + { name: 'react', version: '19.2.8' }, + { name: 'vite', version: '8.3.2' }, + ]); +}); + +test('actions pinned to a major tag are found once per repository', () => { + const workflow = ` + - uses: actions/checkout@v7 + - uses: github/codeql-action/init@v4 + - uses: github/codeql-action/analyze@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: ./.github/actions/local + - uses: actions/setup-node@v7 +`; + assert.deepEqual(parseWorkflowActions(workflow), [ + { name: 'actions/checkout', version: 'v7' }, + { name: 'github/codeql-action', version: 'v4' }, + { name: 'actions/setup-node', version: 'v7' }, + ]); +}); + +test('runtimes and installed tools are found', () => { + const workflow = ` + with: + node-version: '24.x' + dotnet-version: "10.0.x" + - uses: taiki-e/install-action@v2 + with: + tool: cargo-audit@0.22.2, wasm-pack +`; + assert.deepEqual(parseWorkflowTools(workflow), [ + { kind: 'node', name: 'Node.js', version: '24.x' }, + { kind: 'dotnet', name: '.NET', version: '10.0.x' }, + { kind: 'crate', name: 'cargo-audit', version: '0.22.2' }, + ]); +}); + +test('only real updates of cargo update --dry-run count', () => { + const report = ` Updating crates.io index + Locking 2 packages to latest compatible versions + Updating autocfg v1.5.0 -> v1.5.1 + Updating thiserror v2.0.20 -> v2.0.21 + Adding tokio v1.53.2 + Unchanged rand v0.8.5 (available: v0.9.2) +warning: aborting update due to dry run +`; + assert.deepEqual(parseCargoUpdates(report), [ + { name: 'autocfg', version: '1.5.0', latest: '1.5.1' }, + { name: 'thiserror', version: '2.0.20', latest: '2.0.21' }, + ]); +}); diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 0000000..14988ee --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,42 @@ +name: Dependencies + +# Every dependency stays on its latest release (PR #106 review): Cargo crates +# and their lock files, NuGet packages, npm packages, GitHub Actions, and the +# Node.js and .NET versions the workflows set up. Dependabot +# (.github/dependabot.yml) opens the pull requests that bump them; this check +# fails while anything is still behind, on every pull request and daily on +# main, so a new upstream release is noticed within a day. + +on: + push: + branches: [main] + pull_request: + types: [opened, synchronize, reopened] + schedule: + - cron: '0 5 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dependencies-${{ github.ref }} + cancel-in-progress: true + +jobs: + latest: + name: Dependencies are on their latest versions + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: '24.x' + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + + - name: Compare every dependency with its registry + run: node .github/scripts/check-latest-dependencies.mjs diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 088f746..ac5558d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -303,6 +303,7 @@ Trigger storage can be: | `.github/workflows/csharp.yml` | .NET restore, build, tests, package, and release. | | `.github/workflows/rust.yml` | Rust formatting, clippy, file-size gate, tests, package, and release. | | `.github/workflows/wasm.yml` | Rust core tests, wasm-pack tests, Vite build, artifact upload, and manual Pages deployment. | +| `.github/workflows/dependencies.yml` | Fails while any Cargo, NuGet or npm dependency, lock file entry, GitHub Action, or the Node.js or .NET version is behind its latest release. Runs on pull requests and daily; `.github/dependabot.yml` opens the update pull requests. | Path filters keep most workflows focused on the parts of the repository they own. From 6d719dadc725fc3bce3716365c7c44dc2b23152c Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:43:57 +0000 Subject: [PATCH 35/44] docs: compare links-queue's binary links notation with ours The case study claimed no project in the organisations had a binary links notation. links-queue has one; the research notes now describe it, measure it against the store archive on the same stores, and record that its spec and encoders disagree on self-references. --- docs/case-studies/issue-105/README.md | 6 ++- .../evidence/links-queue-sizes/encode.mjs | 25 ++++++++++ .../evidence/links-queue-sizes/run.sh | 25 ++++++++++ .../links-queue-sizes/self-reference.mjs | 12 +++++ docs/case-studies/issue-105/research-notes.md | 50 +++++++++++++++++++ 5 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 docs/case-studies/issue-105/evidence/links-queue-sizes/encode.mjs create mode 100755 docs/case-studies/issue-105/evidence/links-queue-sizes/run.sh create mode 100644 docs/case-studies/issue-105/evidence/links-queue-sizes/self-reference.mjs diff --git a/docs/case-studies/issue-105/README.md b/docs/case-studies/issue-105/README.md index 76d481d..c687367 100644 --- a/docs/case-studies/issue-105/README.md +++ b/docs/case-studies/issue-105/README.md @@ -83,10 +83,12 @@ Full notes with quotes and file paths are in [`research-notes.md`](research-note | [link-foundation/lino-rest-api](https://github.com/link-foundation/lino-rest-api) | HTTP REST with `text/lino-line` and RFC 9457 problem details written in LiNo | The `(error: 'message')` reply, a minimal form of the same idea | | [linksplatform/IdDistributorServer](https://github.com/linksplatform/IdDistributorServer), IdClient | A C TCP ping-pong benchmark with fixed 32-byte messages and `TCP_NODELAY` | Only the low-latency stance: `NoDelay` is set on every connection | | [linksplatform/Protocols](https://github.com/linksplatform/Protocols) | UDP string sender and receiver (`Platform.Protocols.Udp`) | Nothing reusable for TCP framing | +| [link-foundation/links-queue](https://github.com/link-foundation/links-queue) | A "Binary Links Notation" codec (`docs/BINARY-NOTATION-SPEC.md`, JS and Rust): an 11-byte frame (`LNKQ` magic, version, flags, big-endian length), then a type byte per link, LEB128 ids and typed inline literals (strings, integers, floats, nested links). Its TCP server frames JSON queue operations with a 4-byte length | Nothing directly. Its links are self-describing trees of values; a store needs implicit addresses and fixed-width references, so the layouts differ (notes §8) | | [link-foundation/links-notation](https://github.com/link-foundation/links-notation) | The LiNo parser and formatter (Rust crate and NuGet package) | Used directly. Upgraded 0.16.1 → 0.22.0 (§6.1) | -No project in these organisations defined a raw TCP protocol for LiNo, so the -framing and the binary layout are new. Where possible they reuse the +No project in these organisations defined a raw TCP protocol for LiNo, and +the one binary notation found (links-queue's) encodes value trees rather than +address-ordered links, so the framing and the binary layout are new. Where possible they reuse the encodings above. ### Public protocols diff --git a/docs/case-studies/issue-105/evidence/links-queue-sizes/encode.mjs b/docs/case-studies/issue-105/evidence/links-queue-sizes/encode.mjs new file mode 100644 index 0000000..3db9b52 --- /dev/null +++ b/docs/case-studies/issue-105/evidence/links-queue-sizes/encode.mjs @@ -0,0 +1,25 @@ +// Writes a store as LiNo for clink, and prints the size of the same links in +// links-queue's Binary Links Notation. +// +// node encode.mjs +import { writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const [, , checkout, shape, count] = process.argv; +const { BinaryNotation } = await import( + pathToFileURL(resolve(checkout, 'js/src/protocol/binary-notation.js')).href +); + +const links = []; +if (shape === 'points') { + for (let i = 1; i <= count; i++) links.push({ id: i, source: i, target: i }); +} else { + // Doublets over earlier addresses, from a fixed linear congruential generator. + let seed = 7; + const below = (limit) => ((seed = (seed * 1103515245 + 12345) % 2147483648) % limit) + 1; + links.push({ id: 1, source: 1, target: 1 }); + for (let i = 2; i <= count; i++) links.push({ id: i, source: below(i - 1), target: below(i - 1) }); +} +writeFileSync(`${shape}-${count}.lino`, links.map(({ id, source, target }) => `(${id}: ${source} ${target})\n`).join('')); +console.log(BinaryNotation.encode(links).length); diff --git a/docs/case-studies/issue-105/evidence/links-queue-sizes/run.sh b/docs/case-studies/issue-105/evidence/links-queue-sizes/run.sh new file mode 100755 index 0000000..a8b39a1 --- /dev/null +++ b/docs/case-studies/issue-105/evidence/links-queue-sizes/run.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Compares the size of one store in LiNo, in links-queue's Binary Links +# Notation and in link-cli's store archive (--export-binary). +# +# ./run.sh +set -euo pipefail +checkout=$(realpath "$1") +clink=$(realpath "$2") +here=$(dirname "$(realpath "$0")") +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +cd "$work" + +echo "| Store | LiNo | links-queue | store archive |" +echo "|---|---|---|---|" +for store in "points 10" "points 1000" "doublets 1000"; do + set -- $store + queue=$(node "$here/encode.mjs" "$checkout" "$1" "$2") + rm -f db.links db.names.links db.bin + "$clink" --db db.links --in "$1-$2.lino" > /dev/null + "$clink" --db db.links --export-binary db.bin > /dev/null + echo "| $2 $1 | $(wc -c < "$1-$2.lino") | $queue | $(wc -c < db.bin) |" +done +echo +node "$here/self-reference.mjs" "$checkout" diff --git a/docs/case-studies/issue-105/evidence/links-queue-sizes/self-reference.mjs b/docs/case-studies/issue-105/evidence/links-queue-sizes/self-reference.mjs new file mode 100644 index 0000000..3da2d93 --- /dev/null +++ b/docs/case-studies/issue-105/evidence/links-queue-sizes/self-reference.mjs @@ -0,0 +1,12 @@ +// links-queue's spec encodes (5: 5 5) as 0F 05. Prints what its encoder writes. +// +// node self-reference.mjs +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const { BinaryNotation, HEADER_SIZE } = await import( + pathToFileURL(resolve(process.argv[2], 'js/src/protocol/binary-notation.js')).href +); +const link = { id: 5, source: 5, target: 5 }; +const body = BinaryNotation.encode(link).slice(HEADER_SIZE + 1); // after the frame and the count +console.log(`(5: 5 5) -> ${Buffer.from(body).toString('hex').replace(/../g, '$& ').trim()}`); diff --git a/docs/case-studies/issue-105/research-notes.md b/docs/case-studies/issue-105/research-notes.md index 01584ce..e533bde 100644 --- a/docs/case-studies/issue-105/research-notes.md +++ b/docs/case-studies/issue-105/research-notes.md @@ -285,3 +285,53 @@ sequence section became sections of any arity. - Text mode: one LiNo query per line (`text/lino-line` style) or length-prefixed UTF-8. Responses use linksql's QueryReport encoded via lino-objects-codec (`((operation ...) (matched ...) (created ...) (updated ...) (deleted ...))`). Errors use linksql error names or the RFC 9457-in-LiNo shape. + +## 8. links-queue Binary Links Notation (`link-foundation/links-queue` at `bdc7631`) + +Found after the design, while looking for a binary notation to share with +[links-notation](https://github.com/link-foundation/links-notation). It came +from links-queue#27 and PR #49. The files are `docs/BINARY-NOTATION-SPEC.md`, +`docs/BINARY-NOTATION-MIGRATION.md`, `js/src/protocol/binary-notation.js` and +`rust/src/backends/binary_notation.rs`. + +- **Frame**: 11 bytes: the magic `LNKQ`, a 2-byte version, a flags byte + (compression, checksum, streaming) and a 4-byte big-endian payload length. + Then come a LEB128 link count and the links. +- **Link**: a type byte (`SOURCE_IS_ID`, `TARGET_IS_ID`, `SELF_REF`, + `HAS_ID`, `HAS_VALUES`, id size), an optional LEB128 id, then the source and + the target. Each is a LEB128 id or a typed literal: null, booleans, + integers, a float, strings, binary data or an inline link. +- **Transport**: the links-queue TCP server (`rust/src/main.rs`) frames JSON + queue requests with a 4-byte length. The binary notation is a codec, and the + JS side offers it in protocol negotiation. + +How it compares with [binary links notation](../../protocol/binary-links-notation.md): + +| | links-queue | link-cli | +|---|---|---| +| Header | 11 bytes | 1 byte, plus section headers | +| Address of a link | written per link, optional | implicit, from the section; gaps leave holes | +| Reference | LEB128, or a literal | 1, 2, 4 or 8 bytes, fixed per section | +| Numbers and strings | typed literals inside the link | links, or Hybrid external references (§5) | +| Links of other arities | a values array after source and target | sections of any arity range | +| Random access to link *n* | no: every link has a variable size | yes, inside a fixed-arity section | + +Sizes of the same stores, measured with +[`evidence/links-queue-sizes/run.sh`](evidence/links-queue-sizes/run.sh) +(the store archive packs both packets with packed widths): + +| Store | LiNo | links-queue | store archive | +|---|---|---|---| +| 10 points | 93 | 42 | 26 | +| 1000 points | 14679 | 4759 | 3500 | +| 1000 doublets | 14104 | 6068 | 3414 | + +The spec and its implementation disagree on self-references. The spec's +example encodes `(5: 5 5)` as `0F 05` ("source == target, only encode once", +with the id standing for both), but both encoders write `0F 05 05`: the id +once and the shared source and target once more. + +Nothing of the layout was reused: a store needs addresses that map one to one +onto the links it holds and fixed-width references, and links-queue's links +are self-describing trees of values. A shared notation would have to cover +both, which is why it is proposed to links-notation rather than adopted here. From f1d61fd0cb96831e4ea2fcd1cb8a88bf80a8d425 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:45:01 +0000 Subject: [PATCH 36/44] docs: link the links-queue and links-notation issues from the case study --- docs/case-studies/issue-105/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/case-studies/issue-105/README.md b/docs/case-studies/issue-105/README.md index c687367..56bcd20 100644 --- a/docs/case-studies/issue-105/README.md +++ b/docs/case-studies/issue-105/README.md @@ -429,6 +429,18 @@ present replies with two deletions. The behaviour is documented in [HOW-IT-WORKS](../../HOW-IT-WORKS.md) and proven by the `DeleteCascades*` tests in C# and the `*cascade*` tests in Rust. +### 6.4 A second binary links notation in links-queue + +links-queue already had a "Binary Links Notation" codec with a different +layout: per-link type bytes, LEB128 ids and typed literals. The comparison is +in [research-notes.md §8](research-notes.md#8-links-queue-binary-links-notation-link-foundationlinks-queue-at-bdc7631). +On the same stores it takes 1.36 to 1.78 times the bytes of the store archive. +Its spec says a self-referencing link takes 2 bytes, while both of its +encoders write 3. That is reported as +[links-queue#51](https://github.com/link-foundation/links-queue/issues/51). +To avoid a third format, a shared spec in links-notation is proposed in +[links-notation#325](https://github.com/link-foundation/links-notation/issues/325). + ## 7. Risks and remaining limits - **No authentication or encryption.** The server is meant for trusted From b786f6e0a31c5ffba832704d140e9fcffdb4ccfe Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:59:23 +0000 Subject: [PATCH 37/44] feat(rust): add LinkStorage::in_memory, a store without a file It never reads or writes the file system: save, flush and reload write nothing, and has_external_changes is always false. The browser has no file system (fs::metadata returns Unsupported on wasm32), so this is the store the WebAssembly workbench needs. --- rust/src/link_storage.rs | 38 +++++++++++++++++++++++++++-- rust/src/storage/decorator_impls.rs | 3 ++- rust/tests/link_storage_tests.rs | 34 ++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/rust/src/link_storage.rs b/rust/src/link_storage.rs index 6f86892..b4bfec0 100644 --- a/rust/src/link_storage.rs +++ b/rust/src/link_storage.rs @@ -91,7 +91,32 @@ impl LinkStorage { Ok(storage) } - /// The database file this storage reads from and writes to. + /// Creates a storage that lives only in memory: it never reads or writes + /// a file, and [`LinkStorage::save`] keeps nothing. + /// + /// This is the store of the browser workbench, where there is no file + /// system, so queries there merge, cascade and reuse addresses exactly as + /// they do in the CLI. + pub fn in_memory(trace: bool) -> Self { + Self { + links: HashMap::new(), + names: HashMap::new(), + name_to_id: HashMap::new(), + allocated: 0, + unused: Vec::new(), + db_path: PathBuf::new(), + revision: StorageRevision::default(), + trace, + } + } + + /// Whether this storage was made by [`LinkStorage::in_memory`]. + pub fn is_in_memory(&self) -> bool { + self.db_path.as_os_str().is_empty() + } + + /// The database file this storage reads from and writes to, empty for an + /// [in-memory](LinkStorage::in_memory) storage. pub fn database_path(&self) -> &Path { &self.db_path } @@ -105,6 +130,9 @@ impl LinkStorage { /// current on-disk state as "seen" for /// [`LinksStorage::has_external_changes`](crate::LinksStorage::has_external_changes). pub fn refresh_observed_revision(&mut self) -> Result<(), LinkError> { + if self.is_in_memory() { + return Ok(()); + } self.revision = StorageRevision::of(&self.db_path)?; Ok(()) } @@ -116,6 +144,9 @@ impl LinkStorage { self.name_to_id.clear(); self.allocated = 0; self.unused.clear(); + if self.is_in_memory() { + return Ok(()); + } if self.db_path.exists() { self.load()?; } @@ -282,8 +313,11 @@ impl LinkStorage { } } - /// Saves all links to the database file + /// Saves all links to the database file; an in-memory storage has none. pub fn save(&self) -> Result<()> { + if self.is_in_memory() { + return Ok(()); + } let file = OpenOptions::new() .write(true) .create(true) diff --git a/rust/src/storage/decorator_impls.rs b/rust/src/storage/decorator_impls.rs index cd09018..00addc6 100644 --- a/rust/src/storage/decorator_impls.rs +++ b/rust/src/storage/decorator_impls.rs @@ -150,7 +150,8 @@ impl_in_memory_links_storage!( storage.refresh_observed_revision() }, |storage: &LinkStorage| { - Ok(StorageRevision::of(storage.database_path())? != storage.observed_revision()) + Ok(!storage.is_in_memory() + && StorageRevision::of(storage.database_path())? != storage.observed_revision()) }, |storage: &mut LinkStorage| storage.reload_from_disk().map_err(storage_error) ); diff --git a/rust/tests/link_storage_tests.rs b/rust/tests/link_storage_tests.rs index af80859..b66f710 100644 --- a/rust/tests/link_storage_tests.rs +++ b/rust/tests/link_storage_tests.rs @@ -430,3 +430,37 @@ fn test_ensure_created_frees_the_addresses_it_passed_over() -> Result<()> { Ok(()) } + +/// The browser workbench has no file system, so an in-memory storage must work +/// without one: writing to its empty path would fail, so `save` and `flush` +/// succeed only because they write nothing. +#[test] +fn in_memory_storage_never_touches_a_file() -> Result<()> { + use link_cli::LinksStorage; + + let mut storage = LinkStorage::in_memory(false); + assert!(storage.is_in_memory()); + assert_eq!(storage.database_path(), std::path::Path::new("")); + + let point = storage.create(0, 0); + storage.update(point, point, point)?; + storage.set_name(point, "point"); + storage.save()?; + LinksStorage::::flush(&mut storage)?; + assert!(!LinksStorage::::has_external_changes(&storage)?); + assert_eq!(storage.lino_lines(), ["(point: point point)"]); + + // Reloading an in-memory storage has nothing to read, so it empties it. + LinksStorage::::reload(&mut storage)?; + assert!(storage.all().is_empty()); + assert_eq!(storage.get_by_name("point"), None); + Ok(()) +} + +#[test] +fn file_storage_is_not_in_memory() -> Result<()> { + let temp_file = NamedTempFile::new()?; + let storage = LinkStorage::new(temp_file.path(), false)?; + assert!(!storage.is_in_memory()); + Ok(()) +} From 931ffa5fd3143ed6ea1efb1cb2486e19f180a254 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 02:59:23 +0000 Subject: [PATCH 38/44] fix(wasm): run the browser workbench on LinkStorage::in_memory The WebAssembly crate stopped compiling (E0046) when NamedTypeLinks began requiring update_observed and delete_observed, which failed the WebAssembly and Docs workflows. Instead of growing its own simplified copy of the store, the workbench now uses the CLI's LinkStorage kept in memory, so deletes cascade to usages and updates merge into an existing pair exactly as in the CLI. Tests pin both behaviours natively and on wasm32. --- docs/ARCHITECTURE.md | 3 +- docs/HOW-IT-WORKS.md | 4 +- ...261005_210000_issue_105_browser_storage.md | 5 + rust/wasm/src/lib.rs | 215 ++++++------------ rust/wasm/tests/web.rs | 12 + 5 files changed, 94 insertions(+), 145 deletions(-) create mode 100644 rust/changelog.d/20261005_210000_issue_105_browser_storage.md diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ac5558d..63744ff 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -121,7 +121,8 @@ Key files: Runtime flow: 1. Vite loads the generated `clink-wasm` package. -2. `Clink` stores links in an in-memory `BrowserStorage`. +2. `Clink` stores links in `LinkStorage::in_memory`, the CLI's store without a + file, so deletes cascade and updates merge exactly as in the CLI. 3. Queries are passed into the Rust `QueryProcessor`. 4. The result includes formatted output plus a structured `links` snapshot. 5. React renders the snapshot and mirrors it into `doublets-web` `UnitedLinks`. diff --git a/docs/HOW-IT-WORKS.md b/docs/HOW-IT-WORKS.md index cf9ceb0..54df4d1 100644 --- a/docs/HOW-IT-WORKS.md +++ b/docs/HOW-IT-WORKS.md @@ -338,7 +338,9 @@ The WebAssembly workbench uses the Rust query processor in the browser. 1. The `rust/wasm` `clink-wasm` crate compiles with `wasm-pack`. 2. `Clink#execute(query, optionsJson)` parses JSON options. -3. Browser storage keeps links and names in memory for the page session. +3. `LinkStorage::in_memory` keeps links and names in memory for the page + session; it is the CLI's store without a file, so the browser and the CLI + give the same results. 4. The Rust query processor applies the LiNo query. 5. The result returns formatted output and a structured `links` array. 6. React renders the output and graph. diff --git a/rust/changelog.d/20261005_210000_issue_105_browser_storage.md b/rust/changelog.d/20261005_210000_issue_105_browser_storage.md new file mode 100644 index 0000000..e0ec0c4 --- /dev/null +++ b/rust/changelog.d/20261005_210000_issue_105_browser_storage.md @@ -0,0 +1,5 @@ +--- +bump: patch +--- + +Fixed the WebAssembly build, which broke when `NamedTypeLinks` started requiring `update_observed` and `delete_observed`. The browser workbench now runs on the new `LinkStorage::in_memory`, the CLI's store without a file, instead of its own simplified copy. In the browser, deleting a link now also deletes its usages, and updating a link into an existing pair merges the two, exactly as in the CLI. diff --git a/rust/wasm/src/lib.rs b/rust/wasm/src/lib.rs index 0236a0b..fcb10ad 100644 --- a/rust/wasm/src/lib.rs +++ b/rust/wasm/src/lib.rs @@ -1,7 +1,5 @@ -use std::collections::HashMap; - use anyhow::Result; -use link_cli::{Link, LinkError, NamedTypeLinks, QueryProcessor}; +use link_cli::{Link, LinkStorage, QueryProcessor}; use serde::{Deserialize, Serialize}; use wasm_bindgen::prelude::*; @@ -47,7 +45,8 @@ pub struct WebLink { #[wasm_bindgen] pub struct Clink { - storage: BrowserStorage, + /// The same store as the CLI's, kept in memory: the browser has no files. + storage: LinkStorage, } #[wasm_bindgen] @@ -56,7 +55,7 @@ impl Clink { pub fn new() -> Clink { set_panic_hook(); Clink { - storage: BrowserStorage::new(), + storage: LinkStorage::in_memory(false), } } @@ -68,7 +67,7 @@ impl Clink { success: false, output: String::new(), error: Some(error.to_string()), - links: self.storage.snapshot(), + links: snapshot(&self.storage), }, }) } @@ -77,15 +76,15 @@ impl Clink { pub fn snapshot(&mut self) -> String { to_json(&ClinkResult { success: true, - output: self.storage.lino_lines().unwrap_or_default().join("\n"), + output: self.storage.lino_lines().join("\n"), error: None, - links: self.storage.snapshot(), + links: snapshot(&self.storage), }) } #[wasm_bindgen] pub fn reset(&mut self) -> String { - self.storage = BrowserStorage::new(); + self.storage = LinkStorage::in_memory(false); self.snapshot() } @@ -122,7 +121,7 @@ impl Clink { } if options.before { - output.extend(self.storage.lino_lines()?); + output.extend(self.storage.lino_lines()); } if !query.trim().is_empty() { @@ -131,13 +130,13 @@ impl Clink { let changes = processor.process_query(&mut self.storage, query)?; if options.changes { for (before, after) in &changes { - output.push(self.storage.format_change(before, after)?); + output.push(format_change(&self.storage, before, after)); } } } if options.after { - output.extend(self.storage.lino_lines()?); + output.extend(self.storage.lino_lines()); } Ok(self.result(output, true, None)) @@ -148,7 +147,7 @@ impl Clink { success, output: output.join("\n"), error, - links: self.storage.snapshot(), + links: snapshot(&self.storage), } } } @@ -175,138 +174,27 @@ fn set_panic_hook() { console_error_panic_hook::set_once(); } -#[derive(Default)] -struct BrowserStorage { - links: HashMap, - names: HashMap, - name_to_id: HashMap, - next_id: u32, -} - -impl BrowserStorage { - fn new() -> Self { - Self { - next_id: 1, - ..Self::default() - } - } - - fn snapshot(&self) -> Vec { - let mut links: Vec<_> = self - .links - .values() - .map(|link| WebLink { - id: link.index, - source: link.source, - target: link.target, - name: self.names.get(&link.index).cloned(), - }) - .collect(); - links.sort_by_key(|link| link.id); - links - } - - fn format_change(&mut self, before: &Option, after: &Option) -> Result { - let before_text = before - .map(|link| self.format_lino(&link)) - .transpose()? - .unwrap_or_default(); - let after_text = after - .map(|link| self.format_lino(&link)) - .transpose()? - .unwrap_or_default(); - Ok(format!("({before_text}) ({after_text})")) - } +/// The links of `storage` with their names, ordered by address. +fn snapshot(storage: &LinkStorage) -> Vec { + storage + .all() + .into_iter() + .map(|link| WebLink { + id: link.index, + source: link.source, + target: link.target, + name: storage.get_name(link.index).cloned(), + }) + .collect() } -impl NamedTypeLinks for BrowserStorage { - fn create(&mut self, source: u32, target: u32) -> u32 { - let id = self.next_id.max(1); - self.next_id = id + 1; - self.links.insert(id, Link::new(id, source, target)); - id - } - - fn ensure_created(&mut self, id: u32) -> u32 { - if id == 0 || self.links.contains_key(&id) { - return id; - } - - self.next_id = self.next_id.max(id + 1); - self.links.insert(id, Link::new(id, 0, 0)); - id - } - - fn get_link(&mut self, id: u32) -> Option { - self.links.get(&id).copied() - } - - fn exists(&mut self, id: u32) -> bool { - self.links.contains_key(&id) - } - - fn update(&mut self, id: u32, source: u32, target: u32) -> Result { - let link = self.links.get_mut(&id).ok_or(LinkError::not_found(id))?; - let before = *link; - link.source = source; - link.target = target; - Ok(before) - } - - fn delete(&mut self, id: u32) -> Result { - self.remove_name(id)?; - self.links - .remove(&id) - .ok_or(LinkError::not_found(id).into()) - } - - fn all_links(&mut self) -> Vec { - self.links.values().copied().collect() - } - - fn search(&mut self, source: u32, target: u32) -> Option { - self.links - .values() - .find(|link| link.source == source && link.target == target) - .map(|link| link.index) - } - - fn get_or_create(&mut self, source: u32, target: u32) -> u32 { - self.search(source, target) - .unwrap_or_else(|| self.create(source, target)) - } - - fn get_name(&mut self, id: u32) -> Result> { - Ok(self.names.get(&id).cloned()) - } - - fn set_name(&mut self, id: u32, name: &str) -> Result { - if let Some(previous_name) = self.names.remove(&id) { - self.name_to_id.remove(&previous_name); - } - if let Some(previous_id) = self.name_to_id.insert(name.to_string(), id) { - if previous_id != id { - self.names.remove(&previous_id); - } - } - self.names.insert(id, name.to_string()); - Ok(id) - } - - fn get_by_name(&mut self, name: &str) -> Result> { - Ok(self.name_to_id.get(name).copied()) - } - - fn remove_name(&mut self, id: u32) -> Result<()> { - if let Some(name) = self.names.remove(&id) { - self.name_to_id.remove(&name); - } - Ok(()) - } - - fn save(&mut self) -> Result<()> { - Ok(()) - } +/// A change as `(before) (after)`, each side in LiNo, empty when absent. +fn format_change(storage: &LinkStorage, before: &Option, after: &Option) -> String { + let format = |link: &Option| { + link.map(|link| storage.format_lino(&link)) + .unwrap_or_default() + }; + format!("({}) ({})", format(before), format(after)) } #[cfg(test)] @@ -328,6 +216,47 @@ mod tests { assert_eq!(parsed["links"].as_array().unwrap().len(), 3); } + fn run(clink: &mut Clink, query: &str) -> String { + let raw = clink.execute(query, r#"{"changes":true,"after":true}"#); + let parsed: Value = serde_json::from_str(&raw).unwrap(); + assert_eq!(parsed["success"], true, "{raw}"); + parsed["output"].as_str().unwrap().to_string() + } + + #[test] + fn deleting_a_link_deletes_its_usages_as_the_cli_does() { + let mut clink = Clink::new(); + run(&mut clink, "() ((1 1) (2 2) (1 2))"); + + assert_eq!( + run(&mut clink, "((2: 2 2)) ()"), + "((2: 2 2)) ()\n((3: 1 2)) ()\n(1: 1 1)" + ); + } + + #[test] + fn updating_a_link_into_an_existing_pair_merges_them_as_the_cli_does() { + let mut clink = Clink::new(); + run(&mut clink, "() ((1 1) (2 2) (1 2) (2 1))"); + + assert_eq!( + run(&mut clink, "(((4: 2 1)) ((4: 1 2)))"), + "((4: 2 1)) ()\n(1: 1 1)\n(2: 2 2)\n(3: 1 2)" + ); + } + + #[test] + fn reset_empties_the_store() { + let mut clink = Clink::new(); + run(&mut clink, "() ((named: named named))"); + let parsed: Value = serde_json::from_str(&clink.snapshot()).unwrap(); + assert_eq!(parsed["links"][0]["name"], "named"); + + let parsed: Value = serde_json::from_str(&clink.reset()).unwrap(); + assert_eq!(parsed["links"].as_array().unwrap().len(), 0); + assert_eq!(parsed["output"], ""); + } + #[test] fn rejects_invalid_options() { let mut clink = Clink::new(); diff --git a/rust/wasm/tests/web.rs b/rust/wasm/tests/web.rs index c2629b5..2c96716 100644 --- a/rust/wasm/tests/web.rs +++ b/rust/wasm/tests/web.rs @@ -69,6 +69,18 @@ fn javascript_wasm_api_round_trips_create_update_delete_and_recreate() { assert_link(&recreated, 1, 1, 1); } +#[wasm_bindgen_test] +fn deleting_a_link_deletes_its_usages_as_the_cli_does() { + let mut clink = Clink::new(); + execute_json(&mut clink, "() ((1 1) (2 2) (1 2))"); + + let deleted = execute_json(&mut clink, "((2: 2 2)) ()"); + assert_eq!(deleted["success"], true); + assert_eq!(deleted["output"], "((2: 2 2)) ()\n((3: 1 2)) ()\n(1: 1 1)"); + assert_link(&deleted, 1, 1, 1); + assert_link_missing(&deleted, 3); +} + fn execute_json(clink: &mut Clink, query: &str) -> Value { let raw = clink.execute( query, From 7f5e00a7c2d5c37174a5fda0bb5b2bb1cfd63a89 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 03:11:33 +0000 Subject: [PATCH 39/44] docs: record the upstream doublets and mem-rs issues found while testing CRUD --- docs/case-studies/issue-105/README.md | 30 +++++++++++++++++++++++++++ docs/case-studies/issue-98/README.md | 3 ++- rust/src/storage/file_mem.rs | 2 ++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/case-studies/issue-105/README.md b/docs/case-studies/issue-105/README.md index 56bcd20..f6a9512 100644 --- a/docs/case-studies/issue-105/README.md +++ b/docs/case-studies/issue-105/README.md @@ -441,6 +441,36 @@ encoders write 3. That is reported as To avoid a third format, a shared spec in links-notation is proposed in [links-notation#325](https://github.com/link-foundation/links-notation/issues/325). +### 6.5 Bugs in the doublets libraries + +Proving every CRUD operation end to end meant also checking the stores +underneath. Small repro programs against doublets-rs 0.5.0, platform-mem 0.3.0 +and Platform.Data.Doublets 0.18.1 found these bugs, each confirmed on the +latest commit and reported upstream: + +| Library | Issue | Bug | +|---|---|---| +| doublets-rs | [#62](https://github.com/linksplatform/doublets-rs/issues/62) | `rebase_with` leaves `(4: 3 3)` as `(4: 3 5)`, not `(4: 5 5)` | +| doublets-rs | [#63](https://github.com/linksplatform/doublets-rs/issues/63) | `usages` lists `(v: v v)`-style usages twice; `delete_usages` and `delete_query_with` then fail with `NotExists` | +| doublets-rs | [#64](https://github.com/linksplatform/doublets-rs/issues/64) | `delete`, `delete_by` and `rebase_and_delete` return `0` | +| doublets-rs | [#65](https://github.com/linksplatform/doublets-rs/issues/65) | `delete_all` fails on a store with holes | +| doublets-rs | [#66](https://github.com/linksplatform/doublets-rs/issues/66) | the resolved `update` of a missing link returns `Ok` when the new pair exists | +| doublets-rs | [#67](https://github.com/linksplatform/doublets-rs/issues/67) | cascaded deletes skip the `(0 0)` reset that C# reports | +| mem-rs | [#36](https://github.com/linksplatform/mem-rs/issues/36) | `grow_filled` overwrites a `FileMapped` file, so a reopened store is empty (found in issue #98) | +| Data.Doublets | [#517](https://github.com/linksplatform/Data.Doublets/issues/517) | `Update` of a missing or deleted address reports success and corrupts the indexes | +| Data.Doublets | [#518](https://github.com/linksplatform/Data.Doublets/issues/518) | `Delete` of a missing address double-frees, or throws `NullReferenceException` when decorated | +| Data.Doublets | [#519](https://github.com/linksplatform/Data.Doublets/issues/519) | `CreateAndUpdate` returns a deleted address when the pair already exists | +| Data.Doublets | [#520](https://github.com/linksplatform/Data.Doublets/issues/520) | `Count` and `Each` with `[Any, v]` see `(v: v v)` twice | +| Data.Doublets | [#521](https://github.com/linksplatform/Data.Doublets/issues/521) | `Count(null)` and `Each(null)` throw `NullReferenceException` | +| Data.Doublets | [#338](https://github.com/linksplatform/Data.Doublets/issues/338#issuecomment-5987416982) | a full `Each` scan returns `Break` (existing issue, repro added) | + +This repository steers around them: +- the Rust storage never calls the affected rebase, usages or `delete_all` helpers; +- `LinkStorage::update_observed` and `delete_observed` check that the link exists before calling the resolved stack; +- the C# query processors call `SearchOrDefault` before `CreateAndUpdate`, so they never create an existing pair; +- `--changes` collapses each link's chain of changes (`changes_simplifier.rs`), so the `(0 0)` steps that doublets-rs#67 skips never appear in its report; +- `PersistentFileMapped` works around mem-rs#36. + ## 7. Risks and remaining limits - **No authentication or encryption.** The server is meant for trusted diff --git a/docs/case-studies/issue-98/README.md b/docs/case-studies/issue-98/README.md index 198dc76..333640e 100644 --- a/docs/case-studies/issue-98/README.md +++ b/docs/case-studies/issue-98/README.md @@ -254,7 +254,8 @@ same matrix in-process on all three CI operating systems. - **Upstream `grow_filled` bug.** `PersistentFileMapped` is a local workaround for a data-loss bug in `platform-mem`'s default - `RawMem::grow_filled`. It should be reported upstream; until it is fixed, + `RawMem::grow_filled`. It is reported upstream as + [mem-rs#36](https://github.com/linksplatform/mem-rs/issues/36); until it is fixed, any consumer that constructs a `FileMapped`-backed `doublets` store *without* this wrapper will lose data on reopen. The reproduction lives in [`evidence/doublets_persistence.rs`](evidence/doublets_persistence.rs). diff --git a/rust/src/storage/file_mem.rs b/rust/src/storage/file_mem.rs index 69c5daf..736cf9e 100644 --- a/rust/src/storage/file_mem.rs +++ b/rust/src/storage/file_mem.rs @@ -19,6 +19,8 @@ //! existing file-mapped `doublets` database zeroes it: every link is //! lost. `docs/case-studies/issue-98/evidence/doublets_persistence.rs` //! reproduces this against upstream `doublets` directly. +//! The bug is reported upstream as +//! . //! //! [`PersistentFileMapped`] fixes this by forwarding to //! `RawMem::grow_filled_exact`, which fills only `uninit[inited..]` and From cbaa1dcbafdfb49a81deb1d7914c5b56f1e11536 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 03:25:41 +0000 Subject: [PATCH 40/44] test(csharp): cover every protocol line; malformed packet tests assert the exact error - AddressTier throws for an address beyond the internal range - drop the unused LinoFormat.IsReference - a stopping server answers 'server is shutting down' instead of hanging up - trace 'client hung up' on a clean close - three malformed-packet cases passed for the wrong reason; every case now checks its detail --- .../.changeset/issue-105-protocol-coverage.md | 5 + ...oundation.Data.Doublets.Cli.Library.csproj | 4 + .../Protocol/LinksPacket.cs | 6 +- .../Protocol/LinksServer.cs | 11 +- .../Protocol/LinoFormat.cs | 3 - .../Protocol/LinoProtocols.cs | 8 +- .../Protocol/BinaryLinksNotationTests.cs | 66 ++++++---- .../Protocol/LinksServerTests.cs | 121 ++++++++++++++++-- .../Protocol/LinoProtocolCodecTests.cs | 34 ++++- .../Protocol/RemoteLinksTests.cs | 17 +++ 10 files changed, 219 insertions(+), 56 deletions(-) create mode 100644 csharp/.changeset/issue-105-protocol-coverage.md diff --git a/csharp/.changeset/issue-105-protocol-coverage.md b/csharp/.changeset/issue-105-protocol-coverage.md new file mode 100644 index 0000000..5c2a417 --- /dev/null +++ b/csharp/.changeset/issue-105-protocol-coverage.md @@ -0,0 +1,5 @@ +--- +'Foundation.Data.Doublets.Cli': patch +--- + +`LinksPacket.AddressTier` now throws an unencodable `LinoProtocolException` for an address beyond the internal range instead of silently returning width 8, and the unused `LinoFormat.IsReference` was removed. A server that is stopping now answers a request it already read with `(error: 'server is shutting down')` instead of hanging up, and its trace notes when a client hangs up. Tests now cover every line of the protocol code, and each malformed-packet test asserts the exact error, which fixed three cases that passed for the wrong reason. diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Foundation.Data.Doublets.Cli.Library.csproj b/csharp/Foundation.Data.Doublets.Cli.Library/Foundation.Data.Doublets.Cli.Library.csproj index 9edfa20..6ccb5d0 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Foundation.Data.Doublets.Cli.Library.csproj +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Foundation.Data.Doublets.Cli.Library.csproj @@ -36,4 +36,8 @@ + + + + diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs index 3b1ac45..1881753 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs @@ -227,7 +227,7 @@ public static byte AddressTier(ulong address, bool externalReferences) return width; } } - return 8; + throw LinoProtocolException.Unencodable($"address {address} exceeds the internal range"); } private static ulong WidthMask(byte width) => InternalCapacity(width, false); @@ -269,10 +269,6 @@ public static byte ReferenceWidth(PacketReference reference, bool externalRefere { if (!reference.IsExternal) { - if (reference.Value > InternalCapacity(8, externalReferences)) - { - throw LinoProtocolException.Unencodable($"address {reference.Value} exceeds the internal range"); - } return AddressTier(reference.Value, externalReferences); } if (!externalReferences) diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs index d4f4b0c..3caa6ae 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs @@ -172,14 +172,11 @@ private void HandleConnection(TcpClient client, INamedTypesLinks links) } if (message is not { } received) { + Trace("client hung up"); return; } var (document, format) = received; var reply = Accepts(format) ? Execute(links, document) : ErrorDocument("this server does not accept this protocol"); - if (reply is null) - { - return; - } format.Protocol(_options.Limits).WriteDocument(stream, reply); } } @@ -198,7 +195,7 @@ private void HandleConnection(TcpClient client, INamedTypesLinks links) } /// Sends every reply at once; a connection that refuses still works, only slower, as in the Rust server. - private void TryDisableNagle(TcpClient client) + internal void TryDisableNagle(TcpClient client) { try { @@ -217,14 +214,14 @@ private void TryDisableNagle(TcpClient client) _ => true, }; - private IReadOnlyList? Execute(INamedTypesLinks links, IReadOnlyList document) + internal IReadOnlyList Execute(INamedTypesLinks links, IReadOnlyList document) { lock (_storeLock) { // No request may touch the store once Serve is returning. if (_stopping) { - return null; + return ErrorDocument("server is shutting down"); } Trace($"request: {LinoFormat.FormatDocument(document)}"); var reply = ExecuteRequest(links, document, _options.AutoCreateMissingReferences); diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs index 54c1dad..c9edb4d 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoFormat.cs @@ -27,9 +27,6 @@ public static class LinoFormat /// A link with an optional id and the given values. public static LinoLink Link(string? id, IList values) => new(id!, values); - /// True when is a reference. - public static bool IsReference(LinoLink link) => link.Values is null; - /// Parses LiNo text into a canonical document. Blank input is the empty document. public static IReadOnlyList ParseDocument(string text) { diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs index 27869a0..bdb6a0b 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinoProtocols.cs @@ -202,11 +202,9 @@ public static (IReadOnlyList Document, MessageFormat Format)? ReadAnyD ? (text, MessageFormat.Text) : null; } - if (LinksPacket.ReadFrom(input, limits) is not { } packet) - { - return null; - } - return (LinoMapping.DecodeDocument(packet, limits), MessageFormat.Binary(BinaryLinoOptions.OfPacket(packet))); + return LinksPacket.ReadFrom(input, limits) is { } packet + ? (LinoMapping.DecodeDocument(packet, limits), MessageFormat.Binary(BinaryLinoOptions.OfPacket(packet))) + : null; } } diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs index f85fb55..cbe318b 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs @@ -113,6 +113,9 @@ public void WidthTiersFollowTheNumberOfLinks() Assert.Equal(2, LinksPacket.AddressTier(128, true)); Assert.Equal(2, LinksPacket.AddressTier(32_767, true)); Assert.Equal(4, LinksPacket.AddressTier(32_768, true)); + Assert.Equal(8, LinksPacket.AddressTier(ulong.MaxValue, false)); + // No width holds an internal address in the external half. + Assert.Equal(LinoProtocolErrorKind.Unencodable, Assert.Throws(() => LinksPacket.AddressTier(1UL << 63, true)).Kind); Assert.Equal(ulong.MaxValue, LinksPacket.InternalCapacity(8, false)); Assert.Equal((ulong)long.MaxValue, LinksPacket.InternalCapacity(8, true)); Assert.Equal(127UL, LinksPacket.ExternalCapacity(1)); @@ -354,33 +357,43 @@ static void Unencodable(Section section) Unencodable(DoubletSection(ulong.MaxValue, ArityRange.Doublets, 1, (1, 1))); } - private static LinoProtocolErrorKind DecodeError(byte[] bytes, DecodeLimits? limits = null) + private static LinoProtocolException DecodeError(byte[] bytes, DecodeLimits? limits = null) { var protocol = new BinaryLinoProtocol { Limits = limits ?? DecodeLimits.Default }; - return Assert.Throws(() => protocol.Decode(bytes)).Kind; + return Assert.Throws(() => protocol.Decode(bytes)); } + /// Each packet breaks exactly one rule, the one its detail names. [Theory] - [InlineData("")] - [InlineData("20 00")] // unknown version nibble - [InlineData("10 01 06 00")] // refers to itself - [InlineData("10 01 07 00")] // refers forward - [InlineData("10 02 00")] // truncated - [InlineData("10 00 00")] // trailing byte - [InlineData("10 ff ff ff ff ff ff ff ff ff 7f")] // LEB128 overflow - [InlineData("10 01 01 01")] // root is not a list - [InlineData("10 02 00 02 04 06")] // marker 2 used as a value - [InlineData("10 01 04 02")] // a bare marker in a list chain - [InlineData("16 00")] // the explicit layout keeps the width bits clear - [InlineData("12 01 04 05 00")] // arity 0 - [InlineData("12 01 18 01 01 05 00")] // length 6 in arity 1..2 - [InlineData("12 01 f8 ff ff ff ff ff ff ff ff 0f ff ff ff ff ff ff ff ff ff 01 00")] // arity overflows - [InlineData("12 02 04 ff ff ff ff ff ff ff ff ff 01 01 20 01")] // addresses overflow - [InlineData("12 01 24 06 01 01 01")] // LiNo links start at 6, not 7 - [InlineData("12 02 24 05 01 24 01 01 01 01 04 06")] // a hole - public void MalformedPacketsAreRejected(string hex) + [InlineData("", "empty input")] + [InlineData("20 00", "unsupported binary header byte 0x20")] + [InlineData("10 01 06 00", "link 6 refers to Internal(6)")] + [InlineData("10 01 07 00", "link 6 refers to Internal(7)")] + [InlineData("10 02 00", "unexpected end of packet")] + [InlineData("10 00 00", "trailing bytes after the packet")] + [InlineData("10 ff ff ff ff ff ff ff ff ff 7f", "LEB128 value overflows 64 bits")] // a tenth byte above 1 + [InlineData("10 80 80 80 80 80 80 80 80 80 81", "LEB128 value overflows 64 bits")] // an eleventh byte + [InlineData("10 01 01 01", "the root link is not a list")] + [InlineData("10 02 00 02 04 06", "broken element chain")] // the chain ends in marker 2, not null + [InlineData("10 01 04 02", "broken element chain")] + [InlineData("10 02 02 00 04 06", "marker 2 used as a value")] + [InlineData("10 03 01 00 06 00 04 07", "marker 1 cannot start a typed value")] + [InlineData("12 02 14 05 01 20 02 02 06 00 04 07", "a number needs exactly one value")] // (Number) alone + [InlineData("10 03 05 00 06 00 04 07", "an identified link needs an id")] + [InlineData("10 04 00 00 05 06 07 00 04 08", "a link id must be a reference")] // the id is () + [InlineData("16 00", "the explicit layout keeps the header width bits clear")] + [InlineData("12 01 04 05 00", "arity must be at least 1")] + [InlineData("12 01 18 01 01 05 00", "link length outside the section arity 1..2")] // a link of 6 + [InlineData("12 01 f8 ff ff ff ff ff ff ff ff 01 ff ff ff ff ff ff ff ff ff 01 00", "arity range overflows 64 bits")] + [InlineData("12 01 24 ff ff ff ff ff ff ff ff ff 01 01", "addresses overflow 64 bits")] // the gap + [InlineData("12 01 20 ff ff ff ff ff ff ff ff ff 01", "addresses overflow 64 bits")] // the count + [InlineData("12 01 24 06 01 01 01", "from address 6, found link 7 where 6 belongs")] + [InlineData("12 02 24 05 01 24 01 01 01 01 04 06", "found link 8 where 7 belongs")] // a hole + public void MalformedPacketsAreRejected(string hex, string detail) { - Assert.Equal(LinoProtocolErrorKind.Malformed, DecodeError(Unhex(hex))); + var error = DecodeError(Unhex(hex)); + Assert.Equal(LinoProtocolErrorKind.Malformed, error.Kind); + Assert.Contains(detail, error.Detail, StringComparison.Ordinal); } private static List<(ulong Address, PacketReference[] References)> FromAddress6(params PacketReference[][] links) => @@ -395,14 +408,14 @@ public void InvalidCodePointsAreRejected() Internals(3, 6), new[] { PacketReference.Internal(7), PacketReference.Null }, Internals(4, 8)); - Assert.Equal(LinoProtocolErrorKind.Malformed, DecodeError(LinksPacket.Pack(true, links, false).ToBytes())); + Assert.Equal("invalid code point 1114112", DecodeError(LinksPacket.Pack(true, links, false).ToBytes()).Detail); } [Fact] public void HostilePacketsHitLimits() { void Limited(string hex, DecodeLimits limits) => - Assert.True(LinoProtocolErrorKind.LimitExceeded == DecodeError(Unhex(hex), limits), hex); + Assert.True(LinoProtocolErrorKind.LimitExceeded == DecodeError(Unhex(hex), limits).Kind, hex); var fewLinks = new DecodeLimits { MaxLinks = 8 }; Limited("10 09", fewLinks); Limited("12 09", fewLinks); // more sections than links allowed @@ -421,11 +434,14 @@ void Limited(string hex, DecodeLimits limits) => chain.Add(new[] { PacketReference.Internal(last), PacketReference.Null }); chain.Add(Internals(4, last + 1)); var bomb = LinksPacket.Pack(false, FromAddress6(chain.ToArray()), false); - Assert.Equal(LinoProtocolErrorKind.LimitExceeded, DecodeError(bomb.ToBytes(), new DecodeLimits { MaxNodes = 1 << 12 })); + Assert.Equal("limit exceeded: too many LiNo nodes", DecodeError(bomb.ToBytes(), new DecodeLimits { MaxNodes = 1 << 12 }).Message); var deep = string.Concat(Enumerable.Repeat("(y ", 40)) + "x" + new string(')', 40); var bytes = new BinaryLinoProtocol().Encode(Parse(deep)); - Assert.Equal(LinoProtocolErrorKind.LimitExceeded, DecodeError(bytes, new DecodeLimits { MaxDepth = 10 })); + Assert.Equal("limit exceeded: nesting deeper than 10", DecodeError(bytes, new DecodeLimits { MaxDepth = 10 }).Message); + + var threeLinks = new BinaryLinoProtocol().Encode(Parse("a b c")); + Assert.Equal("limit exceeded: chain too long", DecodeError(threeLinks, new DecodeLimits { MaxNodes = 2 }).Message); Assert.Equal(ulong.MaxValue, DecodeLimits.Unlimited.MaxLinks); } } diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs index 17aeec0..2a04aef 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinksServerTests.cs @@ -92,15 +92,22 @@ public void MalformedMessagesGetAnErrorAndTheConnectionCloses() Assert.EndsWith("\n.\n", reply, StringComparison.Ordinal); } - [Fact] - public void ServersCanRestrictTheAcceptedProtocol() + [Theory] + [InlineData(AcceptedProtocols.Binary)] + [InlineData(AcceptedProtocols.Text)] + public void ServersCanRestrictTheAcceptedProtocol(AcceptedProtocols accept) { - using var server = new RunningServer(new LinksServerOptions { Accept = AcceptedProtocols.Binary }); - using var text = server.Client(new TextLinoProtocol()); - var error = Assert.Throws(() => text.Query("() ((1 1))")); - Assert.Equal(LinoProtocolErrorKind.Remote, error.Kind); - using var binary = server.Client(new BinaryLinoProtocol()); - Assert.Equal("() ((1: 1 1))", binary.QueryText("() ((1 1))")); + using var server = new RunningServer(new LinksServerOptions { Accept = accept }); + ILinoProtocol accepted = accept == AcceptedProtocols.Binary ? new BinaryLinoProtocol() : new TextLinoProtocol(); + ILinoProtocol refused = accept == AcceptedProtocols.Binary ? new TextLinoProtocol() : new BinaryLinoProtocol(); + using (var client = server.Client(refused)) + { + var error = Assert.Throws(() => client.Query("() ((1 1))")); + Assert.Equal(LinoProtocolErrorKind.Remote, error.Kind); + Assert.Equal("this server does not accept this protocol", error.Detail); + } + using var allowed = server.Client(accepted); + Assert.Equal("() ((1: 1 1))", allowed.QueryText("() ((1 1))")); } [Fact] @@ -145,6 +152,104 @@ public void ShuttingDownRightAfterAConnectionArrivesIsClean() } } + [Fact] + public void ServersBindToAnEndPointAndClientsConnectToIt() + { + using var server = LinksServer.Bind("127.0.0.1:0"); + var endPoint = $"127.0.0.1:{server.LocalEndPoint.Port}"; + using var client = LinksClient.Connect(endPoint, new TextLinoProtocol()); + + Assert.IsType(client.Protocol); + } + + [Fact] + public void ConnectingToAClosedPortIsAnIoError() + { + int port; + using (var server = LinksServer.Bind("127.0.0.1", 0)) + { + port = server.LocalEndPoint.Port; + } + + var error = Assert.Throws(() => LinksClient.Connect("127.0.0.1", port, new TextLinoProtocol())); + Assert.Equal(LinoProtocolErrorKind.Io, error.Kind); + } + + [Fact] + public void AClientThatHangsUpLeavesTheServerServing() + { + using var server = new RunningServer(); + using (var connection = RawConnection(server)) + { + connection.GetStream().Write("() ((1 1))\n.\n"u8); + using var reader = new StreamReader(connection.GetStream(), new UTF8Encoding(false)); + Assert.Equal("() ((1: 1 1))", reader.ReadLine()); + } + using var client = server.Client(new TextLinoProtocol()); + Assert.Equal("(1: 1 1)", client.QueryText("")); + } + + [Fact] + public void TracingLogsRequestsRepliesAndSocketProblems() + { + using var log = new StringWriter(); + var standardError = Console.Error; + Console.SetError(log); + try + { + using (var server = new RunningServer(new LinksServerOptions { Trace = true })) + { + using (var client = server.Client(new TextLinoProtocol())) + { + client.Query("() ((1 1))"); + } + WaitForLog(log, "[server] client hung up"); + } + using var bound = LinksServer.Bind("127.0.0.1", 0, new LinksServerOptions { Trace = true }); + // Nagle's algorithm is a TCP option: a UDP socket refuses it. + using var datagram = new TcpClient { Client = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp) }; + bound.TryDisableNagle(datagram); + } + finally + { + Console.SetError(standardError); + } + + var lines = log.ToString(); + Assert.Contains("[server] request: () ((1 1))", lines, StringComparison.Ordinal); + Assert.Contains("[server] reply: () ((1: 1 1))", lines, StringComparison.Ordinal); + Assert.Contains("[server] could not disable Nagle's algorithm: ", lines, StringComparison.Ordinal); + } + + /// Waits until the server, on its own thread, has logged . + private static void WaitForLog(StringWriter log, string line) + { + var deadline = DateTime.UtcNow.AddSeconds(10); + while (true) + { + // Console.SetError synchronizes writes on the writer Console.Error returns. + lock (Console.Error) + { + if (log.ToString().Contains(line, StringComparison.Ordinal)) + { + return; + } + } + Assert.True(DateTime.UtcNow < deadline, $"the server never logged '{line}'"); + Thread.Sleep(10); + } + } + + [Fact] + public void AStoppedServerExecutesNothing() + { + using var server = LinksServer.Bind("127.0.0.1", 0); + server.Shutdown(); + // A request read just before the shutdown must not reach the store, so the store is never touched. + var reply = server.Execute(null!, LinoFormat.ParseDocument("() ((1 1))")); + Assert.Equal("server is shutting down", LinksServer.ErrorMessage(reply)); + } + [Fact] public void EndPointsAreParsed() { diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs index fed293f..94566c0 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs @@ -173,9 +173,12 @@ public void TextMessagesAcceptCrlfAndRejectTruncation() Assert.Empty(new TextLinoProtocol().ReadDocument(LinoStreamReader.FromBytes(".\n"u8.ToArray()))!); - var truncated = LinoStreamReader.FromBytes("() ((1 1))\n"u8.ToArray()); - var error = Assert.Throws(() => new TextLinoProtocol().ReadDocument(truncated)); - Assert.Equal(LinoProtocolErrorKind.Malformed, error.Kind); + foreach (var truncatedText in new[] { "() ((1 1))\n", "() ((1 1))" }) + { + var truncated = LinoStreamReader.FromBytes(System.Text.Encoding.UTF8.GetBytes(truncatedText)); + var error = Assert.Throws(() => new TextLinoProtocol().ReadDocument(truncated)); + Assert.Equal("stream ended before the '.' terminator line", error.Detail); + } var tiny = new TextLinoProtocol { Limits = new DecodeLimits { MaxTextBytes = 8 } }; var longMessage = LinoStreamReader.FromBytes(System.Text.Encoding.UTF8.GetBytes(new string('a', 100) + "\n.\n")); @@ -185,6 +188,31 @@ public void TextMessagesAcceptCrlfAndRejectTruncation() Assert.Equal(LinoProtocolErrorKind.Malformed, Assert.Throws(() => new TextLinoProtocol().ReadDocument(invalidUtf8)).Kind); } + [Fact] + public void AClosedStreamIsAnIoError() + { + var stream = new MemoryStream("() ()\n.\n"u8.ToArray()); + var reader = new LinoStreamReader(stream); + stream.Dispose(); + + var error = Assert.Throws(() => new TextLinoProtocol().ReadDocument(reader)); + Assert.Equal(LinoProtocolErrorKind.Io, error.Kind); + Assert.IsType(error.InnerException); + } + + [Fact] + public void ProtocolErrorsNameTheirKind() + { + Assert.Equal("malformed message: unknown protocol error", new LinoProtocolException().Message); + Assert.Equal(LinoProtocolErrorKind.Malformed, new LinoProtocolException("bad").Kind); + var inner = new IOException("reset"); + var io = new LinoProtocolException("reset", inner); + Assert.Equal((LinoProtocolErrorKind.Io, "I/O error: reset", inner), (io.Kind, io.Message, io.InnerException)); + Assert.Equal("invalid LiNo: x", new LinoProtocolException(LinoProtocolErrorKind.InvalidLino, "x").Message); + Assert.Equal("server error: x", new LinoProtocolException(LinoProtocolErrorKind.Remote, "x").Message); + Assert.Equal("42: x", new LinoProtocolException((LinoProtocolErrorKind)42, "x").Message); + } + [Fact] public void ProtocolsAreDetectedPerMessage() { diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RemoteLinksTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RemoteLinksTests.cs index 9a752ac..5c8ee66 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RemoteLinksTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/RemoteLinksTests.cs @@ -279,6 +279,17 @@ public void ServerFailuresAreProtocolExceptions() Assert.Equal(0u, remote.Count(null)); } + [Fact] + public void RemoteStoresConnectByEndPoint() + { + using var server = new RunningServer(); + var protocol = new BinaryLinoProtocol(); + using var remote = RemoteLinks.Connect($"127.0.0.1:{server.Port}", protocol); + + Assert.Equal(1u, remote.Create()); + Assert.Equal(1u, remote.Count(null)); + } + [Fact] public void ALostConnectionThrows() { @@ -357,6 +368,8 @@ public void MalformedRepliesAreRejected(string reply) var document = LinoFormat.ParseDocument(reply); Assert.Throws(() => LinksOperation.ParseCount(document)); Assert.Throws(() => LinksOperation.ParseChanges(document)); + Assert.Throws(() => LinksOperation.ParseName(document)); + Assert.Throws(() => LinksOperation.ParseLinkReply(document)); } [Fact] @@ -369,6 +382,10 @@ public void RepliesParseBackToWhatTheyCarry() Assert.Equal( new[] { (default(DoubletLink), new DoubletLink(1, 0, 0)), (new DoubletLink(1, 0, 0), default) }, LinksOperation.ParseChanges(LinoFormat.ParseDocument("() ((1: 0 0))\n((1: 0 0)) ()"))); + // A side that is a single named link may lose its wrapper. + Assert.Equal( + new[] { (new DoubletLink(1, 1, 1), new DoubletLink(1, 1, 2)) }, + LinksOperation.ParseChanges(LinoFormat.ParseDocument("(1: 1 1) (1: 1 2)"))); Assert.Equal("a name", LinksOperation.ParseName(LinoFormat.ParseDocument("(name: 'a name')"))); Assert.Null(LinksOperation.ParseName(Array.Empty>())); Assert.Equal(5u, LinksOperation.ParseLinkReply(LinoFormat.ParseDocument("(link: 5)"))); From f171858ddec4408cb5016c40aecd38b3cbc74749 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 03:54:21 +0000 Subject: [PATCH 41/44] test(rust): cover every protocol line; malformed tests assert the exact error - address_tier fails as unencodable beyond the internal range - a stopping server replies (error: "server is shutting down") to a read request - operation errors print links in LiNo, not Debug syntax - width_from_code is total over two-bit codes; dead fallbacks removed --- ...1005_230000_issue_105_protocol_coverage.md | 5 + rust/src/protocol/archive.rs | 8 +- rust/src/protocol/links_operations.rs | 29 +- rust/src/protocol/mapping.rs | 22 +- rust/src/protocol/packet.rs | 53 ++-- rust/src/protocol/protocols.rs | 15 +- rust/src/protocol/server.rs | 26 +- rust/tests/protocol_packet_tests.rs | 288 ++++++++++++++---- rust/tests/protocol_tcp_tests.rs | 90 +++++- rust/tests/remote_links_tests.rs | 180 ++++++++++- rust/tests/store_archive_tests.rs | 23 +- 11 files changed, 587 insertions(+), 152 deletions(-) create mode 100644 rust/changelog.d/20261005_230000_issue_105_protocol_coverage.md diff --git a/rust/changelog.d/20261005_230000_issue_105_protocol_coverage.md b/rust/changelog.d/20261005_230000_issue_105_protocol_coverage.md new file mode 100644 index 0000000..333508a --- /dev/null +++ b/rust/changelog.d/20261005_230000_issue_105_protocol_coverage.md @@ -0,0 +1,5 @@ +--- +bump: patch +--- + +`protocol::packet::address_tier` now returns a `ProtocolResult` and fails as unencodable for an address beyond the internal range instead of silently returning width 8. A server that is stopping now answers a request it already read with `(error: "server is shutting down")` instead of hanging up, and its trace notes when a client hangs up. Malformed-operation errors print the offending link in LiNo instead of Rust `Debug` syntax, and the trailing-bytes error no longer depends on the byte count. Tests now cover every line of the protocol code: each malformed-packet test asserts the exact error, which fixed cases that passed for the wrong reason, and new tests cover misshapen `name`/`link` replies, archive file errors and a stopped server. diff --git a/rust/src/protocol/archive.rs b/rust/src/protocol/archive.rs index 4698448..ad7b49d 100644 --- a/rust/src/protocol/archive.rs +++ b/rust/src/protocol/archive.rs @@ -154,10 +154,8 @@ fn decode_name(references: &[Reference]) -> ProtocolResult<(u32, String)> { )), }) .collect::>>()?; - let (&address, code_points) = values - .split_first() - .ok_or_else(|| ProtocolError::malformed("an archive name needs a link address"))?; - let name = code_points + // A packet link holds at least one reference: arity 0 is malformed. + let name = values[1..] .iter() .map(|&code_point| { u32::try_from(code_point) @@ -166,5 +164,5 @@ fn decode_name(references: &[Reference]) -> ProtocolResult<(u32, String)> { .ok_or_else(|| ProtocolError::malformed(format!("invalid code point {code_point}"))) }) .collect::>()?; - Ok((store_address(address)?, name)) + Ok((store_address(values[0])?, name)) } diff --git a/rust/src/protocol/links_operations.rs b/rust/src/protocol/links_operations.rs index c679109..487b210 100644 --- a/rust/src/protocol/links_operations.rs +++ b/rust/src/protocol/links_operations.rs @@ -32,6 +32,7 @@ //! differ between the C# and the Rust stores. use super::error::{ProtocolError, ProtocolResult}; +use super::format::{format_document, format_link}; use super::mapping::LinoDocument; use crate::changes_simplifier::simplify_changes; use crate::link::Link; @@ -158,10 +159,8 @@ impl LinksOperation { .map(link_lino) .collect(), Self::Create { source, target } => { - let index = storage.create(*source, *target); - let created = storage - .get_link(index) - .unwrap_or_else(|| Link::new(index, *source, *target)); + // `create` makes exactly `(index: source target)`. + let created = Link::new(storage.create(*source, *target), *source, *target); storage.save()?; changes_document(&[(Link::null(), created)]) } @@ -327,7 +326,8 @@ fn parse_change_side(side: &LiNo) -> ProtocolResult { // single named link itself. link @ LiNo::Link { id: Some(_), .. } => parse_link(link), _ => Err(ProtocolError::malformed(format!( - "expected a change side, found {side:?}" + "expected a change side, found {}", + format_link(side) ))), } } @@ -344,11 +344,13 @@ fn parse_link(link: &LiNo) -> ProtocolResult { parse_number(target)?, )), _ => Err(ProtocolError::malformed(format!( - "expected (index: source target), found {link:?}" + "expected (index: source target), found {}", + format_link(link) ))), }, _ => Err(ProtocolError::malformed(format!( - "expected (index: source target), found {link:?}" + "expected (index: source target), found {}", + format_link(link) ))), } } @@ -403,7 +405,8 @@ fn parse_number(value: &LiNo) -> ProtocolResult { match value { LiNo::Ref(text) => parse_text_number(text), _ => Err(ProtocolError::malformed(format!( - "expected a number, found {value:?}" + "expected a number, found {}", + format_link(value) ))), } } @@ -429,9 +432,15 @@ fn group(values: Vec>) -> LiNo { } fn malformed_argument(expected: &str, argument: &LiNo) -> ProtocolError { - ProtocolError::malformed(format!("expected {expected}, found {argument:?}")) + ProtocolError::malformed(format!( + "expected {expected}, found {}", + format_link(argument) + )) } fn malformed_reply(expected: &str, document: &[LiNo]) -> ProtocolError { - ProtocolError::malformed(format!("expected a {expected} reply, found {document:?}")) + ProtocolError::malformed(format!( + "expected a {expected} reply, found {}", + format_document(document) + )) } diff --git a/rust/src/protocol/mapping.rs b/rust/src/protocol/mapping.rs index cd01413..2ad6b55 100644 --- a/rust/src/protocol/mapping.rs +++ b/rust/src/protocol/mapping.rs @@ -462,12 +462,9 @@ impl<'a> Decoder<'a> { let LiNo::Ref(id) = self.decode(id, depth + 1, budget)? else { return Err(ProtocolError::malformed("a link id must be a reference")); }; - let LiNo::Link { values, .. } = self.list(values, depth, budget)? else { - unreachable!("list always returns a link"); - }; Ok(LiNo::Link { id: Some(id), - values, + values: self.values(values, depth, budget)?, }) } _ => Err(ProtocolError::malformed(format!( @@ -482,11 +479,22 @@ impl<'a> Decoder<'a> { depth: usize, budget: &mut usize, ) -> ProtocolResult> { - let values = elements + Ok(LiNo::Link { + id: None, + values: self.values(elements, depth, budget)?, + }) + } + + fn values( + &self, + elements: &[Reference], + depth: usize, + budget: &mut usize, + ) -> ProtocolResult>> { + elements .iter() .map(|&element| self.decode(element, depth + 1, budget)) - .collect::>>()?; - Ok(LiNo::Link { id: None, values }) + .collect() } fn decode( diff --git a/rust/src/protocol/packet.rs b/rust/src/protocol/packet.rs index 5b6ab0e..5460e9f 100644 --- a/rust/src/protocol/packet.rs +++ b/rust/src/protocol/packet.rs @@ -293,11 +293,13 @@ pub fn external_capacity(width: u8) -> u64 { } /// The narrowest width able to hold the internal address `address`. -pub fn address_tier(address: u64, external_references: bool) -> u8 { +pub fn address_tier(address: u64, external_references: bool) -> ProtocolResult { WIDTHS .into_iter() .find(|&width| internal_capacity(width, external_references) >= address) - .unwrap_or(8) + .ok_or_else(|| { + ProtocolError::Unencodable(format!("address {address} exceeds the internal range")) + }) } fn width_mask(width: u8) -> u64 { @@ -333,24 +335,15 @@ fn width_code(width: u8) -> ProtocolResult { .ok_or_else(|| ProtocolError::Unencodable(format!("invalid width {width}"))) } -fn width_from_code(code: u64) -> ProtocolResult { - usize::try_from(code) - .ok() - .and_then(|code| WIDTHS.get(code).copied()) - .ok_or_else(|| ProtocolError::malformed(format!("invalid width code {code}"))) +/// The width of a two-bit width code; every code names a width. +fn width_from_code(code: u64) -> u8 { + WIDTHS[(code & 0b11) as usize] } /// The narrowest width able to hold `reference`. pub fn reference_width(reference: Reference, external_references: bool) -> ProtocolResult { match reference { - Reference::Internal(address) => { - if address > internal_capacity(8, external_references) { - return Err(ProtocolError::Unencodable(format!( - "address {address} exceeds the internal range" - ))); - } - Ok(address_tier(address, external_references)) - } + Reference::Internal(address) => address_tier(address, external_references), Reference::External(value) => { if !external_references { return Err(ProtocolError::Unencodable( @@ -392,7 +385,7 @@ impl Section { /// link count. fn read_header(reader: &mut dyn Read) -> ProtocolResult<(Self, u64)> { let shape = read_leb128(reader)?; - let width = width_from_code(shape & SHAPE_WIDTH_BITS)?; + let width = width_from_code(shape & SHAPE_WIDTH_BITS); let gap = if shape & SHAPE_HAS_GAP != 0 { read_leb128(reader)? } else { @@ -607,21 +600,16 @@ impl LinksPacket { let packet = Self::read_from(&mut cursor, limits)? .ok_or_else(|| ProtocolError::malformed("empty input"))?; if !cursor.is_empty() { - return Err(ProtocolError::malformed(format!( - "{} trailing byte(s) after the packet", - cursor.len() - ))); + return Err(ProtocolError::malformed("trailing bytes after the packet")); } Ok(packet) } /// Reads one packet from `reader`; `Ok(None)` on a clean end of stream. pub fn read_from(reader: &mut dyn Read, limits: &DecodeLimits) -> ProtocolResult> { - let mut header = [0u8; 1]; - if read_exact_or_eof(reader, &mut header)? { + let Some(header) = read_byte_or_eof(reader)? else { return Ok(None); - } - let header = header[0]; + }; if header & 0xF0 != BINARY_VERSION_1 { return Err(ProtocolError::malformed(format!( "unsupported binary header byte 0x{header:02X}" @@ -630,7 +618,7 @@ impl LinksPacket { let mut packet = LinksPacket::new(header & FLAG_EXTERNAL_REFERENCES != 0); let mut counts = Vec::new(); if header & FLAG_EXPLICIT_LAYOUT == 0 { - let width = width_from_code(u64::from((header & WIDTH_BITS) >> WIDTH_SHIFT))?; + let width = width_from_code(u64::from((header & WIDTH_BITS) >> WIDTH_SHIFT)); let count = read_leb128(reader)?; if count > 0 { packet.sections.push(Section { @@ -880,16 +868,15 @@ fn read_exact(reader: &mut dyn Read, buffer: &mut [u8]) -> ProtocolResult<()> { }) } -/// Fills `buffer`, returning `true` if the stream ended before the first byte. -fn read_exact_or_eof(reader: &mut dyn Read, buffer: &mut [u8]) -> ProtocolResult { +/// The next byte of `reader`, or `None` at a clean end of stream. +fn read_byte_or_eof(reader: &mut dyn Read) -> ProtocolResult> { + let mut byte = [0u8; 1]; loop { - match reader.read(&mut buffer[..1]) { - Ok(0) => return Ok(true), - Ok(_) => break, - Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + match reader.read(&mut byte) { + Ok(0) => return Ok(None), + Ok(_) => return Ok(Some(byte[0])), + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} Err(error) => return Err(error.into()), } } - read_exact(reader, &mut buffer[1..])?; - Ok(false) } diff --git a/rust/src/protocol/protocols.rs b/rust/src/protocol/protocols.rs index 308db6f..9c9dde5 100644 --- a/rust/src/protocol/protocols.rs +++ b/rust/src/protocol/protocols.rs @@ -232,12 +232,15 @@ pub fn read_any_document( .read_document(reader)? .map(|document| (document, MessageFormat::Text))); } - let Some(packet) = LinksPacket::read_from(reader, limits)? else { - return Ok(None); - }; - let options = BinaryLinoOptions::of_packet(&packet); - let document = decode_document(&packet, limits)?; - Ok(Some((document, MessageFormat::Binary(options)))) + LinksPacket::read_from(reader, limits)? + .map(|packet| { + let document = decode_document(&packet, limits)?; + Ok(( + document, + MessageFormat::Binary(BinaryLinoOptions::of_packet(&packet)), + )) + }) + .transpose() } /// A byte stream decorated with a [`LinoProtocol`]. diff --git a/rust/src/protocol/server.rs b/rust/src/protocol/server.rs index 2806174..2159da5 100644 --- a/rust/src/protocol/server.rs +++ b/rust/src/protocol/server.rs @@ -6,7 +6,7 @@ //! nor locking. The protocol is detected per message and the reply uses the //! same protocol, so text and binary clients can share one server. -use super::error::{ProtocolError, ProtocolResult}; +use super::error::ProtocolResult; use super::format::format_document; use super::links_operations::LinksOperation; use super::mapping::LinoDocument; @@ -75,15 +75,6 @@ pub struct ShutdownHandle { address: SocketAddr, } -impl std::fmt::Debug for Job { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Job::Request { document, .. } => write!(formatter, "Request({document:?})"), - Job::Shutdown => write!(formatter, "Shutdown"), - } - } -} - impl ShutdownHandle { /// Asks the server to stop; `serve` returns after the current request. pub fn shutdown(&self) { @@ -202,7 +193,12 @@ fn handle_connection( loop { let (document, format) = match read_any_document(&mut reader, &options.limits) { Ok(Some(message)) => message, - Ok(None) => return Ok(()), + Ok(None) => { + if options.trace { + eprintln!("[server] client hung up"); + } + return Ok(()); + } Err(error) => { // The stream may be out of sync; answer in text and hang up. let reply = error_document(&error.to_string()); @@ -214,11 +210,11 @@ fn handle_connection( }; let response = if options.accept.accepts(format) { let (reply, response) = mpsc::channel(); + // Once `serve` returns, no request may touch the store. jobs.send(Job::Request { document, reply }) - .map_err(|_| ProtocolError::malformed("server is shutting down"))?; - response - .recv() - .map_err(|_| ProtocolError::malformed("server is shutting down"))? + .ok() + .and_then(|()| response.recv().ok()) + .unwrap_or_else(|| error_document("server is shutting down")) } else { error_document("this server does not accept this protocol") }; diff --git a/rust/tests/protocol_packet_tests.rs b/rust/tests/protocol_packet_tests.rs index 51a9451..7817ace 100644 --- a/rust/tests/protocol_packet_tests.rs +++ b/rust/tests/protocol_packet_tests.rs @@ -10,9 +10,10 @@ use link_cli::protocol::packet::{ use link_cli::protocol::{ decode_document, encode_document, format_document, format_reference, parse_document, read_any_document, ArityRange, BinaryLinoOptions, BinaryLinoProtocol, DecodeLimits, - LinksPacket, LinoProtocol, MessageFormat, ProtocolError, Reference, Section, TextLinoProtocol, + LinksPacket, LinoConnection, LinoProtocol, MessageFormat, ProtocolError, Reference, Section, + TextLinoProtocol, }; -use std::io::Cursor; +use std::io::{self, Cursor, Read}; const CORPUS: &[&str] = &[ "() ((1 1))", @@ -186,6 +187,16 @@ fn canonical_text_round_trips_the_corpus() { format_document(&parse_document("((1: 1 1)) ((1: 1 2))").unwrap()), "((1: 1 1)) ((1: 1 2))" ); + let empty_named_link = links_notation::LiNo::Link { + id: Some("a".to_string()), + values: vec![], + }; + // Only a binary message carries an id without values; links-notation reads its text back as the reference. + assert_eq!(format_document(&[empty_named_link]), "(a:)"); + assert_eq!( + parse_document("(a:)").unwrap(), + vec![links_notation::LiNo::Ref("a".to_string())] + ); } #[test] @@ -228,18 +239,24 @@ fn references_are_quoted_only_when_needed() { #[test] fn width_tiers_follow_the_number_of_links() { - assert_eq!(address_tier(0, false), 1); - assert_eq!(address_tier(255, false), 1); - assert_eq!(address_tier(256, false), 2); - assert_eq!(address_tier(65_535, false), 2); - assert_eq!(address_tier(65_536, false), 4); - assert_eq!(address_tier(u64::from(u32::MAX), false), 4); - assert_eq!(address_tier(u64::from(u32::MAX) + 1, false), 8); + assert_eq!(address_tier(0, false).unwrap(), 1); + assert_eq!(address_tier(255, false).unwrap(), 1); + assert_eq!(address_tier(256, false).unwrap(), 2); + assert_eq!(address_tier(65_535, false).unwrap(), 2); + assert_eq!(address_tier(65_536, false).unwrap(), 4); + assert_eq!(address_tier(u64::from(u32::MAX), false).unwrap(), 4); + assert_eq!(address_tier(u64::from(u32::MAX) + 1, false).unwrap(), 8); // External references take the top bit, halving every range. - assert_eq!(address_tier(127, true), 1); - assert_eq!(address_tier(128, true), 2); - assert_eq!(address_tier(32_767, true), 2); - assert_eq!(address_tier(32_768, true), 4); + assert_eq!(address_tier(127, true).unwrap(), 1); + assert_eq!(address_tier(128, true).unwrap(), 2); + assert_eq!(address_tier(32_767, true).unwrap(), 2); + assert_eq!(address_tier(32_768, true).unwrap(), 4); + assert_eq!(address_tier(u64::MAX, false).unwrap(), 8); + // No width holds an internal address in the external half. + assert!(matches!( + address_tier(1 << 63, true), + Err(ProtocolError::Unencodable(_)) + )); assert_eq!(internal_capacity(8, false), u64::MAX); assert_eq!(internal_capacity(8, true), i64::MAX as u64); assert_eq!(external_capacity(1), 127); @@ -573,44 +590,102 @@ fn expect_error(bytes: &[u8], limits: DecodeLimits) -> ProtocolError { protocol.decode(bytes).expect_err("must be rejected") } +/// Asserts that `bytes` is rejected as malformed because of `detail`. +fn assert_malformed(bytes: &[u8], detail: &str) { + match expect_error(bytes, DecodeLimits::default()) { + ProtocolError::Malformed(message) => { + assert_eq!(message, detail, "{}", hex(bytes)) + } + other => panic!("{}: {other}", hex(bytes)), + } +} + #[test] fn malformed_packets_are_rejected() { - let limits = DecodeLimits::default(); - let malformed = |bytes: &[u8]| { - assert!( - matches!(expect_error(bytes, limits), ProtocolError::Malformed(_)), - "{}", - hex(bytes) - ); - }; - malformed(&[]); - malformed(&[0x20, 0x00]); // unknown version nibble - malformed(&[0x10, 0x01, 0x06, 0x00]); // refers to itself - malformed(&[0x10, 0x01, 0x07, 0x00]); // refers forward - malformed(&[0x10, 0x02, 0x00]); // truncated - malformed(&[0x10, 0x00, 0x00]); // trailing byte - malformed(&[ - 0x10, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, - ]); // LEB128 overflow - malformed(&[0x10, 0x01, 0x01, 0x01]); // root is not a list - malformed(&[0x10, 0x02, 0x00, 0x02, 0x04, 0x06]); // marker 2 used as a value - malformed(&[0x10, 0x01, 0x04, 0x02]); // a bare marker in a list chain - malformed(&[0x16, 0x00]); // the explicit layout keeps the width bits clear - malformed(&[0x12, 0x01, 0x04, 0x05, 0x00]); // arity 0 - malformed(&[0x12, 0x01, 0x18, 0x01, 0x01, 0x05, 0x00]); // length 6 in arity 1..2 - malformed(&[ - 0x12, 0x01, 0xf8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x00, - ]); // arity overflows - malformed(&[ - 0x12, 0x02, 0x04, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x01, 0x20, - 0x01, - ]); // addresses overflow - malformed(&[0x12, 0x01, 0x24, 0x06, 0x01, 0x01, 0x01]); // LiNo links start at 6, not 7 - malformed(&[ - 0x12, 0x02, 0x24, 0x05, 0x01, 0x24, 0x01, 0x01, 0x01, 0x01, 0x04, 0x06, - ]); // a hole - // (String (0x110000)) is not a valid code point. + // Each case names the exact problem, so none passes for another reason. + for (bytes, detail) in [ + ("", "empty input"), + ("20 00", "unsupported binary header byte 0x20"), + ("10 01 06 00", "link 6 refers to Internal(6), which is not an earlier link"), + ("10 01 07 00", "link 6 refers to Internal(7), which is not an earlier link"), + ("10 02 00", "unexpected end of packet"), + ("10 00 00", "trailing bytes after the packet"), + // A tenth byte above 1, then an eleventh byte. + ( + "10 ff ff ff ff ff ff ff ff ff 7f", + "LEB128 value overflows 64 bits", + ), + ( + "10 80 80 80 80 80 80 80 80 80 81", + "LEB128 value overflows 64 bits", + ), + ("10 01 01 01", "the root link is not a list"), + // The chain ends in marker 2, not null. + ("10 02 00 02 04 06", "broken element chain"), + ("10 01 04 02", "broken element chain"), + ("10 02 02 00 04 06", "marker 2 used as a value"), + ( + "10 03 01 00 06 00 04 07", + "marker 1 cannot start a typed value", + ), + // (Number) alone. + ( + "12 02 14 05 01 20 02 02 06 00 04 07", + "a number needs exactly one value", + ), + ("10 03 05 00 06 00 04 07", "an identified link needs an id"), + // (Number 3): marker 3 is no number. + ("10 03 02 03 06 00 04 07", "expected a unary number"), + // (Number 6) where link 6 is (5 5), not a unary number. + ( + "10 05 05 05 06 00 02 07 08 00 04 09", + "expected a unary number", + ), + // The id is (). + ( + "10 04 00 00 05 06 07 00 04 08", + "a link id must be a reference", + ), + ( + "16 00", + "the explicit layout keeps the header width bits clear", + ), + ("12 01 04 05 00", "arity must be at least 1"), + // A link of 6 in a section of arity 1..2. + ( + "12 01 18 01 01 05 00", + "link length outside the section arity 1..2", + ), + ( + "12 01 f8 ff ff ff ff ff ff ff ff 01 ff ff ff ff ff ff ff ff ff 01 00", + "arity range overflows 64 bits", + ), + // The gap, then the count. + ( + "12 01 24 ff ff ff ff ff ff ff ff ff 01 01", + "addresses overflow 64 bits", + ), + ( + "12 01 20 ff ff ff ff ff ff ff ff ff 01", + "addresses overflow 64 bits", + ), + ( + "12 01 24 06 01 01 01", + "a LiNo packet stores its links contiguously from address 6, found link 7 where 6 belongs", + ), + // A hole. + ( + "12 02 24 05 01 24 01 01 01 01 04 06", + "a LiNo packet stores its links contiguously from address 6, found link 8 where 7 belongs", + ), + ] { + assert_malformed(&unhex(bytes), detail); + } +} + +#[test] +fn invalid_code_points_are_rejected() { + // (String (0x110000)) is not a valid code point. let links = [ vec![Reference::External(0x11_0000), Reference::NULL], vec![Reference::Internal(3), Reference::Internal(6)], @@ -618,11 +693,12 @@ fn malformed_packets_are_rejected() { vec![Reference::Internal(4), Reference::Internal(8)], ]; let links = (6..).zip(links).collect::>(); - malformed( + assert_malformed( &LinksPacket::pack(true, &links, false) .unwrap() .to_bytes() .unwrap(), + "invalid code point 1114112", ); } @@ -666,7 +742,11 @@ fn hostile_packets_hit_limits() { max_nodes: 1 << 12, ..DecodeLimits::default() }; - limited(&bomb.to_bytes().unwrap(), small_budget); + let limit = |bytes: &[u8], limits| expect_error(bytes, limits).to_string(); + assert_eq!( + limit(&bomb.to_bytes().unwrap(), small_budget), + "limit exceeded: too many LiNo nodes" + ); let deep = format!("{}x{}", "(y ".repeat(40), ")".repeat(40)); let bytes = BinaryLinoProtocol::new() @@ -676,7 +756,22 @@ fn hostile_packets_hit_limits() { max_depth: 10, ..DecodeLimits::default() }; - limited(&bytes, shallow); + assert_eq!( + limit(&bytes, shallow), + "limit exceeded: nesting deeper than 10" + ); + + let three_links = BinaryLinoProtocol::new() + .encode(&parse_document("a b c").unwrap()) + .unwrap(); + let two_nodes = DecodeLimits { + max_nodes: 2, + ..DecodeLimits::default() + }; + assert_eq!( + limit(&three_links, two_nodes), + "limit exceeded: chain too long" + ); assert!(DecodeLimits::unlimited().max_links == u64::MAX); } @@ -701,6 +796,62 @@ fn text_messages_are_dot_stuffed() { .is_none()); } +/// Fails once with `kind`, then reads `bytes`. +struct Hiccup { + kind: Option, + bytes: Cursor<&'static [u8]>, +} + +impl Hiccup { + fn new(kind: io::ErrorKind, bytes: &'static [u8]) -> Self { + Self { + kind: Some(kind), + bytes: Cursor::new(bytes), + } + } +} + +impl Read for Hiccup { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self.kind.take() { + Some(kind) => Err(kind.into()), + None => self.bytes.read(buffer), + } + } +} + +#[test] +fn stream_failures_are_io_errors_and_interruptions_are_retried() { + let limits = DecodeLimits::default(); + let read = |reader: &mut dyn Read| LinksPacket::read_from(reader, &limits); + + let packet = read(&mut Hiccup::new(io::ErrorKind::Interrupted, &[0x10, 0x00])); + assert_eq!(packet.unwrap().unwrap().links().count(), 0); + // Before the header, then inside the packet. + let reset = io::ErrorKind::ConnectionReset; + assert!(matches!( + read(&mut Hiccup::new(reset, &[])), + Err(ProtocolError::Io(_)) + )); + assert!(matches!( + read(&mut [0x10u8].chain(Hiccup::new(reset, &[]))), + Err(ProtocolError::Io(_)) + )); +} + +#[test] +fn connections_decorate_any_byte_stream() { + let document = parse_document("() ((1 1))").unwrap(); + let mut connection = LinoConnection::new(Cursor::new(Vec::new()), TextLinoProtocol::new()); + assert_eq!(connection.protocol().limits, DecodeLimits::default()); + // The cursor sits after what was sent, so no reply follows. + assert_eq!( + connection.request(&document).unwrap_err().to_string(), + "malformed message: connection closed before the reply" + ); + assert_eq!(connection.into_inner().into_inner(), b"() ((1 1))\n.\n"); +} + #[test] fn text_messages_accept_crlf_and_reject_truncation() { let mut reader = Cursor::new(b"() ((1 1))\r\n.\r\n".to_vec()); @@ -716,11 +867,17 @@ fn text_messages_accept_crlf_and_reject_truncation() { Some(Vec::new()) ); - let mut truncated = Cursor::new(b"() ((1 1))\n".to_vec()); - assert!(matches!( - TextLinoProtocol::new().read_document(&mut truncated), - Err(ProtocolError::Malformed(_)) - )); + // Without the terminator line, with or without the last newline. + for truncated in ["() ((1 1))\n", "() ((1 1))"] { + let mut truncated = Cursor::new(truncated.as_bytes()); + assert_eq!( + TextLinoProtocol::new() + .read_document(&mut truncated) + .unwrap_err() + .to_string(), + "malformed message: stream ended before the '.' terminator line" + ); + } let tiny = TextLinoProtocol { limits: DecodeLimits { @@ -729,10 +886,19 @@ fn text_messages_accept_crlf_and_reject_truncation() { }, }; let mut long = Cursor::new(format!("{}\n.\n", "a".repeat(100)).into_bytes()); - assert!(matches!( - tiny.read_document(&mut long), - Err(ProtocolError::LimitExceeded(_)) - )); + assert_eq!( + tiny.read_document(&mut long).unwrap_err().to_string(), + "limit exceeded: text message longer than 8 bytes" + ); + + let mut invalid_utf8 = Cursor::new(vec![0xC3, 0x28, b'\n', b'.', b'\n']); + assert_eq!( + TextLinoProtocol::new() + .read_document(&mut invalid_utf8) + .unwrap_err() + .to_string(), + "malformed message: text message is not valid UTF-8" + ); } #[test] diff --git a/rust/tests/protocol_tcp_tests.rs b/rust/tests/protocol_tcp_tests.rs index aed09ec..a49227a 100644 --- a/rust/tests/protocol_tcp_tests.rs +++ b/rust/tests/protocol_tcp_tests.rs @@ -4,8 +4,8 @@ mod common; use common::{protocols, RunningServer}; use link_cli::protocol::{ - AcceptedProtocols, ArityRange, BinaryLinoOptions, BinaryLinoProtocol, LinksClient, - ProtocolError, ServerOptions, TextLinoProtocol, + error_message, parse_document, AcceptedProtocols, ArityRange, BinaryLinoOptions, + BinaryLinoProtocol, LinksClient, ProtocolError, ServerOptions, TextLinoProtocol, }; use std::io::{BufRead, BufReader, Read, Write}; use std::net::TcpStream; @@ -33,6 +33,8 @@ fn crud_works_over_every_protocol() { "((1: 1 1)) ((1: 1 1))", "{name}" ); + // A query that creates and deletes nothing changes nothing. + assert_eq!(client.query_text("() ()").unwrap(), "", "{name}"); // Update. assert_eq!( client.query_text("((1: 1 1)) ((1: 1 2))").unwrap(), @@ -125,20 +127,88 @@ fn malformed_messages_get_an_error_and_the_connection_closes() { assert!(reply.ends_with("\n.\n"), "{reply}"); } +#[test] +fn a_stopped_server_executes_nothing() { + let mut server = RunningServer::start(ServerOptions::default(), false); + let mut client = server.client(TextLinoProtocol::new()); + assert_eq!(client.query_text("() ((1 1))").unwrap(), "() ((1: 1 1))"); + server.stop(); + // The connection outlives `serve`, but the store is gone. + assert_eq!( + client.query_text("").unwrap_err().to_string(), + "server error: server is shutting down" + ); +} + #[test] fn servers_can_restrict_the_accepted_protocol() { + for accept in [AcceptedProtocols::Binary, AcceptedProtocols::Text] { + let options = ServerOptions { + accept, + ..ServerOptions::default() + }; + let server = RunningServer::start(options, false); + let (mut refused, mut accepted) = if accept == AcceptedProtocols::Binary { + ( + server.client(TextLinoProtocol::new()), + server.client(BinaryLinoProtocol::new()), + ) + } else { + ( + server.client(BinaryLinoProtocol::new()), + server.client(TextLinoProtocol::new()), + ) + }; + assert_eq!( + refused.query("() ((1 1))").unwrap_err().to_string(), + "server error: this server does not accept this protocol" + ); + assert_eq!(accepted.query_text("() ((1 1))").unwrap(), "() ((1: 1 1))"); + } +} + +#[test] +fn only_error_documents_carry_an_error_message() { + let message = |text: &str| error_message(&parse_document(text).unwrap()).map(str::to_string); + assert_eq!( + message("(error: 'it failed')").as_deref(), + Some("it failed") + ); + for other in [ + "", + "(error: a b)", + "(error: (a b))", + "(fault: a)", + "(error: a) (error: b)", + ] { + assert_eq!(message(other), None, "{other}"); + } +} + +#[test] +fn tracing_servers_keep_serving() { let options = ServerOptions { - accept: AcceptedProtocols::Binary, + trace: true, ..ServerOptions::default() }; let server = RunningServer::start(options, false); - let mut text = server.client(TextLinoProtocol::new()); - assert!(matches!( - text.query("() ((1 1))"), - Err(ProtocolError::Remote(_)) - )); - let mut binary = server.client(BinaryLinoProtocol::new()); - assert_eq!(binary.query_text("() ((1 1))").unwrap(), "() ((1: 1 1))"); + let mut client = server.client(TextLinoProtocol::new()); + assert_eq!(client.query_text("() ((1 1))").unwrap(), "() ((1: 1 1))"); + drop(client); + let mut stream = TcpStream::connect(server.address).unwrap(); + stream + .set_read_timeout(Some(std::time::Duration::from_secs(10))) + .unwrap(); + stream.write_all(b"() ((1 1))\n").unwrap(); + stream.shutdown(std::net::Shutdown::Write).unwrap(); + let mut reply = String::new(); + stream.read_to_string(&mut reply).unwrap(); + assert_eq!( + reply, + "(error: \"malformed message: stream ended before the '.' terminator line\")\n.\n" + ); + let mut client = server.client(TextLinoProtocol::new()); + assert_eq!(client.query_text("").unwrap(), "(1: 1 1)"); } #[test] diff --git a/rust/tests/remote_links_tests.rs b/rust/tests/remote_links_tests.rs index 646ab45..a6a64c8 100644 --- a/rust/tests/remote_links_tests.rs +++ b/rust/tests/remote_links_tests.rs @@ -6,12 +6,17 @@ mod common; use common::{protocols, RunningServer}; use doublets::data::Flow; use doublets::{Doublets, Error, Link as DoubletsLink, Links}; +use link_cli::protocol::links_operations::{ + matches, parse_changes, parse_count, parse_link_reply, parse_links, parse_name, +}; use link_cli::protocol::{ - links_operations::matches, parse_document, LinksOperation, LinoProtocol, RemoteLinks, - ServerOptions, TextLinoProtocol, + parse_document, LinksOperation, LinoProtocol, ProtocolError, RemoteLinks, ServerOptions, + TextLinoProtocol, }; use link_cli::{simplify_changes, Link, NamedTypeLinks, NamedTypesDecorator, QueryProcessor}; +use std::net::TcpListener; use std::panic::{self, AssertUnwindSafe}; +use std::thread; use tempfile::NamedTempFile; /// Changes as the query processor reports them. @@ -296,13 +301,182 @@ fn only_operation_shaped_documents_are_operations() { "(create: (1))", "(update: (1 2))", "(delete: x)", + "(delete: -1)", + "(get-name: (1 2))", "(set-name: (1))", + "(set-name: (1 (a b)))", "(get-by-name: (a b))", + "(remove-name: 4294967296)", ] { - assert!(parse(malformed).is_err(), "{malformed}"); + assert!( + matches!(parse(malformed), Err(ProtocolError::Malformed(_))), + "{malformed}" + ); } } +#[test] +fn replies_parse_back_to_what_they_carry() { + let parse = |text: &str| parse_document(text).unwrap(); + + assert_eq!(parse_count(&parse("(count: 3)")).unwrap(), 3); + assert_eq!( + parse_links(&parse("(1: 2 3)")).unwrap(), + vec![Link::new(1, 2, 3)] + ); + assert_eq!( + parse_changes(&parse("() ((1: 0 0))\n((1: 0 0)) ()")).unwrap(), + vec![ + (Link::null(), Link::new(1, 0, 0)), + (Link::new(1, 0, 0), Link::null()) + ] + ); + // A side that is a single named link may lose its wrapper. + assert_eq!( + parse_changes(&parse("(1: 1 1) (1: 1 2)")).unwrap(), + vec![(Link::new(1, 1, 1), Link::new(1, 1, 2))] + ); + assert_eq!( + parse_name(&parse("(name: 'a name')")).unwrap().as_deref(), + Some("a name") + ); + assert_eq!(parse_name(&[]).unwrap(), None); + assert_eq!(parse_link_reply(&parse("(link: 5)")).unwrap(), Some(5)); + assert_eq!(parse_link_reply(&[]).unwrap(), None); +} + +#[test] +fn malformed_replies_are_rejected() { + for reply in [ + "(count: 1 2)", + "(count: x)", + "(name: a b)", + "(link: 1 2)", + "(1 2)", + "((1 2) x)", + ] { + let document = parse_document(reply).unwrap(); + for result in [ + parse_count(&document).map(|_| ()), + parse_links(&document).map(|_| ()), + parse_changes(&document).map(|_| ()), + parse_name(&document).map(|_| ()), + parse_link_reply(&document).map(|_| ()), + ] { + assert!( + matches!(result, Err(ProtocolError::Malformed(_))), + "{reply}: {result:?}" + ); + } + } + // Each kind of misshapen change side and link. + for (reply, detail) in [ + ("(1 2) ()", "expected a change side, found (1 2)"), + ("((1 2)) ()", "expected (index: source target), found (1 2)"), + ( + "((1: 2)) ()", + "expected (index: source target), found (1: 2)", + ), + ("((x: 1 2)) ()", "expected a number, found 'x'"), + ("((1: (2 3) 4)) ()", "expected a number, found (2 3)"), + ] { + match parse_changes(&parse_document(reply).unwrap()) { + Err(ProtocolError::Malformed(message)) => { + assert!(message.contains(detail), "{reply}: {message}") + } + other => panic!("{reply}: {other:?}"), + } + } +} + +/// A server that answers every request with `reply`, whatever it asks. +fn scripted_server(reply: &'static str) -> RemoteLinks { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut reader = std::io::BufReader::new(stream.try_clone().unwrap()); + let mut writer = stream; + let protocol = TextLinoProtocol::new(); + let reply = parse_document(reply).unwrap(); + while let Ok(Some(_)) = protocol.read_document(&mut reader) { + if protocol.write_document(&mut writer, &reply).is_err() { + return; + } + } + }); + RemoteLinks::connect(address, TextLinoProtocol::new()).unwrap() +} + +#[test] +fn replies_that_miss_the_requested_change_are_errors() { + // An empty reply is a valid document that reports no change at all. + let mut remote = scripted_server(""); + let error = |result: anyhow::Result| result.unwrap_err().to_string(); + assert_eq!( + error(remote.update_observed(1, 1, 1, &mut |_, _| {})), + "malformed message: the reply holds no change of 1" + ); + assert_eq!( + error(remote.delete_observed(2, &mut |_, _| {})), + "malformed message: the reply holds no change of 2" + ); + assert_eq!( + NamedTypeLinks::set_name(&mut remote, 1, "x") + .unwrap_err() + .to_string(), + "the set-name reply holds no link" + ); + assert_eq!(NamedTypeLinks::get_by_name(&mut remote, "x").unwrap(), None); + let panic = panic::catch_unwind(AssertUnwindSafe(|| { + NamedTypeLinks::create(&mut remote, 1, 1) + })) + .expect_err("a create must create a link"); + let message = panic.downcast_ref::().unwrap(); + assert!( + message.ends_with("the create reply holds no created link"), + "{message}" + ); +} + +#[test] +fn reads_stop_where_the_handler_breaks() { + let server = RunningServer::start(ServerOptions::default(), false); + let mut remote = RemoteLinks::connect(server.address, TextLinoProtocol::new()).unwrap(); + NamedTypeLinks::create(&mut remote, 1, 1); + NamedTypeLinks::create(&mut remote, 2, 2); + let mut visited = 0; + let flow = remote.each_links(&[], &mut |_| { + visited += 1; + Flow::Break + }); + assert_eq!((flow, visited), (Flow::Break, 1)); + let mut visited = 0; + let flow = remote.each_links(&[], &mut |_| { + visited += 1; + Flow::Continue + }); + assert_eq!((flow, visited), (Flow::Continue, 2)); + assert_eq!(remote.count_links(&[]), 2); +} + +#[test] +fn misshapen_name_replies_are_errors() { + let mut remote = scripted_server("(count: 1)"); + assert_eq!( + NamedTypeLinks::get_name(&mut remote, 1) + .unwrap_err() + .to_string(), + "malformed message: expected a name reply, found (count: 1)" + ); + assert_eq!( + NamedTypeLinks::get_by_name(&mut remote, "x") + .unwrap_err() + .to_string(), + "malformed message: expected a link reply, found (count: 1)" + ); +} + #[test] fn restrictions_match_like_the_raw_links_interface() { let link = Link::new(1, 2, 3); diff --git a/rust/tests/store_archive_tests.rs b/rust/tests/store_archive_tests.rs index 2abc0a3..442c96c 100644 --- a/rust/tests/store_archive_tests.rs +++ b/rust/tests/store_archive_tests.rs @@ -4,8 +4,8 @@ use link_cli::cli::{Cli, CliCommand}; use link_cli::protocol::{ - export_store, import_store, LinksPacket, Reference, RemoteLinks, ServerOptions, - TextLinoProtocol, + export_store, export_store_file, import_store, import_store_file, LinksPacket, Reference, + RemoteLinks, ServerOptions, TextLinoProtocol, }; use link_cli::{Link, NamedTypeLinks, NamedTypesDecorator}; use std::fs; @@ -362,3 +362,22 @@ fn the_cli_reports_a_missing_archive() { assert!(!output.status.success()); assert!(String::from_utf8_lossy(&output.stderr).contains("Failed to read the store archive")); } + +#[test] +fn archive_file_failures_name_the_file() { + let directory = tempdir().unwrap(); + let mut source = store(&directory, "source"); + let missing = directory.path().join("missing").join("store.bin"); + assert_eq!( + export_store_file(&mut source, &missing) + .unwrap_err() + .to_string(), + format!("Failed to write the store archive: {}", missing.display()) + ); + assert_eq!( + import_store_file(&mut source, &missing) + .unwrap_err() + .to_string(), + format!("Failed to read the store archive: {}", missing.display()) + ); +} From 170f9b42c6c1be97168cb00820bc6cbc20c7e5bc Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 03:57:12 +0000 Subject: [PATCH 42/44] test(csharp): exact-error parity with Rust; drop the untestable stopping branch of the accept loop --- csharp/.changeset/issue-105-protocol-coverage.md | 2 +- .../Protocol/LinksPacket.cs | 6 ++---- .../Protocol/LinksServer.cs | 8 ++------ .../Protocol/BinaryLinksNotationTests.cs | 12 +++++++----- .../Protocol/LinoProtocolCodecTests.cs | 7 +++++-- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/csharp/.changeset/issue-105-protocol-coverage.md b/csharp/.changeset/issue-105-protocol-coverage.md index 5c2a417..c422227 100644 --- a/csharp/.changeset/issue-105-protocol-coverage.md +++ b/csharp/.changeset/issue-105-protocol-coverage.md @@ -2,4 +2,4 @@ 'Foundation.Data.Doublets.Cli': patch --- -`LinksPacket.AddressTier` now throws an unencodable `LinoProtocolException` for an address beyond the internal range instead of silently returning width 8, and the unused `LinoFormat.IsReference` was removed. A server that is stopping now answers a request it already read with `(error: 'server is shutting down')` instead of hanging up, and its trace notes when a client hangs up. Tests now cover every line of the protocol code, and each malformed-packet test asserts the exact error, which fixed three cases that passed for the wrong reason. +`LinksPacket.AddressTier` now throws an unencodable `LinoProtocolException` for an address beyond the internal range instead of silently returning width 8, and the unused `LinoFormat.IsReference` was removed. A server that is stopping now answers a request it already read with `(error: 'server is shutting down')` instead of hanging up, and its trace notes when a client hangs up. `LinksPacket.WidthFromCode` is total over two-bit codes, and the accept loop no longer keeps a separate branch for a client accepted while stopping (its worker closes it). Tests now cover every line of the protocol code, and each malformed-packet test asserts the exact error, which fixed three cases that passed for the wrong reason. diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs index 1881753..dbab59e 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksPacket.cs @@ -259,10 +259,8 @@ internal static ulong WidthCode(byte width) return code >= 0 ? (ulong)code : throw LinoProtocolException.Unencodable($"invalid width {width}"); } - internal static byte WidthFromCode(ulong code) => - code < (ulong)WidthValues.Length - ? WidthValues[code] - : throw LinoProtocolException.Malformed($"invalid width code {code}"); + /// The width of a two-bit width code; every code names a width. + internal static byte WidthFromCode(ulong code) => WidthValues[code & 0b11]; /// The narrowest width able to hold . public static byte ReferenceWidth(PacketReference reference, bool externalReferences) diff --git a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs index 3caa6ae..7e1b143 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Library/Protocol/LinksServer.cs @@ -111,12 +111,8 @@ public void Serve(INamedTypesLinks links) } lock (_clients) { - if (_stopping) - { - client.Dispose(); - break; - } - // Here, where Shutdown cannot have disposed the client: the socket of a disposed client is null. + // Shutdown disposes only listed clients, so this one still has its socket. + // Accepted while stopping, it closes in its worker, which sees _stopping. TryDisableNagle(client); _clients.Add(client); } diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs index cbe318b..2085f8b 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/BinaryLinksNotationTests.cs @@ -367,8 +367,8 @@ private static LinoProtocolException DecodeError(byte[] bytes, DecodeLimits? lim [Theory] [InlineData("", "empty input")] [InlineData("20 00", "unsupported binary header byte 0x20")] - [InlineData("10 01 06 00", "link 6 refers to Internal(6)")] - [InlineData("10 01 07 00", "link 6 refers to Internal(7)")] + [InlineData("10 01 06 00", "link 6 refers to Internal(6), which is not an earlier link")] + [InlineData("10 01 07 00", "link 6 refers to Internal(7), which is not an earlier link")] [InlineData("10 02 00", "unexpected end of packet")] [InlineData("10 00 00", "trailing bytes after the packet")] [InlineData("10 ff ff ff ff ff ff ff ff ff 7f", "LEB128 value overflows 64 bits")] // a tenth byte above 1 @@ -380,6 +380,8 @@ private static LinoProtocolException DecodeError(byte[] bytes, DecodeLimits? lim [InlineData("10 03 01 00 06 00 04 07", "marker 1 cannot start a typed value")] [InlineData("12 02 14 05 01 20 02 02 06 00 04 07", "a number needs exactly one value")] // (Number) alone [InlineData("10 03 05 00 06 00 04 07", "an identified link needs an id")] + [InlineData("10 03 02 03 06 00 04 07", "expected a unary number")] // (Number 3): marker 3 is no number + [InlineData("10 05 05 05 06 00 02 07 08 00 04 09", "expected a unary number")] // (Number 6) where link 6 is (5 5) [InlineData("10 04 00 00 05 06 07 00 04 08", "a link id must be a reference")] // the id is () [InlineData("16 00", "the explicit layout keeps the header width bits clear")] [InlineData("12 01 04 05 00", "arity must be at least 1")] @@ -387,13 +389,13 @@ private static LinoProtocolException DecodeError(byte[] bytes, DecodeLimits? lim [InlineData("12 01 f8 ff ff ff ff ff ff ff ff 01 ff ff ff ff ff ff ff ff ff 01 00", "arity range overflows 64 bits")] [InlineData("12 01 24 ff ff ff ff ff ff ff ff ff 01 01", "addresses overflow 64 bits")] // the gap [InlineData("12 01 20 ff ff ff ff ff ff ff ff ff 01", "addresses overflow 64 bits")] // the count - [InlineData("12 01 24 06 01 01 01", "from address 6, found link 7 where 6 belongs")] - [InlineData("12 02 24 05 01 24 01 01 01 01 04 06", "found link 8 where 7 belongs")] // a hole + [InlineData("12 01 24 06 01 01 01", "a LiNo packet stores its links contiguously from address 6, found link 7 where 6 belongs")] + [InlineData("12 02 24 05 01 24 01 01 01 01 04 06", "a LiNo packet stores its links contiguously from address 6, found link 8 where 7 belongs")] // a hole public void MalformedPacketsAreRejected(string hex, string detail) { var error = DecodeError(Unhex(hex)); Assert.Equal(LinoProtocolErrorKind.Malformed, error.Kind); - Assert.Contains(detail, error.Detail, StringComparison.Ordinal); + Assert.Equal(detail, error.Detail); } private static List<(ulong Address, PacketReference[] References)> FromAddress6(params PacketReference[][] links) => diff --git a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs index 94566c0..434806c 100644 --- a/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs +++ b/csharp/Foundation.Data.Doublets.Cli.Tests/Protocol/LinoProtocolCodecTests.cs @@ -86,6 +86,9 @@ public void CanonicalTextRoundTripsTheCorpus() } Assert.Equal("() ((1 1))", LinoFormat.FormatDocument(Parse("(()((1 1)))"))); Assert.Equal("((1: 1 1)) ((1: 1 2))", LinoFormat.FormatDocument(Parse("((1: 1 1)) ((1: 1 2))"))); + // Only a binary message carries an id without values; links-notation reads its text back as the reference. + Assert.Equal("(a:)", LinoFormat.FormatDocument(new[] { LinoFormat.Link("a", new List()) })); + Assert.True(Parse("(a:)").SequenceEqual(new[] { LinoFormat.Reference("a") })); } [Fact] @@ -182,10 +185,10 @@ public void TextMessagesAcceptCrlfAndRejectTruncation() var tiny = new TextLinoProtocol { Limits = new DecodeLimits { MaxTextBytes = 8 } }; var longMessage = LinoStreamReader.FromBytes(System.Text.Encoding.UTF8.GetBytes(new string('a', 100) + "\n.\n")); - Assert.Equal(LinoProtocolErrorKind.LimitExceeded, Assert.Throws(() => tiny.ReadDocument(longMessage)).Kind); + Assert.Equal("limit exceeded: text message longer than 8 bytes", Assert.Throws(() => tiny.ReadDocument(longMessage)).Message); var invalidUtf8 = LinoStreamReader.FromBytes(new byte[] { 0xC3, 0x28, (byte)'\n', (byte)'.', (byte)'\n' }); - Assert.Equal(LinoProtocolErrorKind.Malformed, Assert.Throws(() => new TextLinoProtocol().ReadDocument(invalidUtf8)).Kind); + Assert.Equal("malformed message: text message is not valid UTF-8", Assert.Throws(() => new TextLinoProtocol().ReadDocument(invalidUtf8)).Message); } [Fact] From 4d31a8de57cb48e2dbd2f4deb9427016dc2aae3e Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 03:58:10 +0000 Subject: [PATCH 43/44] fix(ci): escape every '/' of an npm package name (CodeQL js/incomplete-sanitization) --- .github/scripts/check-latest-dependencies.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/check-latest-dependencies.mjs b/.github/scripts/check-latest-dependencies.mjs index 60e525d..a74d90d 100644 --- a/.github/scripts/check-latest-dependencies.mjs +++ b/.github/scripts/check-latest-dependencies.mjs @@ -165,7 +165,7 @@ const registries = { crate: async (name) => (await fetchJson(`https://crates.io/api/v1/crates/${name}`)).crate.max_stable_version, nuget: async (name) => latestStable((await fetchJson(`https://api.nuget.org/v3-flatcontainer/${name.toLowerCase()}/index.json`)).versions), - npm: async (name) => (await fetchJson(`https://registry.npmjs.org/${name.replace('/', '%2F')}/latest`)).version, + npm: async (name) => (await fetchJson(`https://registry.npmjs.org/${name.replaceAll('/', '%2F')}/latest`)).version, action: async (name) => { const tags = execFileSync('git', ['ls-remote', '--tags', `https://github.com/${name}`], { encoding: 'utf8' }) .split('\n') From 6417179b0ce7669f482e50268f4a8bdb506bb034 Mon Sep 17 00:00:00 2001 From: konard Date: Mon, 5 Oct 2026 03:58:49 +0000 Subject: [PATCH 44/44] docs: how the protocol code reaches full coverage and how to measure it --- docs/case-studies/issue-105/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/case-studies/issue-105/README.md b/docs/case-studies/issue-105/README.md index f6a9512..ce9083a 100644 --- a/docs/case-studies/issue-105/README.md +++ b/docs/case-studies/issue-105/README.md @@ -471,6 +471,21 @@ This repository steers around them: - `--changes` collapses each link's chain of changes (`changes_simplifier.rs`), so the `(0 0)` steps that doublets-rs#67 skips never appear in its report; - `PersistentFileMapped` works around mem-rs#36. +### Coverage + +The tests run every line of the protocol code in both ports: `rust/src/protocol/` +and `csharp/Foundation.Data.Doublets.Cli.Library/Protocol/`. That covers each +client and server under each protocol and each CRUD call. A malformed-input +test asserts the exact error, not just that some error occurred, because a +substring match let a few tests pass on the wrong error. Code that no test +could reach was removed rather than excluded: dead fallbacks, impossible +width codes and an accept-loop branch for a race. To reproduce: + +```bash +cd rust && cargo llvm-cov --summary-only # protocol/*.rs: 100% lines +cd csharp && dotnet test Foundation.Data.Doublets.Cli.Tests --collect:"XPlat Code Coverage" +``` + ## 7. Risks and remaining limits - **No authentication or encryption.** The server is meant for trusted