diff --git a/ctest-test/build.rs b/ctest-test/build.rs index 5f6118bb94688..001be0ab20a66 100644 --- a/ctest-test/build.rs +++ b/ctest-test/build.rs @@ -19,19 +19,19 @@ fn test_ctest() { .header("t1.h") .include("src") .skip_private(true) - .rename_fn(|f| f.link_name().unwrap_or(f.ident()).to_string().into()) - .rename_static(|s| s.link_name().unwrap_or(s.ident()).to_string().into()) + .rename_fn(|f| f.link_name().unwrap_or(f.path()).to_string().into()) + .rename_static(|s| s.link_name().unwrap_or(s.path()).to_string().into()) .rename_union_ty(|ty| (ty == "T1Union").then_some(ty.to_string())) .rename_struct_ty(|ty| (ty == "Transparent").then_some(ty.to_string())) - .volatile_struct_field(|s, f| s.ident() == "V" && f.ident() == "v") - .volatile_static(|s| s.ident() == "vol_ptr") - .volatile_static(|s| s.ident() == "T1_fn_ptr_vol") - .volatile_fn_arg(|f, p| f.ident() == "T1_vol0" && p.ident() == "arg0") - .volatile_fn_arg(|f, p| f.ident() == "T1_vol2" && p.ident() == "arg1") - .volatile_fn_return_type(|f| f.ident() == "T1_vol1") - .volatile_fn_return_type(|f| f.ident() == "T1_vol2") + .volatile_struct_field(|s, f| s.path() == "V" && f.ident() == "v") + .volatile_static(|s| s.path() == "vol_ptr") + .volatile_static(|s| s.path() == "T1_fn_ptr_vol") + .volatile_fn_arg(|f, p| f.path() == "T1_vol0" && p.ident() == "arg0") + .volatile_fn_arg(|f, p| f.path() == "T1_vol2" && p.ident() == "arg1") + .volatile_fn_return_type(|f| f.path() == "T1_vol1") + .volatile_fn_return_type(|f| f.path() == "T1_vol2") // The parameter `a` of the functions `T1r`, `T1s`, `T1t`, `T1v` is an array. - .array_arg(|f, p| matches!(f.ident(), "T1r" | "T1s" | "T1t" | "T1v") && p.ident() == "a") + .array_arg(|f, p| matches!(f.path(), "T1r" | "T1s" | "T1t" | "T1v") && p.ident() == "a") .skip_roundtrip(|n| n == "Arr"); ctest::generate_test(&mut t1gen, "src/t1.rs", "t1gen.rs").unwrap(); diff --git a/ctest/src/ast/constant.rs b/ctest/src/ast/constant.rs index b14a0db6b0ee1..cb0f18934d868 100644 --- a/ctest/src/ast/constant.rs +++ b/ctest/src/ast/constant.rs @@ -1,16 +1,64 @@ +use syn::{ + Path, + Type, +}; + use crate::BoxStr; +use crate::ast::{ + PathAggregateStore, + PathUtils, +}; /// Represents a constant variable defined in Rust. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Const { pub(crate) public: bool, - pub(crate) ident: BoxStr, - pub(crate) ty: syn::Type, + pub(crate) cached_path: BoxStr, + pub(crate) cached_ident: BoxStr, + pub(crate) path: Path, + pub(crate) path_aggregate: PathAggregateStore, + pub(crate) ty: Type, } impl Const { - /// Return the identifier of the constant as a string. + /// Return the full path to the constant variable as a string. + /// + /// If inside a nested module, this will return a top-level-relative path, + /// but not a crate-relative path. For some item `foo` in module + /// `crate::bar`, the returned string will be `bar::foo`, and not + /// `crate::bar::foo`. + pub fn path(&self) -> &str { + &self.cached_path + } + + /// Returns the last segment of the constant's path; Namely, its identifier. pub fn ident(&self) -> &str { - &self.ident + &self.cached_ident + } +} + +impl PathUtils for Const { + fn path_aggregate(&self) -> &PathAggregateStore { + &self.path_aggregate + } + + fn path(&self) -> &Path { + &self.path + } + + fn mk( + self, + cached_path: BoxStr, + cached_ident: BoxStr, + path: Path, + path_aggregate: PathAggregateStore, + ) -> Self { + Self { + cached_path, + cached_ident, + path, + path_aggregate, + ..self + } } } diff --git a/ctest/src/ast/field.rs b/ctest/src/ast/field.rs index c3403961f8823..a0f9a3b97774f 100644 --- a/ctest/src/ast/field.rs +++ b/ctest/src/ast/field.rs @@ -1,7 +1,7 @@ use crate::BoxStr; /// Represents a field in a struct or union defined in Rust. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Field { pub(crate) public: bool, pub(crate) ident: BoxStr, diff --git a/ctest/src/ast/function.rs b/ctest/src/ast/function.rs index e266a53efdbbd..b13bacc5c51a0 100644 --- a/ctest/src/ast/function.rs +++ b/ctest/src/ast/function.rs @@ -1,3 +1,9 @@ +use syn::Path; + +use crate::ast::{ + PathAggregateStore, + PathUtils, +}; use crate::{ Abi, BoxStr, @@ -7,23 +13,33 @@ use crate::{ /// Represents a function signature defined in Rust. /// /// This structure is only used for parsing functions in extern blocks. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Fn { pub(crate) public: bool, - #[expect(unused)] pub(crate) abi: Abi, - pub(crate) ident: BoxStr, + pub(crate) cached_path: BoxStr, + pub(crate) cached_ident: BoxStr, + pub(crate) path: Path, + pub(crate) path_aggregate: PathAggregateStore, pub(crate) link_name: Option, - #[expect(unused)] pub(crate) parameters: Vec, - #[expect(unused)] pub(crate) return_type: Option, } impl Fn { - /// Return the identifier of the function as a string. + /// Return the full path to the function item as a string. + /// + /// If inside a nested module, this will return a top-level-relative path, + /// but not a crate-relative path. For some item `foo` in module + /// `crate::bar`, the returned string will be `bar::foo`, and not + /// `crate::bar::foo`. + pub fn path(&self) -> &str { + &self.cached_path + } + + /// Returns the last segment of the function's path; Namely, its identifier. pub fn ident(&self) -> &str { - &self.ident + &self.cached_ident } /// Return the name of the function to be linked C side with. @@ -31,3 +47,29 @@ impl Fn { self.link_name.as_deref() } } + +impl PathUtils for Fn { + fn path_aggregate(&self) -> &PathAggregateStore { + &self.path_aggregate + } + + fn path(&self) -> &Path { + &self.path + } + + fn mk( + self, + cached_path: BoxStr, + cached_ident: BoxStr, + path: Path, + addressable_paths: PathAggregateStore, + ) -> Self { + Self { + cached_path, + cached_ident, + path, + path_aggregate: addressable_paths, + ..self + } + } +} diff --git a/ctest/src/ast/mod.rs b/ctest/src/ast/mod.rs index 325e05cb40056..865bc9129806a 100644 --- a/ctest/src/ast/mod.rs +++ b/ctest/src/ast/mod.rs @@ -1,23 +1,246 @@ mod constant; mod field; mod function; +mod module; mod parameter; mod static_variable; mod structure; mod type_alias; mod union; -use std::fmt; +use std::slice::Iter; +use std::vec::IntoIter; +use std::{ + fmt, + ops, +}; pub use constant::Const; pub use field::Field; pub use function::Fn; +pub use module::Module; pub use parameter::Parameter; +use quote::ToTokens; pub use static_variable::Static; pub use structure::Struct; +use syn::Path; pub use type_alias::Type; pub use union::Union; +use crate::{ + BoxStr, + MapInput, +}; + +/// Container for all paths by which a given item may be addressed throughout the crate. +/// +/// This is used to store all paths by which an item may be referred to, after it has potentially +/// been imported to modules other than the one it was declared in. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct PathAggregateStore { + value: Vec, +} + +impl PathAggregateStore { + pub(crate) const fn new() -> Self { + Self { value: Vec::new() } + } + + /// Appends a new path to the path aggregate store. + pub(crate) fn push(self, p: Path) -> Self { + let PathAggregateStore { mut value } = self; + value.push(p); + + Self { value } + } + + /// Checks if any two path aggregates are known to correspond with those of the same item. + /// + /// In other words, it checks if the item owning the receiver path aggregate and the item owning + /// the `other` path aggregate are different occurrences of the same item across the crate's + /// module tree. + pub(crate) fn akin(&self, other: &Self) -> bool { + self.value + .iter() + .any(|p1| other.value.iter().any(|p2| p1 == p2)) + } + + /// Diffs any two given path aggregates and returns the set of paths that were present in + /// `other` but missing in the receiver. + /// + /// Note that if the receiver contains more paths than `other`, the returned collection will be + /// empty. In other words, the operation is not reflexive + /// (`self.diff(other) != other.diff(self)`.) + pub(crate) fn diff(&self, other: &Self) -> Vec { + let ours = &self.value; + let theirs = &other.value; + theirs + .clone() + .into_iter() + .filter(|p1| ours.iter().any(|p2| p1 == p2)) + .collect() + } + + /// Yields a new path aggregate that integrates the paths of the receiver with those of `other`. + /// + /// Unlike diffing, the resulting path aggregate store unifies the paths from both parties; The + /// receiver and `other`. + #[expect(unused)] + pub(crate) fn merge(self, other: &Self) -> Self { + self.diff(other) + .into_iter() + .fold(self, PathAggregateStore::push) + } +} + +pub(crate) type AddressablePaths = IntoIter; +pub(crate) type RefAddressablePaths<'a> = Iter<'a, Path>; + +impl IntoIterator for PathAggregateStore { + type IntoIter = AddressablePaths; + type Item = Path; + + fn into_iter(self) -> Self::IntoIter { + self.value.into_iter() + } +} + +impl<'a> IntoIterator for &'a PathAggregateStore { + type IntoIter = RefAddressablePaths<'a>; + type Item = &'a Path; + + fn into_iter(self) -> Self::IntoIter { + self.value.iter() + } +} + +impl Extend for PathAggregateStore { + fn extend>(&mut self, iter: I) { + self.value.extend(iter); + } +} + +impl Default for PathAggregateStore { + fn default() -> Self { + Self::new() + } +} + +impl From> for PathAggregateStore { + fn from(value: Vec) -> Self { + Self { value } + } +} + +impl FromIterator for PathAggregateStore { + fn from_iter>(iter: I) -> Self { + Self { + value: iter.into_iter().collect(), + } + } +} + +/// Utility interface for keeping a single renaming function for all parsed items. +/// +/// They all share the same set of fields relative to their identifier and path, all of which need +/// updating whenever the path is renamed. We ensure that the only place where this happens is in +/// this function for convenience's sake. +pub(crate) trait PathUtils: Clone { + fn path(&self) -> &Path; + fn path_aggregate(&self) -> &PathAggregateStore; + fn mk( + self, + cached_path: BoxStr, + cached_ident: BoxStr, + path: Path, + addressable_paths: PathAggregateStore, + ) -> Self; + + /// Provided a path, it ensures the path aggregate of a given item integrates the new path. + /// + /// Unlike methods `rename` and `rename_with`, this does not change the path of the item itself, + /// but rather produces an updated receiver with only the path aggregate store expanded. + fn add_path(self, path: Path) -> Self { + let old_path = self.path().clone(); + let old_aggregate = self.path_aggregate().clone(); + self.mk( + path_to_string(&old_path), + ident_to_string(&old_path), + old_path, + old_aggregate.push(path), + ) + } + + /// Provided some parsed type, this method will run the given closure on each variation of a + /// term of that type (`self`.) + /// + /// The variations of a given term are given by their addressable paths. This is an append-only + /// aggregate all implementors of this interface have as one of their projections. The + /// variations are themselves the possible paths by which an item may be referred to as a + /// consequence of having been imported over to other modules of the crate. + /// + /// This interface gets access to it through the [`RenameUtils::addressable_paths`] projection. + /// + /// This is a utility to avoid having to go through the variations, as well as updating the + /// identifier and paths caches whenever some operation needs to be performed on all variations. + fn with_variations(&self, f: impl ops::Fn(&Self) -> T) -> Vec { + self.path_aggregate() + .into_iter() + .cloned() + .map(|p| f(&self.clone().rename(p))) + .collect() + } + + fn rename_with(self, f: impl ops::Fn(&Path) -> Path + Clone) -> Self { + let path = self.path().clone(); + + self.rename(f(&path)) + } + + fn rename(self, new_path: Path) -> Self { + let old_aggregate = self.path_aggregate().clone(); + + self.mk( + path_to_string(&new_path), + ident_to_string(&new_path), + new_path.clone(), + old_aggregate.push(new_path), + ) + } +} + +/// Transforms an item's absolute path to use `_` as path separator instead of `::`. +pub(crate) fn escape_item_path<'a>(item: impl Into>) -> String { + let path = match item.into() { + MapInput::Struct(s) => s.path(), + MapInput::Union(u) => u.path(), + MapInput::Fn(f) => f.path(), + MapInput::Alias(a) => a.path(), + MapInput::Const(c) => c.path(), + MapInput::Static(s) => s.path(), + + _ => unimplemented!("other MapInput data constructors do not represent rust items"), + }; + path.replace("::", "_") +} + +/// Returns a stringified `syn::Path` that is meant to match one-to-one the path to the item from +/// the crate root. +pub(crate) fn path_to_string(path: &Path) -> BoxStr { + path.into_token_stream() + .to_string() + .replace(|c: char| c.is_ascii_whitespace(), "") + .into_boxed_str() +} + +/// Returns a stringified representation of the last segment in the passed identifier. +pub(crate) fn ident_to_string(path: &Path) -> BoxStr { + path.segments + .last() + .map(|it| it.ident.to_string().into_boxed_str()) + .unwrap_or_default() +} + /// The ABI as defined by the extern block. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Abi { diff --git a/ctest/src/ast/module.rs b/ctest/src/ast/module.rs new file mode 100644 index 0000000000000..96ab8b29649c5 --- /dev/null +++ b/ctest/src/ast/module.rs @@ -0,0 +1,272 @@ +use std::cmp::Ordering; +use std::ops; + +use syn::Path; +use syn::punctuated::Punctuated; + +use crate::ast::{ + PathAggregateStore, + PathUtils, +}; +use crate::ffi_items::{ + FfiItems, + GenericItem, +}; +use crate::{ + BoxStr, + Const, + Fn, + Static, + Struct, + Type, + Union, + ast, +}; + +/// Represents a Rust module. `ctest` only considers items for which there is a corresponding type +/// in this crate. +#[derive(Debug, Clone)] +pub struct Module { + pub(crate) public: bool, + pub(crate) cached_path: BoxStr, + pub(crate) cached_ident: BoxStr, + pub(crate) path: Path, + pub(crate) path_aggregate: PathAggregateStore, + pub(crate) items: FfiItems, +} + +impl Module { + /// Returns the full path to the module item. + /// + /// If inside a nested module, this will return a top-level-relative path, but not a + /// crate-relative path. For some item `foo` in module `crate::bar`, the returned string will be + /// `bar::foo`, and not `crate::bar::foo`. + pub fn path(&self) -> &str { + &self.cached_path + } + + /// Returns the last segment of the module's path; Namely, its identifier. + pub fn ident(&self) -> &str { + &self.cached_ident + } + + /// Comparison function for use when sorting modules in [`FfiItems::dedup`]. + /// + /// This is not implemented as part of [`Ord`] because the implementation is inconsistent with + /// the [`PartialEq`] implementation. + pub(crate) fn custom_cmp(&self, other: &Self) -> Ordering { + match self.cached_path.cmp(&other.cached_path) { + Ordering::Equal => self.items.custom_cmp(&other.items), + other => other, + } + } +} + +impl PathUtils for Module { + fn path_aggregate(&self) -> &PathAggregateStore { + &self.path_aggregate + } + + fn path(&self) -> &Path { + &self.path + } + + fn mk( + self, + cached_path: BoxStr, + cached_ident: BoxStr, + path: Path, + addressable_paths: PathAggregateStore, + ) -> Self { + Self { + cached_path, + cached_ident, + path, + path_aggregate: addressable_paths, + ..self + } + } + + fn add_path(self, path: Path) -> Self { + fn handle_item(it: GenericItem, original: &Path, new: &Path) -> GenericItem { + macro_rules! map { + ($($ty:tt),+ $(,)?) => { + match it { + $( + GenericItem::$ty(it) => { + let kept_segments = <$ty as PathUtils>::path(&it) + .segments + .clone() + .into_iter() + .skip(original.segments.len()); + let new_path = Path { + segments: new + .segments + .clone() + .into_iter() + .chain(kept_segments) + .collect(), + ..new.clone() + }; + + GenericItem::$ty($ty { + path_aggregate: it.path_aggregate().clone().push(new_path), + ..it + }) + } + )+ + GenericItem::Module(m) => { + let kept_segments = ::path(&m) + .segments + .clone() + .into_iter() + .skip(original.segments.len()); + let new_path = Path { + segments: new + .segments + .clone() + .into_iter() + .chain(kept_segments) + .collect(), + ..new.clone() + }; + + let new_items = m + .items + .items() + .map(|it| handle_item(it, &original, &new)) + .collect(); + + GenericItem::Module(Module { + path_aggregate: m.path_aggregate().clone().push(new_path), + items: new_items, + ..m + }) + } + } + }; + } + + map! { + Type, + Struct, + Union, + Const, + Fn, + Static, + } + } + + let new_items = self + .items + .items() + .map(|it| handle_item(it, ::path(&self), &path)) + .collect(); + + Module { + path_aggregate: self.path_aggregate.push(path), + items: new_items, + ..self + } + } + + // [NOTE]: this is implemented in terms of rename_with but it doesn't just + // yield the passed path, because that would mean all items would end up + // with the same path. Instead, it calls into rename_with and skips as many + // segments of all paths in the subtree as the current module has. Then it + // appends whatever segments are left to the new path (new_path) we got + // passed here. In the case of the module on which this was called, that + // means all segments will be skipped and nothing will be appended to the + // new path. In all other cases of items within its subtree, they will have + // the previous identifier of the ancestor in which this rename request was + // originally called, trimmed, and whatever segments are left will be + // appended to the new path of that (potentially grand-) ancestor. + fn rename(self, new_path: Path) -> Self { + let old_path_len = self.path.segments.len(); + + self.rename_with(|p| { + p.segments + .iter() + .skip(old_path_len) + .cloned() + .fold(new_path.clone(), |new_path, s| Path { + segments: new_path.segments.clone().into_iter().chain([s]).collect(), + ..new_path + }) + }) + } + + fn rename_with(self, f: impl ops::Fn(&Path) -> Path + Clone) -> Self { + let new_items = self + .items + .items() + .map(|it| { + macro_rules! exe { + ($($ty:tt),+ $(,)?) => { + match it { + $( + GenericItem::$ty(it) => { + let new_path = f(&it.path); + + GenericItem::$ty(it.rename(new_path)) + } + )+ + GenericItem::Module(it) => GenericItem::Module(it.rename_with(f.clone())), + } + }; + } + + exe! { + Type, + Struct, + Union, + Const, + Fn, + Static, + } + }) + .collect(); + + let new_path = f(&self.path); + + Module { + cached_path: ast::path_to_string(&new_path), + cached_ident: ast::ident_to_string(&new_path), + path: new_path.clone(), + path_aggregate: self.path_aggregate.push(new_path), + items: new_items, + ..self + } + } +} + +impl Default for Module { + fn default() -> Self { + let dpath = Path { + leading_colon: None, + segments: Punctuated::new(), + }; + + Self { + public: Default::default(), + cached_path: Default::default(), + cached_ident: Default::default(), + path: dpath.clone(), + path_aggregate: PathAggregateStore::from_iter([dpath]), + items: Default::default(), + } + } +} + +// [NOTE]: a custom implementation is used because FfiItems does not have a +// PartialEq implementation. It is best if it does not have one. The +// current_module field in FfiItems is not meant for use post-parse-time. It +// would likely be unintuitive to have its PartialEq not take into account. +impl PartialEq for Module { + fn eq(&self, other: &Self) -> bool { + self.public == other.public + && self.path == other.path + && self.path_aggregate == other.path_aggregate + && self.items.custom_eq(&other.items) + } +} diff --git a/ctest/src/ast/parameter.rs b/ctest/src/ast/parameter.rs index 58ed170931a70..a7553992e94af 100644 --- a/ctest/src/ast/parameter.rs +++ b/ctest/src/ast/parameter.rs @@ -1,10 +1,9 @@ use crate::BoxStr; /// Represents a parameter in a function signature defined in Rust. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Parameter { pub(crate) ident: BoxStr, - #[expect(unused)] pub(crate) ty: syn::Type, } diff --git a/ctest/src/ast/static_variable.rs b/ctest/src/ast/static_variable.rs index 665bd06eb8e4b..42b31d51cd883 100644 --- a/ctest/src/ast/static_variable.rs +++ b/ctest/src/ast/static_variable.rs @@ -1,3 +1,12 @@ +use syn::{ + Path, + Type, +}; + +use crate::ast::{ + PathAggregateStore, + PathUtils, +}; use crate::{ Abi, BoxStr, @@ -7,20 +16,32 @@ use crate::{ /// /// This structure is only used for parsing statics in extern blocks, /// as a result it does not have a field for storing the expression. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Static { pub(crate) public: bool, - #[expect(unused)] pub(crate) abi: Abi, - pub(crate) ident: BoxStr, + pub(crate) cached_path: BoxStr, + pub(crate) cached_ident: BoxStr, + pub(crate) path: Path, + pub(crate) path_aggregate: PathAggregateStore, pub(crate) link_name: Option, - pub(crate) ty: syn::Type, + pub(crate) ty: Type, } impl Static { - /// Return the identifier of the static variable as a string. + /// Return the full path to the static variable as a string. + /// + /// If inside a nested module, this will return a top-level-relative path, + /// but not a crate-relative path. For some item `foo` in module + /// `crate::bar`, the returned string will be `bar::foo`, and not + /// `crate::bar::foo`. + pub fn path(&self) -> &str { + &self.cached_path + } + + /// Returns the last segment of the static's path; Namely, its identifier. pub fn ident(&self) -> &str { - &self.ident + &self.cached_ident } /// Return the name of the function to be linked C side with. @@ -28,3 +49,29 @@ impl Static { self.link_name.as_deref() } } + +impl PathUtils for Static { + fn path_aggregate(&self) -> &PathAggregateStore { + &self.path_aggregate + } + + fn path(&self) -> &Path { + &self.path + } + + fn mk( + self, + cached_path: BoxStr, + cached_ident: BoxStr, + path: Path, + addressable_paths: PathAggregateStore, + ) -> Self { + Self { + cached_path, + cached_ident, + path, + path_aggregate: addressable_paths, + ..self + } + } +} diff --git a/ctest/src/ast/structure.rs b/ctest/src/ast/structure.rs index 1c935890d4d23..f9ed404225ea1 100644 --- a/ctest/src/ast/structure.rs +++ b/ctest/src/ast/structure.rs @@ -1,19 +1,82 @@ +use syn::Path; +use syn::punctuated::Punctuated; + +use crate::ast::{ + PathAggregateStore, + PathUtils, +}; use crate::{ BoxStr, Field, }; /// Represents a struct defined in Rust. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Struct { pub(crate) public: bool, - pub(crate) ident: BoxStr, + pub(crate) cached_path: BoxStr, + pub(crate) cached_ident: BoxStr, + pub(crate) path: Path, + pub(crate) path_aggregate: PathAggregateStore, pub(crate) fields: Vec, } impl Struct { - /// Return the identifier of the struct as a string. + /// Return the full path to the struct item as a string. + /// + /// If inside a nested module, this will return a top-level-relative path, + /// but not a crate-relative path. For some item `foo` in module + /// `crate::bar`, the returned string will be `bar::foo`, and not + /// `crate::bar::foo`. + pub fn path(&self) -> &str { + &self.cached_path + } + + /// Returns the last segment of the struct's path; Namely, its identifier. pub fn ident(&self) -> &str { - &self.ident + &self.cached_ident + } +} + +impl PathUtils for Struct { + fn path_aggregate(&self) -> &PathAggregateStore { + &self.path_aggregate + } + + fn path(&self) -> &Path { + &self.path + } + + fn mk( + self, + cached_path: BoxStr, + cached_ident: BoxStr, + path: Path, + addressable_paths: PathAggregateStore, + ) -> Self { + Self { + cached_path, + cached_ident, + path, + path_aggregate: addressable_paths, + ..self + } + } +} + +impl Default for Struct { + fn default() -> Self { + let dpath = Path { + leading_colon: None, + segments: Punctuated::new(), + }; + Self { + public: Default::default(), + cached_path: Default::default(), + cached_ident: Default::default(), + path: dpath.clone(), + path_aggregate: PathAggregateStore::from_iter([dpath]), + fields: vec![], + } } } diff --git a/ctest/src/ast/type_alias.rs b/ctest/src/ast/type_alias.rs index 4947e934dde51..c0ae9a3d62e70 100644 --- a/ctest/src/ast/type_alias.rs +++ b/ctest/src/ast/type_alias.rs @@ -1,16 +1,62 @@ +use syn::Path; + use crate::BoxStr; +use crate::ast::{ + PathAggregateStore, + PathUtils, +}; /// Represents a type alias defined in Rust. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Type { pub(crate) public: bool, - pub(crate) ident: BoxStr, + pub(crate) cached_path: BoxStr, + pub(crate) cached_ident: BoxStr, + pub(crate) path: Path, + pub(crate) path_aggregate: PathAggregateStore, pub(crate) ty: syn::Type, } impl Type { - /// Return the identifier of the type alias as a string. + /// Return the full path to the type alias as a string. + /// + /// If inside a nested module, this will return a top-level-relative path, + /// but not a crate-relative path. For some item `foo` in module + /// `crate::bar`, the returned string will be `bar::foo`, and not + /// `crate::bar::foo`. + pub fn path(&self) -> &str { + &self.cached_path + } + + /// Returns the last segment of the type alias's path; Namely, its + /// identifier. pub fn ident(&self) -> &str { - &self.ident + &self.cached_ident + } +} + +impl PathUtils for Type { + fn path_aggregate(&self) -> &PathAggregateStore { + &self.path_aggregate + } + + fn path(&self) -> &Path { + &self.path + } + + fn mk( + self, + cached_path: BoxStr, + cached_ident: BoxStr, + path: Path, + addressable_paths: PathAggregateStore, + ) -> Self { + Self { + cached_path, + cached_ident, + path, + path_aggregate: addressable_paths, + ..self + } } } diff --git a/ctest/src/ast/union.rs b/ctest/src/ast/union.rs index 990d6db0efca5..3ebe92bde8af4 100644 --- a/ctest/src/ast/union.rs +++ b/ctest/src/ast/union.rs @@ -1,19 +1,64 @@ +use syn::Path; + +use crate::ast::{ + PathAggregateStore, + PathUtils, +}; use crate::{ BoxStr, Field, }; /// Represents a union defined in Rust. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Union { pub(crate) public: bool, - pub(crate) ident: BoxStr, + pub(crate) cached_path: BoxStr, + pub(crate) cached_ident: BoxStr, + pub(crate) path: Path, + pub(crate) path_aggregate: PathAggregateStore, pub(crate) fields: Vec, } impl Union { - /// Return the identifier of the union as a string. + /// Return the full path to the union item as a string. + /// + /// If inside a nested module, this will return a top-level-relative path, + /// but not a crate-relative path. For some item `foo` in module + /// `crate::bar`, the returned string will be `bar::foo`, and not + /// `crate::bar::foo`. + pub fn path(&self) -> &str { + &self.cached_path + } + + /// Returns the last segment of the union's path; Namely, its identifier. pub fn ident(&self) -> &str { - &self.ident + &self.cached_ident + } +} + +impl PathUtils for Union { + fn path_aggregate(&self) -> &PathAggregateStore { + &self.path_aggregate + } + + fn path(&self) -> &Path { + &self.path + } + + fn mk( + self, + cached_path: BoxStr, + cached_ident: BoxStr, + path: Path, + addressable_paths: PathAggregateStore, + ) -> Self { + Self { + cached_path, + cached_ident, + path, + path_aggregate: addressable_paths, + ..self + } } } diff --git a/ctest/src/ffi_items.rs b/ctest/src/ffi_items.rs index c7198082c946a..7dbae031f437f 100644 --- a/ctest/src/ffi_items.rs +++ b/ctest/src/ffi_items.rs @@ -1,27 +1,280 @@ //! Conversion of Rust code to a simplified abstract syntax tree. -use std::ops::Deref; +use std::cmp::Ordering; +use std::fmt::{ + self, + Debug, + Formatter, +}; +use std::ops::{ + self, + ControlFlow, + Deref, + Not, +}; +use std::{ + iter, + mem, + vec, +}; +use proc_macro2::Span; use syn::punctuated::Punctuated; -use syn::visit::Visit; +use syn::token::{ + Brace, + Mod, + PathSep, +}; +use syn::visit::{ + self, + Visit, +}; +use syn::{ + File, + Ident, + Item, + ItemMod, + ItemUse, + Path, + PathArguments, + PathSegment, + UseName, + UseRename, + UseTree, + Visibility, +}; +use crate::ast::{ + PathAggregateStore, + PathUtils, +}; use crate::{ Abi, BoxStr, Const, Field, Fn, + Module, Parameter, Static, Struct, Type, Union, + ast, }; +// [NOTE]: for use only in printf-debugging. It doesn't automatically derive +// Debug to selectively print out the fields I'm interested in while debugging. +#[derive(Clone)] +struct DebugFfiItems { + aliases: Vec, + structs: Vec, + unions: Vec, + constants: Vec, + foreign_functions: Vec, + foreign_statics: Vec, + modules: Vec, +} + +impl Debug for DebugFfiItems { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let mut dbg_struct = f.debug_struct("DebugFfiItems"); + + #[derive(Clone)] + struct PrettyPath<'a>(&'a Path); + + impl Debug for PrettyPath<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.pad(&ast::path_to_string(self.0)) + } + } + + macro_rules! pretty_print { + ($($field:tt: $ty:tt),+ $(,)?) => {$( + dbg_struct.field_with(stringify!($field), |f| { + f.pad("[ ") + .and_then(|_| { + self.$field + .iter() + .map(|it| { + f.pad(&format!( + concat!( + stringify!($ty), + " {{ path: {:?}, path_aggregate: {:?} }}", + ), + PrettyPath(&it.path), + (&it.path_aggregate) + .into_iter() + .map(PrettyPath) + .collect::>(), + )) + }) + .collect::() + }) + .and_then(|_| f.pad(" ]")) + }); + )+}; + } + + pretty_print! { + aliases: Type, + structs: Struct, + unions: Union, + constants: Const, + foreign_functions: Fn, + foreign_statics: Static, + } + + dbg_struct.field_with("modules", |f| { + f.pad("[ ") + .and_then(|_| { + self.modules + .iter() + .map(|m| { + f.pad("[ ") + .and_then(|_| { + f.pad(&format!( + "path: {:?}, path_aggregate: {:?}, items: {{ {:?} }}", + PrettyPath(&m.path), + (&m.path_aggregate) + .into_iter() + .map(PrettyPath) + .collect::>(), + DebugFfiItems::from(m.items.clone()), + )) + }) + .and_then(|_| f.pad(" ]")) + }) + .collect::() + }) + .and_then(|_| f.pad(" ]")) + }); + + dbg_struct.finish() + } +} + +// [NOTE]: for debugging purposes only. Pretty prints a refined type for +// syn::UseTree. +#[derive(Clone)] +struct PrettyUseTree<'a>(&'a RefinedUseTree); + +impl Debug for PrettyUseTree<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.pad(&refined_tree_to_string(&self.0)) + } +} + +fn refined_tree_to_string(t: &RefinedUseTree) -> String { + match t { + RefinedUseTree::Path(p) => { + format!("{}::{}", p.ident, refined_tree_to_string(&p.tree)) + } + RefinedUseTree::Name(n) => format!("{}", n.ident), + RefinedUseTree::Rename(r) => format!("{} as {}", r.ident, r.rename), + RefinedUseTree::Glob => format!("*"), + } +} + +impl From for DebugFfiItems { + fn from( + FfiItems { + aliases, + structs, + unions, + constants, + foreign_functions, + foreign_statics, + modules, + .. + }: FfiItems, + ) -> Self { + Self { + aliases, + structs, + unions, + constants, + foreign_functions, + foreign_statics, + modules, + } + } +} + +/// Tuple-based state monad used for keeping track of mutable state while parsing items for +/// FfiItems. +/// +/// The monad is currently used for tracking the current module of the item being parsed, as well as +/// the unique ID of all parsed items. The monad in those cases corresponds with the concrete type +/// `StateMonad`. The implementation of `syn::visit::Visit` doesn't avoid +/// the impure mutable state of that interface but scopes it to polymorphic parameter `T` in a +/// separate copy of the state monad. After the recursive visitor traversal is done in the copy, the +/// original and mutated state monads are reconciled by binding the former to the latter. +// [FIXME]: it may be a better idea to change this into a cps state monad just +// to see whether they work out better with linear terms. +struct StateMonad { + seq: Box (S, T)>, +} + +impl StateMonad { + fn pure(value: T) -> Self + where + for<'a> T: 'a, + { + Self { + seq: Box::new(|s| (s, value)), + } + } + + fn bind(self, f: F) -> StateMonad + where + for<'a> S: 'a, + for<'a> T: 'a, + for<'a> F: ops::FnOnce(T) -> StateMonad + 'a, + { + StateMonad { + seq: Box::new(|s| { + let (state, value) = (self.seq)(s); + + (f(value).seq)(state) + }), + } + } + + fn set(f: F) -> StateMonad + where + for<'a> F: ops::FnOnce(S) -> (S, T) + 'a, + { + StateMonad { seq: Box::new(f) } + } + + fn run(self, init: S) -> (S, T) { + (self.seq)(init) + } +} + +#[test] +fn tmpmonad() { + let (state, value) = StateMonad::pure(2) + .bind(|x| StateMonad::set(move |s| (s * x, x))) + .run(3); + + assert_eq!((state, value), (6, 2)); +} + +#[derive(Debug, Clone)] +struct ParsingState { + uid: usize, + current_module: Path, + module_is_public: bool, +} + /// Represents a collected set of top-level Rust items relevant to FFI generation or analysis. /// -/// Includes foreign functions/statics, type aliases, structs, unions, and constants. -#[derive(Default, Clone, Debug)] +/// Includes foreign functions/statics, type aliases, structs, unions, and constants. Modules are +/// collected as recursive `FfiItems`, and currently used to narrow down tested items to those in a +/// specific module. +#[derive(Clone, Debug)] pub(crate) struct FfiItems { pub(crate) aliases: Vec, pub(crate) structs: Vec, @@ -29,6 +282,14 @@ pub(crate) struct FfiItems { pub(crate) constants: Vec, pub(crate) foreign_functions: Vec, pub(crate) foreign_statics: Vec, + pub(crate) uses: Vec, + pub(crate) modules: Vec, + + /// This is used while recursing through parsed modules to gather absolute paths to them as + /// identifiers for both the modules and the items within them. + // [FIXME]: get rid of this and refactor parsing into using a helper that + // stores this ancilliary state. + pub(crate) current_module: syn::Path, } impl FfiItems { @@ -78,12 +339,1361 @@ impl FfiItems { pub(crate) fn foreign_statics(&self) -> &Vec { &self.foreign_statics } + + /// Entry point to parse a [`syn::File`]. + /// + /// This will also resolve `use`-trees such that the resulting `FfiItems` instance is always + /// left with no reexports that are not sourced from third-party crates. + pub(crate) fn visit_file(&mut self, file: &syn::File) { + let File { + attrs, + items: mod_items, + .. + } = file; + let file_mod = ItemMod { + attrs: attrs.clone(), + vis: Visibility::Public(syn::Token![pub](Span::call_site())), + unsafety: None, + mod_token: Mod::default(), + ident: Ident::new("__ctest_root_mod", Span::call_site()), + content: Some((Brace::default(), mod_items.clone())), + semi: None, + }; + + /* + [TODO]: the idea here is to change this into using a state monad that + lets us worry not about mutable state (even though syn's visitor + interface is mutable in nature, it would hide those details from us.) + This would carry both the current_module state that is now part of + FfiItems, as well as the unique ID that is assigned to each parsed + item. Then, we can get rid of the path_aggregate field of each of the + parsed items and the whole machinery built around it. Once we're + actually generating the test templates, we'll have to carry around + both the whole-crate FfiItems as well as the FfiItems specific to the + module for which the current test is being generated. That may not + need any specific type to carry the state because the whole-crate + FfiItems would only be needed up to filtering. That's when, for each + given item, we search its unique ID in a whole-crate traversal, such + that we run the user-configured skip with each of the occurrences of + the item in question. + */ + self.visit_item_mod(&file_mod); + + let root_module = Module { + items: self.clone(), + ..Default::default() + }; + + *self = resolve_all(root_module).items; + } + + /// Yields an iterator over all items in the calling `FfiItems`. + /// + /// The items in the item container do not include `use` declarations. + pub(crate) fn items(&self) -> ItemIterator { + let base = self + .aliases + .clone() + .into_iter() + .map(GenericItem::Type) + .chain(self.structs.clone().into_iter().map(GenericItem::Struct)) + .chain(self.unions.clone().into_iter().map(GenericItem::Union)) + .chain(self.constants.clone().into_iter().map(GenericItem::Const)) + .chain( + self.foreign_functions + .clone() + .into_iter() + .map(GenericItem::Fn), + ) + .chain( + self.foreign_statics + .clone() + .into_iter() + .map(GenericItem::Static), + ) + .chain(self.modules.clone().into_iter().map(GenericItem::Module)); + + Box::new(base) + } + + /// Performs a comparison for equality between `FfiItems`, without taking fields + /// `current_module` nor `uses` into consideration. + /// + /// This is not an implementation of `PartialEq` for `FfiItems` because, at the time of writing, + /// the logic for use declaration resolution is not yet finished. + pub(super) fn custom_eq(&self, other: &Self) -> bool { + self.aliases == other.aliases + && self.structs == other.structs + && self.unions == other.unions + && self.constants == other.constants + && self.foreign_functions == other.foreign_functions + && self.foreign_statics == other.foreign_statics + && self.modules == other.modules + } + + /// Comparison function for use when sorting and deduplicating modules in [`FfiItems::dedup`]. + /// + /// This is used in [`Module::custom_cmp`]. It trusts that the modules to compare have the same + /// overall shape, and one will be a "natural" extension of the other. + /// + /// We consider "extension" here to mean that one module subtree is based on the other, and thus + /// only contains both the items from the other and possibly some more items. + pub(super) fn custom_cmp(&self, other: &Self) -> Ordering { + let (ours, theirs) = (self.modules.len(), other.modules.len()); + + // [NOTE]: the goal here is to have modules with "larger" subtrees + // appear first in the item container, so declaring them to be "smaller" + // is required for slice::sort_unstable_by to put them first in the + // sorted list of modules in FfiItems::dedup. The same applies the other + // way around with "smaller" subtrees that we would like to have appear + // later in the sorted list of modules. + match ours.cmp(&theirs) { + Ordering::Equal => self.modules.iter().zip(&other.modules).fold( + Ordering::Equal, + |_, (our_m, their_m)| { + // [NOTE]: this is intended to not start anew a "regular" + // module comparison through Module::custom_cmp, but to + // rather have all subsequent comparisons be based off of + // the sizes of module subtrees rather than the + // lexicographical comparison of their paths. + our_m.items.custom_cmp(&their_m.items) + }, + ), + other => other.reverse(), + } + } + + /// Searches for an item in the container, and returns an owned instance of that item. + /// + /// The generic item may be pattern-matched to find out specifically which item was found. + fn search(&self, ident: &Ident) -> Option { + self.items().find(|it| { + macro_rules! cmp { + ($($var:tt),+ $(,)?) => { + match it {$( + GenericItem::$var(it) => ident == it.ident(), + )+} + }; + } + + cmp! { + Type, + Struct, + Union, + Const, + Fn, + Static, + Module, + } + }) + } + + /// Deduplicates the items in an item container by their path. + /// + /// Items of differing kind, like functions and constants, that may happen to have the same path + /// will not get deduplicated. A consequence of calling this method is that the items are also + /// sorted by their paths. + fn dedup(self) -> FfiItems { + macro_rules! dedup { + ($($it:tt),+ $(,)?) => {{ + macro_rules! key_fn { + () => { + |it| it.cached_path.clone() + }; + } + + let items = self.clone(); + + $( + let mut $it = self.$it; + + $it.sort_unstable_by_key(key_fn!()); + $it.dedup_by_key(key_fn!()); + + let items = FfiItems { + $it: $it, + ..items + }; + )+ + + items + }}; + } + + let new_items = dedup!( + aliases, + structs, + unions, + constants, + foreign_functions, + foreign_statics, + ); + + let mut new_modules = self.modules; + new_modules.sort_unstable_by(Module::custom_cmp); + new_modules.dedup_by(|Module { path: lhs, .. }, Module { path: rhs, .. }| *lhs == *rhs); + + FfiItems { + modules: new_modules, + ..new_items + } + } +} + +impl Default for FfiItems { + fn default() -> Self { + Self { + aliases: Default::default(), + structs: Default::default(), + unions: Default::default(), + constants: Default::default(), + foreign_functions: Default::default(), + foreign_statics: Default::default(), + modules: Default::default(), + uses: Default::default(), + current_module: Path { + leading_colon: None, + segments: Default::default(), + }, + } + } +} + +impl FromIterator for FfiItems { + fn from_iter>(iter: I) -> FfiItems { + let mut partitioned_items: Vec<_> = iter.into_iter().collect(); + + println!( + "[FfiItems::from_iter] partitioned_items: {:?}", + partitioned_items + .iter() + .map(|it| { + macro_rules! exe { + ($($ty:tt),+ $(,)?) => { + match it {$( + GenericItem::$ty(it) => it.cached_path.clone(), + )+} + }; + } + + exe! { + Type, + Struct, + Union, + Const, + Fn, + Static, + Module, + } + }) + .collect::>() + ); + + // [NOTE]: this is expected to sort all items by their kind, such that + // we end up with the following ordering: + // + // 1. Type aliases. + // 2. Stucts. + // 3. Unions. + // 4. Constants. + // 5. Foreign functions. + // 6. Foreign statics. + // 7. Modules. + // + // This is then used to partition over the resulting collection and get + // "buckets" of GenericItems that can be mapped to a concrete type + // through partial mapping. + partitioned_items.sort_by(|it1, it2| match (it1, it2) { + (GenericItem::Type(_), GenericItem::Type(_)) + | (GenericItem::Struct(_), GenericItem::Struct(_)) + | (GenericItem::Union(_), GenericItem::Union(_)) + | (GenericItem::Const(_), GenericItem::Const(_)) + | (GenericItem::Fn(_), GenericItem::Fn(_)) + | (GenericItem::Static(_), GenericItem::Static(_)) + | (GenericItem::Module(_), GenericItem::Module(_)) => Ordering::Equal, + (GenericItem::Type(_), GenericItem::Struct(_)) + | (GenericItem::Type(_), GenericItem::Union(_)) + | (GenericItem::Type(_), GenericItem::Const(_)) + | (GenericItem::Type(_), GenericItem::Fn(_)) + | (GenericItem::Type(_), GenericItem::Static(_)) + | (GenericItem::Type(_), GenericItem::Module(_)) + | (GenericItem::Struct(_), GenericItem::Union(_)) + | (GenericItem::Struct(_), GenericItem::Const(_)) + | (GenericItem::Struct(_), GenericItem::Fn(_)) + | (GenericItem::Struct(_), GenericItem::Static(_)) + | (GenericItem::Struct(_), GenericItem::Module(_)) + | (GenericItem::Union(_), GenericItem::Const(_)) + | (GenericItem::Union(_), GenericItem::Fn(_)) + | (GenericItem::Union(_), GenericItem::Static(_)) + | (GenericItem::Union(_), GenericItem::Module(_)) + | (GenericItem::Const(_), GenericItem::Fn(_)) + | (GenericItem::Const(_), GenericItem::Static(_)) + | (GenericItem::Const(_), GenericItem::Module(_)) + | (GenericItem::Fn(_), GenericItem::Static(_)) + | (GenericItem::Fn(_), GenericItem::Module(_)) + | (GenericItem::Static(_), GenericItem::Module(_)) => Ordering::Less, + (GenericItem::Struct(_), GenericItem::Type(_)) + | (GenericItem::Union(_), GenericItem::Type(_)) + | (GenericItem::Union(_), GenericItem::Struct(_)) + | (GenericItem::Const(_), GenericItem::Type(_)) + | (GenericItem::Const(_), GenericItem::Struct(_)) + | (GenericItem::Const(_), GenericItem::Union(_)) + | (GenericItem::Fn(_), GenericItem::Type(_)) + | (GenericItem::Fn(_), GenericItem::Struct(_)) + | (GenericItem::Fn(_), GenericItem::Union(_)) + | (GenericItem::Fn(_), GenericItem::Const(_)) + | (GenericItem::Static(_), GenericItem::Type(_)) + | (GenericItem::Static(_), GenericItem::Struct(_)) + | (GenericItem::Static(_), GenericItem::Union(_)) + | (GenericItem::Static(_), GenericItem::Const(_)) + | (GenericItem::Static(_), GenericItem::Fn(_)) + | (GenericItem::Module(_), GenericItem::Type(_)) + | (GenericItem::Module(_), GenericItem::Struct(_)) + | (GenericItem::Module(_), GenericItem::Union(_)) + | (GenericItem::Module(_), GenericItem::Const(_)) + | (GenericItem::Module(_), GenericItem::Fn(_)) + | (GenericItem::Module(_), GenericItem::Static(_)) => Ordering::Greater, + }); + + macro_rules! get_partitions { + ($($field:ident: $ty:tt),+ $(,)?) => {$( + macro_rules! search_idx { + ($start:expr, $pred:expr) => {{ + partitioned_items + .iter() + .enumerate() + .skip($start) + .skip_while(|(_, it)| $pred(it)) + .next() + .map(|(idx, _)| idx) + .unwrap_or(partitioned_items.len()) + }}; + } + + let start = + search_idx!(0, |it: &&GenericItem| matches!(it, GenericItem::$ty(_)).not()); + let end = search_idx!(start, |it: &&GenericItem| matches!(it, GenericItem::$ty(_))); + + let $field = if start >= end { + Vec::new() + } else { + let partition = partitioned_items + .get(start..end) + .map(|it| { + it.into_iter() + .map(|it| match it { + GenericItem::$ty(it) => it.clone(), + _ => unreachable!(concat!( + "Items have been sorted by their kind \ + before. This is expected to \ + correspond with the bucket ", + stringify!($field), + ".", + )), + }) + .collect::>() + }) + .expect( + "The slice must be valid because the range is \ + logical, and the components of the range are \ + indices into the collection.", + ); + + println!( + concat!( + "[FfiItems::from_iter] ", + stringify!($field), + "_partition: {:?}" + ), + partition + .iter() + .map(|it| &it.cached_path) + .collect::>(), + ); + + partition + }; + )+}; + } + + get_partitions! { + aliases: Type, + structs: Struct, + unions: Union, + constants: Const, + foreign_functions: Fn, + foreign_statics: Static, + modules: Module, + } + + FfiItems { + aliases, + structs, + unions, + constants, + foreign_functions, + foreign_statics, + modules, + ..Default::default() + } + } +} + +/// Sum over any item held within an `FfiItems` term. +#[derive(Debug, Clone)] +pub(crate) enum GenericItem { + Type(Type), + Struct(Struct), + Union(Union), + Const(Const), + Fn(Fn), + Static(Static), + Module(Module), +} + +/// An iterator over all items but `use` declarations in an `FfiItems` term. +type ItemIterator = Box>; + +/// Refined type for [`syn::UsePath`] that does not consider group paths. +/// +/// This is a lot like [`RefinedUse`] in that it only exists as replacement for [`syn::UsePath`] but +/// uses [`RefinedUseTree`] instead of [`syn::UseTree`]. See [`RefinedUseTree`] for details. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct RefinedUsePath { + ident: Ident, + tree: Box, +} + +impl Default for RefinedUsePath { + fn default() -> Self { + Self { + ident: Ident::new("__dummy", Span::call_site()), + tree: Box::default(), + } + } +} + +/// Refined type for [`syn::UseTree`] that does not consider group paths. +/// +/// This is the byproduct of normalizing a `use` declaration's path through [`normalize_path`]. That +/// function ensures some such path containing a group as its tail end is converted into a set of +/// paths without groups. +/// +/// This enumeration is thus equivalent to `syn::UseTree` but does not consider the case of +/// [`syn::UseTree::Group`]. +#[derive(Clone, Debug, PartialEq, Default)] +pub(crate) enum RefinedUseTree { + Path(RefinedUsePath), + Name(UseName), + Rename(UseRename), + #[default] + Glob, +} + +/// Refined type for `use` declarations that do not consider group paths. +/// +/// This exists only for the purpose of refining [`syn::UseTree`], which is used within +/// [`syn::ItemUse`]. See type [`RefinedUseTree`] for the actual refinement and details on the +/// purpose of this type. +#[derive(Clone, Debug, PartialEq, Default)] +pub(crate) struct RefinedUse { + public: bool, + tree: RefinedUseTree, +} + +/// Gets rid of group reexports in a `use` declaration by flattenning them into a set of individual +/// reexports. +fn normalize_path(path: syn::UseTree) -> Vec { + match path { + UseTree::Name(n) => vec![RefinedUseTree::Name(n)], + UseTree::Rename(r) => vec![RefinedUseTree::Rename(r)], + UseTree::Glob(_) => vec![RefinedUseTree::Glob], + + UseTree::Path(syn::UsePath { ident, tree, .. }) => normalize_path(*tree) + .into_iter() + .map(Box::new) + .map(|tree| { + RefinedUseTree::Path(RefinedUsePath { + ident: ident.clone(), + tree, + }) + }) + .collect(), + + UseTree::Group(syn::UseGroup { items, .. }) => { + items.into_iter().map(normalize_path).flatten().collect() + } + } +} + +/// Stateful pair carried through resolution passes. +/// +/// Each `use` declaration requires access to either one of (1) the module where it was sourced +/// from, or (2) the crate root's subtree, or (3) any one ascendant or descendant module of the +/// module where it was sourced from. +/// +/// Points (2) and (3) require having access to the entire crate's module tree. Some ancestor item +/// is looked up by the parent path to the item being searched with respect to. Point (1) is +/// satisfied by the `subtree` field. +#[derive(Debug, Clone)] +struct PassPair { + subtree: Module, + whole_crate: Module, +} + +/// This is the resolution driver. It is the entrypoint for whole crate resolution. +/// +/// It performs passes on a given module. It diffs the results of a whole-crate pass with the +/// pre-pass state. Equality in that comparison determines termination. +fn resolve_all(root: Module) -> Module { + let (ControlFlow::Break(m) | ControlFlow::Continue(m)) = + iter::repeat(()).try_fold(root, |root, _| { + // [NOTE]: it holds that a pass over the crate root will yield the + // crate root as its subtree. It is thus safe to ignore the + // whole-crate module tree. They are the same at this point. + let PassPair { + subtree: new_root, .. + } = resolve_pass(PassPair { + subtree: root.clone(), + whole_crate: root.clone(), + }); + + match new_root == root { + true => ControlFlow::Break(new_root), + false => ControlFlow::Continue(new_root), + } + }); + m +} + +/// Represents a resolved or unresolved `use` declaration. +/// +/// The state of resolution may change across resolution passes. It is used in [`merge_module`] to +/// merge whichever `use` declarations resolved to a set of items back into the module they were +/// sourced from. +#[derive(Debug, Clone)] +enum Resolution { + Resolved(FfiItems), + Unresolved, +} + +/// Performs one pass through the provided pair of a subtree and the whole-crate subtree. +/// +/// The input pair's fields correspond with the subtree in which to perform the resolution pass on, +/// and the tree at the crate root. The latter is required to resolve absolute imports that refer to +/// some item in an ancestor module that is more than one level up the tree than the subtree being +/// resolved. +/// +/// The returned pair contains updated snapshots of the input pair. The subtree may have had new +/// items added to it. The whole-crate module tree is guaranteed to have integrated these changes. +fn resolve_pass(src: PassPair) -> PassPair { + let ( + PassPair { + whole_crate: whole_crate_with_resolved_children, + .. + }, + resolved_children, + ) = src.subtree.items.modules.clone().into_iter().fold( + (src.clone(), Vec::new()), + |(PassPair { whole_crate, .. }, children), m| { + let pair = resolve_pass(PassPair { + subtree: m, + whole_crate: whole_crate, + }); + + let mut new_children = children; + new_children.push(pair.subtree.clone()); + + (pair, new_children) + }, + ); + + let root_with_resolved_children = Module { + items: FfiItems { + modules: resolved_children, + ..src.subtree.items + }, + ..src.subtree + }; + + let resolved_uses = resolve_one(PassPair { + subtree: root_with_resolved_children.clone(), + whole_crate: whole_crate_with_resolved_children.clone(), + }); + + let PassPair { + subtree: new_root_with_duplicates, + whole_crate: new_whole_crate_with_resolved_children, + } = merge_module( + PassPair { + subtree: root_with_resolved_children, + whole_crate: whole_crate_with_resolved_children, + }, + resolved_uses, + ); + + let new_root = Module { + items: new_root_with_duplicates.items.dedup(), + ..new_root_with_duplicates + }; + let new_whole_crate = update_tree( + &new_root.path, + |_| GenericItem::Module(new_root.clone()), + new_whole_crate_with_resolved_children, + ); + + println!( + "[resolve_pass] finished module pass on module {}", + if &*new_root.cached_path == "" { + "root".into() + } else { + &*new_root.cached_path + }, + ); + + PassPair { + subtree: new_root, + whole_crate: new_whole_crate, + } +} + +/// Attempts to resolve the `use` declarations in a given module. +/// +/// The input pair consists of the module from which to source the `use` declarations, and of the +/// whole-crate module tree. The latter is used to resolve certain types of paths in `use` +/// declarations. +/// +/// See [`resolve_pass`] for details. +fn resolve_one(state: PassPair) -> Vec { + println!( + "[resolve_one] found {} use declarations in module {}: {:?}", + state.subtree.items.uses.len(), + if &*state.subtree.cached_path == "" { + "root" + } else { + &*state.subtree.cached_path + }, + state + .subtree + .items + .uses + .iter() + .map(|RefinedUse { tree, .. }| tree) + .map(PrettyUseTree) + .collect::>(), + ); + + state + .subtree + .items + .uses + .clone() + .into_iter() + .map(|u| resolve_use(u, state.clone())) + .collect() +} + +/// Attempts to resolve a single `use` declaration. +/// +/// This can go one of two ways. +/// +/// 1. Either the `use` declaration matches a specific module in the stateful pair, or +/// 2. the `use` declaration is assumed to be unresolved in this resolution pass. +/// +/// The stateful pair initially consists of the module in which the `use` declaration appears at. If +/// the `use` declaration refers to an item by a path with more than one segment, this function +/// recursively munches through it. +/// +/// These recursive calls will only be made if the extracted path segment matches against some item +/// in the stateful subtree. Currently, the only types of items that can match on a path that still +/// has another unmunched segment are modules. +/// +/// Upon finding the module, the new stateful subtree of the pair becomes this module. This is then +/// used in the recursive call with the path tail. +/// +/// The other element of the pair is the whole-crate module tree. That is used to resolve ancestor +/// paths (when `super` is found) and absolute paths (when `crate` is found.) +fn resolve_use(r#use: RefinedUse, state: PassPair) -> Resolution { + macro_rules! log { + ($msg:literal $(, $complement:expr)* $(,)?) => { + println!( + "[resolve_use {:?}] {}", + PrettyUseTree(&(r#use.tree)), + format_args!($msg $(, $complement)*), + ); + }; + } + + macro_rules! single_item { + ($field:ident: $it:expr) => { + return Resolution::Resolved(FfiItems { + $field: vec![$it], + ..Default::default() + }) + }; + } + + match r#use.tree { + RefinedUseTree::Name(UseName { ref ident }) => { + log!( + "entered name resolution case with {} in module {}", + ident, + state.subtree.cached_path, + ); + + // [NOTE]: self may not appear as the name basecase of a well-formed + // path. And yet it is still considered here. That is because of + // path normalization. self may appear as part of a tail group in a + // use declaration's path. Path normalization flattens groups into + // individual use statements. Path normalization is done through + // normalize_path above. That happens at parse-time. That is why + // this function must consider the case of self as a name basecase. + // [NOTE]: crate does not appear as the tail end of a path in a use + // declaration (that is not a rename) under no circumstances, so + // that case is not considered here. + if is_self(&ident) { + single_item!(modules: state.subtree); + } else if is_super(&ident) { + let parent_path = get_parent_path(&state.subtree); + let Some(parent) = get_module(&parent_path, &state.whole_crate) else { + unreachable!( + "The module path is guaranteed to be found in the \ + whole crate module tree. It is the path of the \ + stateful subtree, which is either the module in which \ + the use declaration appears at, some descendant of \ + that module, or some descendant module to the crate \ + root after having stumbled on some ancestor-referring \ + use declaration. The parent some such module must be \ + below the crate root, as otherwise it would be \ + considered malformed input." + ); + }; + + single_item!(modules: parent.clone()); + } + + match state.subtree.items.search(&ident) { + Some(GenericItem::Type(t)) => single_item!(aliases: t), + Some(GenericItem::Struct(s)) => single_item!(structs: s), + Some(GenericItem::Union(u)) => single_item!(unions: u), + Some(GenericItem::Const(c)) => single_item!(constants: c), + Some(GenericItem::Fn(f)) => single_item!(foreign_functions: f), + Some(GenericItem::Static(s)) => single_item!(foreign_statics: s), + Some(GenericItem::Module(m)) => single_item!(modules: m), + None => Resolution::Unresolved, + } + } + RefinedUseTree::Glob => { + log!( + "entered glob resolution case in module {}", + if &*state.subtree.cached_path == "" { + "root" + } else { + &*state.subtree.cached_path + }, + ); + + Resolution::Resolved(state.subtree.items) + } + RefinedUseTree::Rename(UseRename { + ref ident, + ref rename, + .. + }) => { + log!( + "entered rename resolution case with {:?} in module {}", + UseRename { + ident: ident.clone(), + as_token: syn::parse_quote! { as }, + rename: rename.clone(), + }, + state.subtree.cached_path, + ); + + macro_rules! new_path { + ($it:expr) => {{ + let new_ident = $it.path.segments.iter().rev().take(1).map(|_| PathSegment { + ident: rename.clone(), + arguments: PathArguments::None, + }); + let new_segments = $it + .path + .segments + .clone() + .into_iter() + .chain(new_ident) + .collect(); + + Path { + segments: new_segments, + ..$it.path + } + }}; + } + + // [NOTE]: it may seem odd that borrowck doesn't mention the fact + // the linear resource in the item at $it hasn't already been used + // by the time the linear PathUtils::rename trait method is used. + // Notice how new_path! does not make use of any linear resources in + // $it; The remaining fields filled in by $it in the syn::Path it + // yields have an unrestricted multiplicity (i.e. they're Copy.) + macro_rules! rename_single_item { + ($field:ident , $it:expr , $ty:tt) => {{ + let new_path = new_path!($it); + + single_item!($field: $it.rename(new_path)); + }}; + } + + if is_self(&ident) { + rename_single_item!(modules, state.subtree, Module); + } else if is_super(&ident) { + let parent_path = get_parent_path(&state.subtree); + let Some(parent) = get_module(&parent_path, &state.whole_crate).cloned() else { + unreachable!( + "See the note on super in the name basecase. The same \ + applies here." + ); + }; + + rename_single_item!(modules, parent, Module); + } else if is_crate(&ident) { + rename_single_item!(modules, state.whole_crate, Module); + } + + match state.subtree.items.search(&ident) { + Some(GenericItem::Type(t)) => rename_single_item!(aliases, t, Type), + Some(GenericItem::Struct(s)) => rename_single_item!(structs, s, Struct), + Some(GenericItem::Union(u)) => rename_single_item!(unions, u, Union), + Some(GenericItem::Const(c)) => rename_single_item!(constants, c, Const), + Some(GenericItem::Fn(f)) => rename_single_item!(foreign_functions, f, Fn), + Some(GenericItem::Static(s)) => rename_single_item!(foreign_statics, s, Static), + Some(GenericItem::Module(m)) => rename_single_item!(modules, m, Module), + None => Resolution::Unresolved, + } + } + + RefinedUseTree::Path(RefinedUsePath { + ref ident, + ref tree, + }) => { + log!( + "entered path resolution case with {} in module {}", + ident, + state.subtree.cached_path, + ); + + let new_state = if is_self(&ident) { + state + } else if is_super(&ident) { + let parent_path = get_parent_path(&state.subtree); + let Some(parent) = get_module(&parent_path, &state.whole_crate).cloned() else { + unreachable!( + "See the note on super in the name basecase. The same \ + applies here." + ); + }; + + PassPair { + subtree: parent, + whole_crate: state.whole_crate, + } + } else if is_crate(&ident) { + PassPair { + subtree: state.whole_crate.clone(), + whole_crate: state.whole_crate, + } + } else if let Some(GenericItem::Module(m)) = state.subtree.items.search(&ident) { + PassPair { + subtree: m, + whole_crate: state.whole_crate, + } + } else { + return Resolution::Unresolved; + }; + + // [FIXME]: remove the below clone once debugging is done and the + // logging macro invocation above is removed. + resolve_use( + RefinedUse { + tree: *tree.clone(), + ..r#use + }, + new_state, + ) + } + } +} + +/// Attempts to find a given module, provided its path (`target`,) within a module subtree (`root`.) +fn get_module<'m>(target: &Path, root: &'m Module) -> Option<&'m Module> { + if root.path == *target { + return root.into(); + } + + root.items + .modules + .iter() + .find_map(|m| get_module(target, m)) +} + +/// Fetches the parent path to a given module. +/// +/// If the module is a top-level item (i.e. it appears at the crate root,) the returned path will be +/// empty. This follows from the parsing strategy for paths in `FfiItems`. The absolute paths are +/// all `crate`-_unprepended_. +fn get_parent_path(root: &Module) -> Path { + // [NOTE]: this performs a saturating_sub because there is one and only one + // item in the crate's module tree that has an empty path; The crate root. + // That is only wrapped in a Module of its own only after parsing. This + // function currently only gets used during parsing, so there is a chance + // this gets called with the virtual module that wraps the crate root's + // FfiItems. That is not logically possible, because well-formed input can + // not have a use declaration with super in the path at the crate root. This + // function is used when stumbling upon super in function resolve_use. + // + // The above assumption does not change the way this is implemented, except + // you could very well change the saturating_sub for overflowing + // subtraction. It is best if this function is protected against what would + // currently be misuse. + let segments_before_ident = root.path.segments.len().saturating_sub(1); + + Path { + leading_colon: None, + segments: root + .path + .segments + .iter() + .take(segments_before_ident) + .cloned() + .collect(), + } +} + +/// Checks if the given identifier was parsed as the Rust keyword `self`. +fn is_self(ident: &Ident) -> bool { + ident.to_string() == "self" +} + +/// Checks if the given identifier was parsed as the Rust keyword `crate`. +fn is_crate(ident: &Ident) -> bool { + ident.to_string() == "crate" +} + +/// Checks if the given identifier was parsed as the Rust keyword `super`. +fn is_super(ident: &Ident) -> bool { + ident.to_string() == "super" +} + +/// Attempts to find some item in module tree `root` with path `target`, then runs the passed +/// closure on the item, and replaces it with the result. +/// +/// # Panics +/// +/// The function will panic if the specific injection of the returned `GenericItem` does not match +/// that of the input `GenericItem` to closure `action`. +fn update_tree( + target: &Path, + action: impl ops::Fn(GenericItem) -> GenericItem + Clone, + root: Module, +) -> Module { + macro_rules! ignored { + () => { + unimplemented!( + "The action is not meant to return an item of a differing kind \ + than what was provided." + ); + }; + } + + if *target == root.path { + let GenericItem::Module(new_root) = action(GenericItem::Module(root)) else { + ignored!(); + }; + + return new_root; + } + + macro_rules! exe { + ($($field:tt: $ty:tt),+ $(,)?) => {$( + let $field: Vec<_> = root + .items + .$field + .into_iter() + .map(|it| { + if it.path == *target { + let GenericItem::$ty(it) = action(GenericItem::$ty(it)) else { + ignored!(); + }; + + it + } else { + it + } + }) + .collect(); + )+}; + } + + exe! { + aliases: Type, + structs: Struct, + unions: Union, + constants: Const, + foreign_functions: Fn, + foreign_statics: Static, + modules: Module, + } + + let modules = modules + .into_iter() + .map(|m| update_tree(target, action.clone(), m)) + .collect(); + + Module { + items: FfiItems { + aliases, + structs, + unions, + constants, + foreign_functions, + foreign_statics, + modules, + ..root.items + }, + ..root + } +} + +/// Integrates the resolution items of `src` into `dst`. +/// +/// Each resolution item is either one of an `Unresolved` or a `Resolved` variant. The former are +/// essentially ignored. The latter pack an `FfiItems` that is integrated into `dst`'s own items. +/// That is the merging part. +/// +/// Note that, during merging, the items to be merged may end up duplicating existing items on +/// `dst`. It is call-site's responsibility to deduplicate those through [`FfiItems::dedup`]. +fn merge_module(state: PassPair, src: Vec) -> PassPair { + println!( + "[merge_module] preprocessed_items: {:#?}", + src.iter() + .map(|r| { + if let Resolution::Resolved(it) = r { + format!("resolved: {:?}", DebugFfiItems::from(it.clone())) + } else { + format!("unresolved") + } + }) + .collect::>(), + ); + + // [NOTE]: we collect all items here because we want the actions in the + // mapping to take place in whole before we start folding over the items to + // merge. + let processed_items: Vec<_> = src + .into_iter() + .filter_map(|r| { + if let Resolution::Resolved(items) = r { + Some(items) + } else { + None + } + }) + .map( + |FfiItems { + aliases, + structs, + unions, + constants, + foreign_functions, + foreign_statics, + modules, + .. + }| { + macro_rules! seek_parent { + ($($it:tt),+ $(,)?) => { seek_parent!(@@ $($it)+) }; + + (@@ $head:tt $($tail:tt)*) => { + seek_parent!(@ $head).or_else(|| seek_parent!(@@ $($tail)*)) + }; + (@@) => { None }; + + (@ $field:tt) => {{ + $field.first().map(|it| { + it.path + .segments + .iter() + .take(it.path.segments.len() - 1) + .cloned() + .collect::>() + }) + }}; + } + + // [NOTE]: this ensures we find the immediate parent to any one + // of the items about to be merged. If there really are no items + // about to be merged, then we can simply return an empty + // FfiItems from the mapping, which would essentially merge no + // items down in the fold that finishes up the merging function. + // We don't invoke unreachable! in the diverging branch because + // a resolution item could pack an empty item container if a use + // declaration consists of a glob import of all items in a + // module with no items. We can be sure that all items about to + // be merged share the same parent because use declarations are + // cleansed of groups at parse-time. + let Some(source_module_path) = seek_parent!( + aliases, + structs, + unions, + constants, + foreign_functions, + foreign_statics, + modules, + ) else { + return FfiItems::default(); + }; + + // [NOTE]: the dummy module is only in charge of wrapping all + // items we want to rename with a path that makes up some + // initial portion of the path segments of all descendant items' + // path. This is so that the below call to function + // RenameUtils::rename can use it as reference for the subtree + // items' path. + let dummy_module = Module { + path: Path { + leading_colon: None, + segments: source_module_path, + }, + items: FfiItems { + aliases, + structs, + unions, + constants, + foreign_functions, + foreign_statics, + modules, + ..Default::default() + }, + ..Default::default() + }; + let Module { + items: new_items, .. + } = dummy_module.rename(state.subtree.path.clone()); + + fn remove_uses(m: Module) -> Module { + Module { + items: FfiItems { + uses: vec![], + modules: m.items.modules.into_iter().map(remove_uses).collect(), + ..m.items + }, + ..m + } + } + + // [NOTE]: this removes all use declarations in all descendant + // modules of the subtrees to merge because otherwise merged + // modules can increase the number of use declarations in the + // crate's module tree, which is one cause of non-termination. + let new_modules_without_uses = + new_items.modules.into_iter().map(remove_uses).collect(); + + FfiItems { + modules: new_modules_without_uses, + ..new_items + } + }, + ) + .collect(); + + println!( + "[merge_module] processed_items: {:#?}", + processed_items + .iter() + .cloned() + .map(|it| format!("{:?}", DebugFfiItems::from(it))) + .collect::>(), + ); + + processed_items.into_iter().fold( + state, + |PassPair { + subtree, + whole_crate, + }, + it| { + // [TODO]: this is still wrong but it also seems to be headed in the + // right direction. + // [FIXME]: get rid of the below clone once debugging is done. + let new_whole_crate = sync_driver(&it, whole_crate.clone()); + let Some(new_subtree) = get_module(&subtree.path, &new_whole_crate).cloned() else { + unreachable!( + "The subtree is bound to be found in the whole-crate \ + module tree because it was sourced from it, and syncing \ + the path aggregates of items should not suddently remove \ + modules." + ); + }; + + assert_ne!( + whole_crate, new_whole_crate, + "old whole-crate module tree matches new whole-crate module \ + tree even though there were items to merge", + ); + + println!( + "[merge_module] new_whole_crate: {:?}", + DebugFfiItems::from(new_whole_crate.items.clone()) + ); + + PassPair { + subtree: Module { + items: new_subtree.items.items().chain(it.items()).collect(), + ..new_subtree + }, + whole_crate: new_whole_crate, + } + }, + ) +} + +/// Provided an item container with the most up-to-date occurrences of some item(s) (`items`,) this +/// function will propagate the state of the path aggregate of each of those items throughout the +/// entirety of the subtree rooted at the `tree` module tree. +fn sync_driver(items: &FfiItems, tree: Module) -> Module { + items.items().fold(tree, |m, it| { + macro_rules! exe { + ($($ty:tt),+ $(,)?) => { + match it {$( + GenericItem::$ty(it) => sync_aggregates(&it.path_aggregate, m), + )+} + }; + } + + exe! { + Type, + Struct, + Union, + Const, + Fn, + Static, + Module, + } + }) +} + +/// Provided a reference path aggregate corresponding with that of an item, this function traverses +/// module `m` in search of all occurrences of such an item and "updates" such ocurrences to +/// integrate the paths from `reference`'s path aggregate store that are not part of any one of +/// those item occurrences. +/// +/// # Implementation notes +/// +/// The strategy follows that for any given item `a`, its path aggregate is bound to be unique with +/// respect to that of another item `b` so long as `a` is not an occurrence of `b` in another module +/// of the crate's module tree. We thus define the relation `a ~ b` as one where `b` contains some +/// path in its path aggregate that is also part of `a`'s path aggregate. +/// +/// If we fix `a` and it holds that `a ~ b`, then it follows that there exists some non-empty set +/// `B` of all occurrences `b` of `a` in the crate's module tree. Under the assumption that some two +/// items `a` and `b` may only share path aggregates it they are of the same kind, the below +/// strategy attempts to find akin path aggregates where `a ~ b` holds. This requires not for a +/// single kind of item to be checked against the fixed point (here `reference`) because of the +/// above assumption; Akin path aggregates will only be found for occurrences of the same item, and +/// thus the same kind of item, in the crate's module tree. +fn sync_aggregates(reference: &PathAggregateStore, m: Module) -> Module { + let akin = |agg: &PathAggregateStore| agg.akin(reference); + + println!( + "[sync_aggregates] preprocessed_module {}: {:?}", + if m.cached_path.is_empty() { + "root" + } else { + &m.cached_path + }, + DebugFfiItems::from(m.items.clone()), + ); + + let new_m = if akin(&m.path_aggregate) { + // [NOTE]: this we diff instead of merging because merging will get us a + // set of path aggregates that contain both the ones already in m, as + // well as the ones that were part of reference but not part of m. + // Diffing gets us only the paths in reference that are not part of m's + // path aggregate. We don't just merge the path aggregates and create + // a new m with the result because adding a path aggregate to a module + // has an effect on the subtree of items rooted at the module in + // question (see the Module::add_path trait method override.) + let diff = m.path_aggregate.diff(reference); + + diff.into_iter().fold(m, Module::add_path) + } else { + m + }; + + let new_items: FfiItems = new_m + .items + .items() + .map(|it| { + macro_rules! exe { + ($($ty:tt),+ $(,)?) => { + match it { + $( + GenericItem::$ty(it) if akin(&it.path_aggregate) => { + GenericItem::$ty($ty { + path_aggregate: reference.clone(), + ..it + }) + } + )+ + GenericItem::Module(it) => { + GenericItem::Module(sync_aggregates(reference, it)) + } + non_matching => non_matching, + } + }; + } + + exe! { + Type, + Struct, + Union, + Const, + Fn, + Static, + } + }) + .collect(); + + println!( + "[sync_aggregates] processed_module {}: {:?}", + if new_m.cached_path.is_empty() { + "root" + } else { + &new_m.cached_path + }, + DebugFfiItems::from(new_items.clone()), + ); + + Module { + items: new_items, + ..new_m + } +} + +/// Appends a new module-local item to an absolute path that does _not_ start with `crate`. +/// +/// This is used whenever we need to create paths for either items or modules. +fn append_path(base: Path, new: &Ident) -> Path { + if base.segments.is_empty() { + syn::parse_quote! { #new } + } else { + syn::parse_quote! { #base::#new } + } } /// Determine whether an item is visible to other crates. /// -/// This function assumes that if the visibility is restricted then it is not -/// meant to be accessed. +/// This function assumes that if the visibility is restricted then it is not meant to be accessed. fn is_visible(vis: &syn::Visibility) -> bool { match vis { syn::Visibility::Public(_) => true, @@ -135,7 +1745,9 @@ fn extract_single_link_name(attrs: &[syn::Attribute]) -> Option { fn visit_foreign_item_fn(table: &mut FfiItems, i: &syn::ForeignItemFn, abi: &Abi) { let public = is_visible(&i.vis); let abi = abi.clone(); - let ident = i.sig.ident.to_string().into_boxed_str(); + let path = append_path(table.current_module.clone(), &i.sig.ident); + let cached_path = ast::path_to_string(&path); + let cached_ident = i.sig.ident.to_string().into_boxed_str(); let parameters = i .sig .inputs @@ -145,13 +1757,19 @@ fn visit_foreign_item_fn(table: &mut FfiItems, i: &syn::ForeignItemFn, abi: &Abi ident: match arg.pat.deref() { syn::Pat::Ident(i) => i.ident.to_string().into_boxed_str(), _ => { - unimplemented!("Foreign functions are unlikely to have any other pattern.") + unimplemented!( + "Foreign functions are unlikely to have any other \ + pattern." + ) } }, ty: arg.ty.deref().clone(), }, syn::FnArg::Receiver(_) => { - unreachable!("Foreign functions can't have self/receiver parameters.") + unreachable!( + "Foreign functions can't have self/receiver \ + parameters." + ) } }) .collect::>(); @@ -164,7 +1782,10 @@ fn visit_foreign_item_fn(table: &mut FfiItems, i: &syn::ForeignItemFn, abi: &Abi table.foreign_functions.push(Fn { public, abi, - ident, + cached_path, + cached_ident, + path: path.clone(), + path_aggregate: PathAggregateStore::new().push(path.clone()), link_name, parameters, return_type, @@ -174,83 +1795,550 @@ fn visit_foreign_item_fn(table: &mut FfiItems, i: &syn::ForeignItemFn, abi: &Abi fn visit_foreign_item_static(table: &mut FfiItems, i: &syn::ForeignItemStatic, abi: &Abi) { let public = is_visible(&i.vis); let abi = abi.clone(); - let ident = i.ident.to_string().into_boxed_str(); + let path = append_path(table.current_module.clone(), &i.ident); + let cached_path = ast::path_to_string(&path); + let cached_ident = i.ident.to_string().into_boxed_str(); let ty = i.ty.deref().clone(); let link_name = extract_single_link_name(&i.attrs); table.foreign_statics.push(Static { public, abi, - ident, + cached_path, + cached_ident, + path: path.clone(), + path_aggregate: PathAggregateStore::new().push(path.clone()), link_name, ty, }); } -impl<'ast> Visit<'ast> for FfiItems { - fn visit_item_type(&mut self, i: &'ast syn::ItemType) { - let public = is_visible(&i.vis); - let ty = i.ty.deref().clone(); - let ident = i.ident.to_string().into_boxed_str(); +fn visit_item_type(table: &mut FfiItems, i: &syn::ItemType) { + let public = is_visible(&i.vis); + let path = append_path(table.current_module.clone(), &i.ident); + let cached_path = ast::path_to_string(&path); + let cached_ident = i.ident.to_string().into_boxed_str(); + let ty = i.ty.deref().clone(); + + table.aliases.push(Type { + public, + cached_path, + cached_ident, + path: path.clone(), + path_aggregate: PathAggregateStore::new().push(path.clone()), + ty, + }); +} + +fn visit_item_struct(table: &mut FfiItems, i: &syn::ItemStruct) { + let public = is_visible(&i.vis); + let path = append_path(table.current_module.clone(), &i.ident); + let cached_path = ast::path_to_string(&path); + let cached_ident = i.ident.to_string().into_boxed_str(); + let fields = match &i.fields { + syn::Fields::Named(fields) => collect_fields(&fields.named), + syn::Fields::Unnamed(fields) => collect_fields(&fields.unnamed), + syn::Fields::Unit => Vec::new(), + }; + + table.structs.push(Struct { + public, + cached_path, + cached_ident, + path: path.clone(), + path_aggregate: PathAggregateStore::new().push(path.clone()), + fields, + }); +} + +fn visit_item_union(table: &mut FfiItems, i: &syn::ItemUnion) { + let public = is_visible(&i.vis); + let path = append_path(table.current_module.clone(), &i.ident); + let cached_path = ast::path_to_string(&path); + let cached_ident = i.ident.to_string().into_boxed_str(); + let fields = collect_fields(&i.fields.named); + + table.unions.push(Union { + public, + cached_path, + cached_ident, + path: path.clone(), + path_aggregate: PathAggregateStore::new().push(path.clone()), + fields, + }); +} - self.aliases.push(Type { public, ident, ty }); +fn visit_item_const(table: &mut FfiItems, i: &syn::ItemConst) { + let public = is_visible(&i.vis); + let path = append_path(table.current_module.clone(), &i.ident); + let cached_path = ast::path_to_string(&path); + let cached_ident = i.ident.to_string().into_boxed_str(); + let ty = i.ty.deref().clone(); + + table.constants.push(Const { + public, + cached_path, + cached_ident, + path: path.clone(), + path_aggregate: PathAggregateStore::new().push(path.clone()), + ty, + }); +} + +fn visit_item_foreign_mod(table: &mut FfiItems, i: &syn::ItemForeignMod) { + // Because we need to store the ABI we can't directly visit the foreign + // functions/statics. + + // Since this is an extern block, assume extern "C" by default. + let abi = i + .abi + .name + .clone() + .map_or(Abi::C, |s| Abi::from(s.value().as_str())); + + for item in &i.items { + match item { + syn::ForeignItem::Fn(function) => visit_foreign_item_fn(table, function, &abi), + syn::ForeignItem::Static(static_variable) => { + visit_foreign_item_static(table, static_variable, &abi) + } + _ => (), + } } +} - fn visit_item_struct(&mut self, i: &'ast syn::ItemStruct) { - let public = is_visible(&i.vis); - let ident = i.ident.to_string().into_boxed_str(); - let fields = match &i.fields { - syn::Fields::Named(fields) => collect_fields(&fields.named), - syn::Fields::Unnamed(fields) => collect_fields(&fields.unnamed), - syn::Fields::Unit => Vec::new(), +/* + * [TODO]: this we can either change into using as value in the state monad a + * Module, or otherwise add further state to the parsing state that would carry + * the information of a given module. The latter option seems like the one that + * would require the least amount of changes. + */ +/* + * [TODO]: implement the unique ID stuff; Take care of which state is carried + * through state monads passed in (eventually) recursive call-sites. + */ +impl<'ast> Visit<'ast> for StateMonad { + fn visit_item_mod(&mut self, i: &'ast ItemMod) { + let ItemMod { + vis, + ident, + content: Some((_, items)), + .. + } = i + else { + /* + * [FIXME]: copy over the comment from the Visit implementation for + * FfiItems. + */ + unreachable!("Ibid.") }; - self.structs.push(Struct { - public, - ident, - fields, - }); - } + let is_crate_root = ident == "__ctest_root_mod"; + let base_monad = mem::replace(self, StateMonad::set(|s| (s, FfiItems::default()))); - fn visit_item_union(&mut self, i: &'ast syn::ItemUnion) { - let public = is_visible(&i.vis); - let ident = i.ident.to_string().into_boxed_str(); - let fields = collect_fields(&i.fields.named); + let current_monad = if is_crate_root { + base_monad + } else { + let ident = ident.clone(); + let vis = vis.clone(); - self.unions.push(Union { - public, - ident, - fields, + base_monad.bind(move |items| { + StateMonad::set( + move |ParsingState { + uid, + current_module, + .. + }| { + ( + ParsingState { + uid, + current_module: append_path(current_module, &ident), + module_is_public: matches!(vis, Visibility::Public(_)), + }, + items, + ) + }, + ) + }) + }; + + // [NOTE]: we don't parse modules here because they will get their state + // updated while folding through the items, which would mean we would + // lose the current_module piece of state that we keep in the state + // monad. + let new_monad = items + .iter() + .filter(|it| match it { + Item::Type(_) + | Item::Struct(_) + | Item::Union(_) + | Item::Const(_) + | Item::ForeignMod(_) + | Item::Mod(_) + | Item::Use(_) => true, + _ => false, + }) + .cloned() + .fold(current_monad, |monad, it| { + /* + [FIXME]: tweak the functions we currently use for parsing + items such that they use the state monad's state rather than + the state currently within FfiItems (i.e. field + current_module.) + */ + + macro_rules! exe { + ($($f:tt: $ty:tt),+ $(,)?) => { + match it { + $( + // [NOTE]: this is the one part where we have to + // fall into sin and allow in-place mutation of the + // items within the monad. On a more pragmatic note, + // this is fair because items is known to be a + // linear resource and this occurrence is + // independent of any other one in the universe. + Item::$ty(it) => monad.bind(move |mut items| { + $f(&mut items, &it); + + StateMonad::pure(items) + }), + )+ + Item::Mod(it) => { + let mut child_monad = StateMonad::pure(FfiItems::default()); + child_monad.visit_item_mod(&it); + + monad.bind(|items| { + StateMonad::set(|state: ParsingState| { + let (ParsingState { + uid, + module_is_public, + current_module, + .. + }, child_items) = child_monad.run(state.clone()); + + // [TODO]: this is still fuzzy. + let new_state = ParsingState { uid, ..state }; + let new_items = FfiItems { + modules: items + .modules + .into_iter() + .chain([Module { + public: module_is_public, + cached_ident: + ast::ident_to_string(¤t_module), + cached_path: + ast::path_to_string(¤t_module), + path: current_module, + items: child_items, + ..Default::default() + }]) + .collect(), + ..items + }; + + (new_state, new_items) + }) + }) + }, + Item::Use(ItemUse { vis, tree, .. }) => { + let base = normalize_path(tree).into_iter().map(move |tree| { + RefinedUse { + public: matches!(vis, Visibility::Public(_)), + tree, + } + }); + + monad.bind(|items| { + StateMonad::pure(FfiItems { + uses: items.uses.into_iter().chain(base).collect(), + ..items + }) + }) + } + _ => monad, + } + }; + } + + exe! { + visit_item_type: Type, + visit_item_struct: Struct, + visit_item_union: Union, + visit_item_const: Const, + visit_item_foreign_mod: ForeignMod, + } + }); + + *self = new_monad.bind(move |it| { + StateMonad::set(move |state: ParsingState| { + let mut child_monad = StateMonad::pure(FfiItems::default()); + + let ( + ParsingState { + uid, + module_is_public, + .. + }, + child_items, + ) = child_monad.run(state.clone()); + let new_state = ParsingState { + uid, + current_module: state.current_module, + module_is_public: state.module_is_public, + }; + let new_items = FfiItems { + modules: it.modules.into_iter().chain(child_items.modules).collect(), + ..it + }; + + (new_state, new_items) + }) }); } +} + +impl<'ast> Visit<'ast> for FfiItems { + fn visit_item_mod(&mut self, i: &'ast ItemMod) { + let ItemMod { + vis, + ident, + content: Some((_, mod_items)), + .. + } = i + else { + unreachable!( + "This runs post cargo-expand, which inlines all \ + modules." + ); + }; + let is_crate_root = ident == "__ctest_root_mod"; + + let mut running_items = if is_crate_root { + self.clone() + } else { + FfiItems { + current_module: append_path(self.current_module.clone(), &ident), + ..Default::default() + } + }; + + let uses = mod_items + .iter() + .filter_map(|it| match it { + Item::Type(t) => { + visit_item_type(&mut running_items, &t); + None + } + Item::Struct(s) => { + visit_item_struct(&mut running_items, &s); + None + } + Item::Union(u) => { + visit_item_union(&mut running_items, &u); + None + } + Item::Const(c) => { + visit_item_const(&mut running_items, &c); + None + } + Item::ForeignMod(m) => { + visit_item_foreign_mod(&mut running_items, &m); + None + } + Item::Use(ItemUse { vis, tree, .. }) => normalize_path(tree.clone()) + .into_iter() + .map(move |tree| RefinedUse { + public: matches!(vis, Visibility::Public(_)), + tree, + }) + .into(), + _ => None, + }) + .flatten() + .collect(); - fn visit_item_const(&mut self, i: &'ast syn::ItemConst) { - let public = is_visible(&i.vis); - let ident = i.ident.to_string().into_boxed_str(); - let ty = i.ty.deref().clone(); + // [NOTE]: this doesn't itself parse any items within the module because + // the only hook in syn::visit::Visit we have overridden is that of + // modules. In other words, this recurses only to attempt to find any + // child modules within the module currently being parsed. + visit::visit_item_mod(&mut running_items, i); + running_items.uses = uses; - self.constants.push(Const { public, ident, ty }); + if is_crate_root { + *self = running_items; + } else { + let path = running_items.current_module.clone(); + + self.modules.push(Module { + public: matches!(vis, Visibility::Public(_)), + cached_path: ast::path_to_string(&running_items.current_module), + cached_ident: ident.to_string().into_boxed_str(), + path: path.clone(), + path_aggregate: PathAggregateStore::from_iter([path]), + items: running_items, + }); + } } +} - fn visit_item_foreign_mod(&mut self, i: &'ast syn::ItemForeignMod) { - // Because we need to store the ABI we can't directly visit the foreign - // functions/statics. +#[test] +fn tmp() { + /* + * time should end up with the following paths: + * [time, test1::time, test1::test2::time, test2::time] + * Foo should end up with the following paths: + * [ + * Foo, + * test1::Foo, + * test1::test3::Foo, + * test1::test2::Foo, + * test1::test2::test3::Foo, + * test2::Foo, + * test2::test3::Foo, + * test3::Foo, + * ] + */ + let source = r#" +use test2::test3; - // Since this is an extern block, assume extern "C" by default. - let abi = i - .abi - .name - .clone() - .map_or(Abi::C, |s| Abi::from(s.value().as_str())); +use test2::*; + +use test1::{*, test3::*}; + +mod test1 { + use test2::{*, test3::*}; + + mod test2 { + use test3::*; - for item in &i.items { - match item { - syn::ForeignItem::Fn(function) => visit_foreign_item_fn(self, function, &abi), - syn::ForeignItem::Static(static_variable) => { - visit_foreign_item_static(self, static_variable, &abi) + extern "C" { + fn time(); + } + + mod test3 { + pub struct Foo; + + fn foo() { + extern "C" { + fn ctime(); } - _ => (), } } } } + "#; + let file = syn::parse_file(source).unwrap(); + + fn remove_uses(it: FfiItems) -> FfiItems { + let modules: Vec<_> = it + .modules + .into_iter() + .map(|m| Module { + items: remove_uses(m.items), + ..m + }) + .collect(); + + FfiItems { + modules, + uses: vec![], + ..it + } + } + + let mut items = FfiItems::default(); + items.visit_file(&file); + + println!("{:#?}", remove_uses(items)); +} + +#[test] +#[ignore] +fn tmp1() { + let source = r#" +pub struct Foo; + +mod test1 { use super::*; } + "#; + let file = syn::parse_file(source).unwrap(); + let mut items = FfiItems::default(); + items.visit_file(&file); + println!("{:#?}", items.clone()); +} + +#[test] +#[ignore] +fn tmp2() { + let m1 = Module { + cached_path: String::from("a").into_boxed_str(), + items: FfiItems::default(), + ..Default::default() + }; + let m2 = Module { + cached_path: String::from("a").into_boxed_str(), + items: FfiItems { + modules: vec![Module::default()], + ..Default::default() + }, + ..Default::default() + }; + let mut ms = vec![m1, m2]; + ms.sort_unstable_by(Module::custom_cmp); + println!("{:#?}", ms.clone()); +} + +// #[test] +// #[ignore] +// fn tmp3() { +// let dst_child_path = Path { +// leading_colon: None, +// segments: { +// let mut out = Punctuated::new(); +// out.push(PathSegment { +// ident: Ident::new("foo", Span::call_site()), +// arguments: PathArguments::None, +// }); +// out.clone() +// }, +// }; +// let dst = Module { +// items: FfiItems { +// modules: Vec::from([Module { +// path: dst_child_path.clone(), +// cached_path: ast::path_to_string(dst_child_path.clone()), +// items: FfiItems { +// structs: Vec::from([{ +// let path = Path { +// leading_colon: None, +// segments: Punctuated::new(), +// }; +// Struct { +// path: path.clone(), +// cached_path: ast::path_to_string(path.clone()), +// ..Default::default() +// } +// }]), +// ..Default::default() +// }, +// ..Default::default() +// }]), +// ..Default::default() +// }, +// ..Default::default() +// }; +// let src = Vec::from([Resolution::Resolved(FfiItems { +// modules: Vec::from([Module { +// path: dst_child_path.clone(), +// cached_path: ast::path_to_string(dst_child_path.clone()), +// items: FfiItems { +// modules: Vec::from([Module::default()]), +// uses: Vec::from([RefinedUse::default()]), +// ..Default::default() +// }, +// ..Default::default() +// }]), +// ..Default::default() +// })]); +// let merged = merge_module(dst.clone(), src.clone()); +// println!("{:#?}", merged.clone()); +// } diff --git a/ctest/src/generator.rs b/ctest/src/generator.rs index 8f31af4488375..717cb1166b062 100644 --- a/ctest/src/generator.rs +++ b/ctest/src/generator.rs @@ -10,7 +10,6 @@ use std::path::{ use std::rc::Rc; use askama::Template; -use syn::visit::Visit; use thiserror::Error; use crate::ffi_items::FfiItems; @@ -26,6 +25,7 @@ use crate::{ Field, Language, MapInput, + Module, Parameter, Result, Static, @@ -40,8 +40,8 @@ use crate::{ /// The default Rust edition used to generate the code. const DEFAULT_EDITION: u32 = 2021; -/// A function that takes a mappable input and returns its mapping as `Some`, otherwise -/// use the default name if `None`. +/// A function that takes a mappable input and returns its mapping as `Some`, +/// otherwise use the default name if `None`. type MappedName = Rc Option>; /// A function that determines whether to skip an item or not. type Skip = Rc bool>; @@ -49,7 +49,8 @@ type Skip = Rc bool>; type VolatileItem = Rc bool>; /// A function that determines whether a function argument is an array. type ArrayArg = Rc bool>; -/// A function that determines whether to skip a test, taking in the identifier name. +/// A function that determines whether to skip a test, taking in the identifier +/// name. type SkipTest = Rc bool>; /// A function that determines whether a type alias is a c enum. type CEnum = Rc bool>; @@ -58,10 +59,10 @@ type CEnum = Rc bool>; #[derive(Clone, Default)] #[expect(missing_debug_implementations)] pub struct TestGenerator { - /// A vector of tuples, the left side being the header itself, and the right - /// being a list of defines that the header is associated with. Note that - /// these defines are only valid for the header, they are immediately undefined - /// afterwards. + /// A vector of tuples, the left side being the header itself, and the + /// right being a list of defines that the header is associated with. + /// Note that these defines are only valid for the header, they are + /// immediately undefined afterwards. pub(crate) headers: Vec<(BoxStr, Vec)>, /// The target that the tests run on. Defaults to the native target. pub(crate) target: Option, @@ -69,7 +70,8 @@ pub struct TestGenerator { pub(crate) includes: Vec, /// The directory to output the generated test files. out_dir: Option, - /// A list of flags to pass to the compiler with checking if they are supported. + /// A list of flags to pass to the compiler with checking if they are + /// supported. pub(crate) flags: Vec, /// A list of flags that are passed to the compiler if supported. pub(crate) flags_if_supported: Vec, @@ -79,7 +81,8 @@ pub struct TestGenerator { cfg: Vec<(String, Option)>, /// A list of functions that remaps names used in the tests. mapped_names: Vec, - /// Extra command line args to pass to cargo when generating macro expansions. + /// Extra command line args to pass to cargo when generating macro + /// expansions. macro_expansion_cargo_args: Vec, /// Crate name to use when performing macro expansion. crate_name: Option, @@ -93,7 +96,8 @@ pub struct TestGenerator { pub(crate) volatile_items: Vec, /// A list of functions that determine if an item is a C style enum. pub(crate) c_enums: Vec, - /// A list of functions that determine if a type is actually an array argument. + /// A list of functions that determine if a type is actually an array + /// argument. pub(crate) array_arg: Option, /// Whether to skip testing private items. pub(crate) skip_private: bool, @@ -110,19 +114,23 @@ pub struct TestGenerator { /// An error that occurs when generating the test files. #[derive(Debug, Error)] pub enum GenerationError { - /// An error that occurs when `rustc -Zunpretty=expand` fails to expand the crate. + /// An error that occurs when `rustc -Zunpretty=expand` fails to expand the + /// crate. #[error("unable to expand crate {0}: {1}")] MacroExpansion(PathBuf, String), - /// An error that occurs when `syn` is unable to parse the expanded crate due to invalid syntax. + /// An error that occurs when `syn` is unable to parse the expanded crate + /// due to invalid syntax. #[error("unable to parse expanded crate {0}: {1}")] RustSyntax(String, String), /// An error that occurs when the Rust to C translation fails. #[error("unable to prepare template input: {0}")] Translation(#[from] TranslationError), - /// An error that occurs when there are errors in the Rust side of the test template. + /// An error that occurs when there are errors in the Rust side of the test + /// template. #[error("unable to render Rust template: {0}")] RustTemplateRender(askama::Error), - /// An error that occurs when there are errors in the C side of the test template. + /// An error that occurs when there are errors in the C side of the test + /// template. #[error("unable to render C template: {0}")] CTemplateRender(askama::Error), #[error("unable to create or write template file: {0}")] @@ -151,20 +159,20 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.header("foo.h") - /// .header("bar.h"); + /// cfg.header("foo.h").header("bar.h"); /// ``` pub fn header(&mut self, header: &str) -> &mut Self { self.headers.push((header.into(), vec![])); self } - /// Add a header to be included as part of the generated C file, as well as defines for it. + /// Add a header to be included as part of the generated C file, as well as + /// defines for it. /// /// The generated C test will be compiled by a C compiler, and this can be /// used to ensure that all the necessary header files are included to test - /// all FFI definitions. The defines are only set for the inclusion of that header file, and are - /// undefined immediately after. + /// all FFI definitions. The defines are only set for the inclusion of that + /// header file, and are undefined immediately after. /// /// # Examples /// @@ -173,7 +181,7 @@ impl TestGenerator { /// /// let mut cfg = TestGenerator::new(); /// cfg.header_with_defines("foo.h", Vec::::new()) - /// .header_with_defines("bar.h", vec!["DEBUG", "DEPRECATED"]); + /// .header_with_defines("bar.h", vec!["DEBUG", "DEPRECATED"]); /// ``` pub fn header_with_defines( &mut self, @@ -189,13 +197,17 @@ impl TestGenerator { /// Sets the programming language, by default it is C. /// - /// This determines what compiler is chosen to compile the C/C++ tests, as well as adding - /// external linkage to the tests if set to C++, so that they can be used in Rust. + /// This determines what compiler is chosen to compile the C/C++ tests, as + /// well as adding external linkage to the tests if set to C++, so that + /// they can be used in Rust. /// /// # Examples /// /// ```no_run - /// use ctest::{TestGenerator, Language}; + /// use ctest::{ + /// Language, + /// TestGenerator, + /// }; /// /// let mut cfg = TestGenerator::new(); /// cfg.language(Language::CXX); @@ -244,7 +256,7 @@ impl TestGenerator { /// /// let mut cfg = TestGenerator::new(); /// cfg.cfg("foo", None) // cfg!(foo) - /// .cfg("bar", Some("baz")); // cfg!(bar = "baz") + /// .cfg("bar", Some("baz")); // cfg!(bar = "baz") /// ``` pub fn cfg(&mut self, k: &str, v: Option<&str>) -> &mut Self { self.cfg.push((k.to_string(), v.map(|s| s.to_string()))); @@ -336,6 +348,7 @@ impl TestGenerator { /// Indicate that a type alias is actually a C enum. /// /// # Examples + /// /// ```no_run /// use ctest::TestGenerator; /// @@ -352,7 +365,10 @@ impl TestGenerator { /// # Examples /// /// ```no_run - /// use ctest::{TestGenerator, VolatileItemKind}; + /// use ctest::{ + /// TestGenerator, + /// VolatileItemKind, + /// }; /// /// let mut cfg = TestGenerator::new(); /// cfg.volatile_struct_field(|s, f| { @@ -378,12 +394,13 @@ impl TestGenerator { /// # Examples /// /// ```no_run - /// use ctest::{TestGenerator, VolatileItemKind}; + /// use ctest::{ + /// TestGenerator, + /// VolatileItemKind, + /// }; /// /// let mut cfg = TestGenerator::new(); - /// cfg.volatile_static(|s| { - /// s.ident() == "foo_t" - /// }); + /// cfg.volatile_static(|s| s.ident() == "foo_t"); /// ``` pub fn volatile_static(&mut self, f: impl Fn(Static) -> bool + 'static) -> &mut Self { self.volatile_items.push(Rc::new(move |item| { @@ -401,12 +418,13 @@ impl TestGenerator { /// # Examples /// /// ```no_run - /// use ctest::{TestGenerator, VolatileItemKind}; + /// use ctest::{ + /// TestGenerator, + /// VolatileItemKind, + /// }; /// /// let mut cfg = TestGenerator::new(); - /// cfg.volatile_fn_arg(|f, _p| { - /// f.ident() == "size_of_T" - /// }); + /// cfg.volatile_fn_arg(|f, _p| f.ident() == "size_of_T"); /// ``` pub fn volatile_fn_arg( &mut self, @@ -427,12 +445,13 @@ impl TestGenerator { /// # Examples /// /// ```no_run - /// use ctest::{TestGenerator, VolatileItemKind}; + /// use ctest::{ + /// TestGenerator, + /// VolatileItemKind, + /// }; /// /// let mut cfg = TestGenerator::new(); - /// cfg.volatile_fn_return_type(|f| { - /// f.ident() == "size_of_T" - /// }); + /// cfg.volatile_fn_return_type(|f| f.ident() == "size_of_T"); /// ``` pub fn volatile_fn_return_type( &mut self, @@ -450,8 +469,8 @@ impl TestGenerator { /// Indicate that a function pointer argument is an array. /// - /// This closure should return true if a pointer argument to a function should be generated - /// with `T foo[]` syntax rather than `T *foo`. + /// This closure should return true if a pointer argument to a function + /// should be generated with `T foo[]` syntax rather than `T *foo`. /// /// # Examples /// @@ -459,17 +478,42 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.array_arg(|func, arg| { - /// match (func.ident(), arg.ident()) { - /// ("foo", "bar") => true, - /// _ => false, - /// }}); + /// cfg.array_arg(|func, arg| match (func.ident().as_str(), arg.ident()) { + /// ("foo", "bar") => true, + /// _ => false, + /// }); /// ``` pub fn array_arg(&mut self, f: impl Fn(crate::Fn, Parameter) -> bool + 'static) -> &mut Self { self.array_arg = Some(Rc::new(f)); self } + /// Skip a specific module in the crate. + /// + /// Module paths are given relative to the crate root, so for example the + /// identifier of a module `bar` inside a top-level module `foo` would be + /// `foo::bar`, and not `crate::foo::bar`. This is returned by the + /// [`Module::path`] function. + /// + /// # Examples + /// + /// ```no_run + /// use ctest::TestGenerator; + /// + /// let mut cfg = TestGenerator::new(); + /// cfg.skip_module(|module| module.ident() == "foo::bar"); + /// ``` + pub fn skip_module(&mut self, f: impl Fn(&Module) -> bool + 'static) -> &mut Self { + self.skips.push(Rc::new(move |item| { + if let MapInput::Module(module) = item { + f(module) + } else { + false + } + })); + self + } + /// Configures whether the tests for a struct are emitted. /// /// # Examples @@ -478,9 +522,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.skip_struct(|s| { - /// s.ident().starts_with("foo_") - /// }); + /// cfg.skip_struct(|s| s.ident().starts_with("foo_")); /// ``` pub fn skip_struct(&mut self, f: impl Fn(&Struct) -> bool + 'static) -> &mut Self { self.skips.push(Rc::new(move |item| { @@ -501,9 +543,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.skip_union(|u| { - /// u.ident().starts_with("foo_") - /// }); + /// cfg.skip_union(|u| u.ident().starts_with("foo_")); /// ``` pub fn skip_union(&mut self, f: impl Fn(&Union) -> bool + 'static) -> &mut Self { self.skips.push(Rc::new(move |item| { @@ -573,9 +613,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.skip_alias(|a| { - /// a.ident().starts_with("foo_") - /// }); + /// cfg.skip_alias(|a| a.ident().starts_with("foo_")); /// ``` pub fn skip_alias(&mut self, f: impl Fn(&Type) -> bool + 'static) -> &mut Self { self.skips.push(Rc::new(move |item| { @@ -596,9 +634,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.skip_const(|s| { - /// s.ident().starts_with("FOO_") - /// }); + /// cfg.skip_const(|s| s.ident().starts_with("FOO_")); /// ``` pub fn skip_const(&mut self, f: impl Fn(&Const) -> bool + 'static) -> &mut Self { self.skips.push(Rc::new(move |item| { @@ -619,9 +655,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.skip_static(|s| { - /// s.ident().starts_with("foo_") - /// }); + /// cfg.skip_static(|s| s.ident().starts_with("foo_")); /// ``` pub fn skip_static(&mut self, f: impl Fn(&Static) -> bool + 'static) -> &mut Self { self.skips.push(Rc::new(move |item| { @@ -642,9 +676,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.skip_fn(|s| { - /// s.ident().starts_with("foo_") - /// }); + /// cfg.skip_fn(|s| s.ident().starts_with("foo_")); /// ``` pub fn skip_fn(&mut self, f: impl Fn(&crate::Fn) -> bool + 'static) -> &mut Self { self.skips.push(Rc::new(move |item| { @@ -659,8 +691,9 @@ impl TestGenerator { /// Configures whether tests for a C enum are generated. /// - /// A C enum consists of a type alias, as well as constants that have the same type. Tests - /// for both the alias as well as the constants are skipped. + /// A C enum consists of a type alias, as well as constants that have the + /// same type. Tests for both the alias as well as the constants are + /// skipped. /// /// # Examples /// @@ -710,8 +743,9 @@ impl TestGenerator { /// Set a `-D` flag for the C compiler being called. /// - /// This can be used to define various global variables to configure how header - /// files are included or what APIs are exposed from header files. + /// This can be used to define various global variables to configure how + /// header files are included or what APIs are exposed from header + /// files. /// /// # Examples /// @@ -719,8 +753,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.define("_GNU_SOURCE", None) - /// .define("_WIN32_WINNT", Some("0x8000")); + /// cfg.define("_GNU_SOURCE", None).define("_WIN32_WINNT", Some("0x8000")); /// ``` pub fn define(&mut self, k: &str, v: Option<&str>) -> &mut Self { self.global_defines @@ -736,9 +769,9 @@ impl TestGenerator { /// Configures the crate name which should be used during macro expansion. /// - /// If the tested crate uses `#![crate_name = "..."]`, this must be called with the - /// same name. Otherwise, there will be an error about `--crate-name` not - /// matching. + /// If the tested crate uses `#![crate_name = "..."]`, this must be called + /// with the same name. Otherwise, there will be an error about + /// `--crate-name` not matching. pub fn crate_name(&mut self, name: String) -> &mut Self { self.crate_name = Some(name); self @@ -822,9 +855,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.rename_constant(|c| { - /// (c.ident() == "FOO").then_some("BAR".to_string()) - /// }); + /// cfg.rename_constant(|c| (c.ident() == "FOO").then_some("BAR".to_string())); /// ``` pub fn rename_constant(&mut self, f: impl Fn(&Const) -> Option + 'static) -> &mut Self { self.mapped_names.push(Rc::new(move |item| { @@ -962,9 +993,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.rename_type(|ty| { - /// Some(format!("{}_t", ty)) - /// }); + /// cfg.rename_type(|ty| Some(format!("{}_t", ty))); /// ``` pub fn rename_type(&mut self, f: impl Fn(&str) -> Option + 'static) -> &mut Self { self.mapped_names.push(Rc::new(move |item| { @@ -989,9 +1018,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.rename_struct_ty(|ty| { - /// (ty == "timeval").then(|| format!("{ty}_t")) - /// }); + /// cfg.rename_struct_ty(|ty| (ty == "timeval").then(|| format!("{ty}_t"))); /// ``` pub fn rename_struct_ty(&mut self, f: impl Fn(&str) -> Option + 'static) -> &mut Self { self.mapped_names.push(Rc::new(move |item| { @@ -1016,9 +1043,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.rename_struct_ty(|ty| { - /// (ty == "T1Union").then(|| format!("__{ty}")) - /// }); + /// cfg.rename_struct_ty(|ty| (ty == "T1Union").then(|| format!("__{ty}"))); /// ``` pub fn rename_union_ty(&mut self, f: impl Fn(&str) -> Option + 'static) -> &mut Self { self.mapped_names.push(Rc::new(move |item| { @@ -1045,9 +1070,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.skip_roundtrip(|s| { - /// s.starts_with("foo_") - /// }); + /// cfg.skip_roundtrip(|s| s.starts_with("foo_")); /// ``` pub fn skip_roundtrip(&mut self, f: impl Fn(&str) -> bool + 'static) -> &mut Self { self.skip_roundtrip = Some(Rc::new(f)); @@ -1067,9 +1090,7 @@ impl TestGenerator { /// use ctest::TestGenerator; /// /// let mut cfg = TestGenerator::new(); - /// cfg.skip_signededness(|s| { - /// s.starts_with("foo_") - /// }); + /// cfg.skip_signededness(|s| s.starts_with("foo_")); /// ``` pub fn skip_signededness(&mut self, f: impl Fn(&str) -> bool + 'static) -> &mut Self { self.skip_signededness = Some(Rc::new(f)); @@ -1163,7 +1184,8 @@ impl TestGenerator { Ok(output_file_path) } - /// Maps Rust identifiers or types to C counterparts, or defaults to the original name. + /// Maps Rust identifiers or types to C counterparts, or defaults to the + /// original name. pub(crate) fn rty_to_cty<'a>(&self, item: impl Into>) -> String { let item = item.into(); if let Some(mapped) = self.mapped_names.iter().find_map(|f| f(&item)) { @@ -1184,6 +1206,10 @@ impl TestGenerator { MapInput::StructFieldType(_, f) => f.ident().to_string(), MapInput::UnionFieldType(_, f) => f.ident().to_string(), MapInput::Type(ty) => translate_primitive_type(ty), + + MapInput::Module(_) => { + unreachable!("modules don't get tested on the c side of things") + } } } } diff --git a/ctest/src/lib.rs b/ctest/src/lib.rs index c03f9de54d524..0e9da383e3bcc 100644 --- a/ctest/src/lib.rs +++ b/ctest/src/lib.rs @@ -27,6 +27,7 @@ pub use ast::{ Const, Field, Fn, + Module, Parameter, Static, Struct, @@ -90,6 +91,7 @@ pub(crate) enum MapInput<'a> { CEnumType(&'a str), StructFieldType(&'a Struct, &'a Field), UnionFieldType(&'a Union, &'a Field), + Module(&'a Module), } /// The language used to generate the tests. @@ -167,3 +169,9 @@ impl<'a> From<&'a Union> for MapInput<'a> { MapInput::Union(u) } } + +impl<'a> From<&'a Module> for MapInput<'a> { + fn from(value: &'a Module) -> Self { + MapInput::Module(value) + } +} diff --git a/ctest/src/template.rs b/ctest/src/template.rs index 4ed9fe559e52f..66aa7b1768bcc 100644 --- a/ctest/src/template.rs +++ b/ctest/src/template.rs @@ -1,10 +1,13 @@ //! Generation of tests from templates for both Rust and C. +use std::convert; + use askama::Template; use proc_macro2::Span; use quote::ToTokens; use syn::spanned::Spanned; +use crate::ast::PathUtils; use crate::cdecl::Constness; use crate::ffi_items::FfiItems; use crate::translator::{ @@ -13,12 +16,20 @@ use crate::translator::{ }; use crate::{ BoxStr, + Const, Field, + Fn, MapInput, + Module, Result, + Static, + Struct, TestGenerator, TranslationError, + Type, + Union, VolatileItemKind, + ast, cdecl, }; @@ -93,7 +104,8 @@ pub(crate) struct TestTemplate { } impl TestTemplate { - /// Populate all tests for all items depending on the configuration provided. + /// Populate all tests for all items depending on the configuration + /// provided. pub(crate) fn new( ffi_items: &FfiItems, generator: &TestGenerator, @@ -113,7 +125,8 @@ impl TestTemplate { Ok(template) } - /// Populates tests for constants and C-str constants, keeping track of the names of each test. + /// Populates tests for constants and C-str constants, keeping track of the + /// names of each test. fn populate_const_and_cstr_tests( &mut self, helper: &TranslateHelper, @@ -125,18 +138,27 @@ impl TestTemplate { && matches!(ptr.mutability, syn::PointerMutability::Const(_)) { let item = TestCStr { + // [NOTE]: the test name needs the full path to the item + // with path separators escaped as `_`. + test_name: cstr_test_ident(&ast::escape_item_path(constant)), + // [NOTE]: the item's identifier ought be the last segment + // of its path because it is used in the C tests. id: constant.ident().into(), - test_name: cstr_test_ident(constant.ident()), - rust_val: constant.ident().into(), + // [NOTE]: the item's Rust identifier ought be the full, + // unescaped path to the item (as it itself is not + // used as an identifier for some other item in the + // generated tests but as a standalone identifier.) + rust_val: constant.path().into(), c_val: helper.c_ident(constant).into(), }; self.const_cstr_tests.push(item.clone()); self.test_idents.push(item.test_name); } else { + // [NOTE]: see the above notes. let item = TestConst { id: constant.ident().into(), - test_name: const_test_ident(constant.ident()), - rust_val: constant.ident().into(), + test_name: const_test_ident(&ast::escape_item_path(constant)), + rust_val: constant.path().into(), rust_ty: constant.ty.to_token_stream().to_string().into_boxed_str(), c_val: helper.c_ident(constant).into(), c_ty: helper.c_type(constant)?.into(), @@ -158,9 +180,9 @@ impl TestTemplate { ) -> Result<(), TranslationError> { for alias in helper.filtered_ffi_items.aliases() { let item = TestSizeAlign { - test_name: size_align_test_ident(alias.ident()), + test_name: size_align_test_ident(&ast::escape_item_path(alias)), id: alias.ident().into(), - rust_ty: alias.ident().into(), + rust_ty: alias.path().into(), c_ty: helper.c_type(alias)?.into(), }; self.size_align_tests.push(item.clone()); @@ -168,9 +190,9 @@ impl TestTemplate { } for struct_ in helper.filtered_ffi_items.structs() { let item = TestSizeAlign { - test_name: size_align_test_ident(struct_.ident()), + test_name: size_align_test_ident(&ast::escape_item_path(struct_)), id: struct_.ident().into(), - rust_ty: struct_.ident().into(), + rust_ty: struct_.path().into(), c_ty: helper.c_type(struct_)?.into(), }; self.size_align_tests.push(item.clone()); @@ -178,9 +200,9 @@ impl TestTemplate { } for union_ in helper.filtered_ffi_items.unions() { let item = TestSizeAlign { - test_name: size_align_test_ident(union_.ident()), + test_name: size_align_test_ident(&ast::escape_item_path(union_)), id: union_.ident().into(), - rust_ty: union_.ident().into(), + rust_ty: union_.path().into(), c_ty: helper.c_type(union_)?.into(), }; self.size_align_tests.push(item.clone()); @@ -202,13 +224,13 @@ impl TestTemplate { .generator .skip_signededness .as_ref() - .is_some_and(|skip| skip(alias.ident())); + .is_some_and(|skip| skip(alias.path())); if !helper.translator.is_signed(&alias.ty) || should_skip_signededness_test { continue; } let item = TestSignededness { - test_name: signededness_test_ident(alias.ident()), + test_name: signededness_test_ident(&ast::escape_item_path(alias)), id: alias.ident().into(), c_ty: helper.c_type(alias)?.into(), }; @@ -238,7 +260,7 @@ impl TestTemplate { }) .map(|(struct_, field)| { ( - struct_.ident(), + MapInput::Struct(struct_), field, helper.c_type(struct_), helper.c_ident(MapInput::StructField(struct_, field)), @@ -254,19 +276,29 @@ impl TestTemplate { }) .map(|(union_, field)| { ( - union_.ident(), + MapInput::Union(union_), field, helper.c_type(union_), helper.c_ident(MapInput::UnionField(union_, field)), ) }); - for (id, field, c_ty, c_field) in struct_fields.chain(union_fields) { + for (ty, field, c_ty, c_field) in struct_fields.chain(union_fields) { + let (escaped_path, id, path) = match ty { + MapInput::Struct(s) => (ast::escape_item_path(s), s.ident(), s.path()), + MapInput::Union(u) => (ast::escape_item_path(u), u.ident(), u.path()), + + _ => unreachable!( + "we consider only records and untagged unions in field \ + size/offset tests" + ), + }; let item = TestFieldSizeOffset { - test_name: field_size_offset_test_ident(id, field.ident()), + test_name: field_size_offset_test_ident(&escaped_path, field.ident()), id: id.into(), - c_ty: c_ty?.into(), field: field.clone(), + rust_ty: path.into(), + c_ty: c_ty?.into(), c_field: c_field.into_boxed_str(), }; self.field_size_offset_tests.push(item.clone()); @@ -288,16 +320,13 @@ impl TestTemplate { if let syn::Type::Array(_) = alias.ty { continue; } - let c_ty = helper.c_type(alias)?; - self.add_roundtrip_test(helper, alias.ident(), &[], &c_ty, true); + self.add_roundtrip_test(helper, MapInput::Alias(alias), &[], true)?; } for struct_ in helper.filtered_ffi_items.structs() { - let c_ty = helper.c_type(struct_)?; - self.add_roundtrip_test(helper, struct_.ident(), &struct_.fields, &c_ty, false); + self.add_roundtrip_test(helper, MapInput::Struct(struct_), &struct_.fields, false)?; } for union_ in helper.filtered_ffi_items.unions() { - let c_ty = helper.c_type(union_)?; - self.add_roundtrip_test(helper, union_.ident(), &union_.fields, &c_ty, false); + self.add_roundtrip_test(helper, MapInput::Union(union_), &union_.fields, false)?; } Ok(()) @@ -306,27 +335,54 @@ impl TestTemplate { fn add_roundtrip_test( &mut self, helper: &TranslateHelper, - ident: &str, + input: MapInput, fields: &[Field], - c_ty: &str, is_alias: bool, - ) { + ) -> Result<(), TranslationError> { + let (escaped_path, ident, path, c_ty) = match input { + MapInput::Struct(s) => ( + ast::escape_item_path(s), + s.ident(), + s.path(), + helper.c_type(s)?, + ), + MapInput::Union(u) => ( + ast::escape_item_path(u), + u.ident(), + u.path(), + helper.c_type(u)?, + ), + MapInput::Alias(a) => ( + ast::escape_item_path(a), + a.ident(), + a.path(), + helper.c_type(a)?, + ), + + _ => unreachable!("other rust items are not tested for roundtrip tests"), + }; let should_skip_roundtrip_test = helper .generator .skip_roundtrip .as_ref() - .is_some_and(|skip| skip(ident)); + .is_some_and(|skip| skip(path)); if !should_skip_roundtrip_test { let item = TestRoundtrip { - test_name: roundtrip_test_ident(ident), + test_name: roundtrip_test_ident(&escaped_path), id: ident.into(), - fields: fields.iter().filter(|f| f.public).cloned().collect(), + fields: fields + .iter() + .filter(|Field { public, .. }| *public) + .cloned() + .collect(), + rust_ty: path.into(), c_ty: c_ty.into(), is_alias, }; self.roundtrip_tests.push(item.clone()); self.test_idents.push(item.test_name); } + Ok(()) } /// Populates field tests for structs/unions. @@ -350,16 +406,15 @@ impl TestTemplate { }) .map(|(s, f)| { ( - s.ident(), + MapInput::Struct(s), f, helper.c_type(s), helper.c_ident(MapInput::StructField(s, f)), - if !helper.generator.volatile_items.is_empty() - && helper - .generator - .volatile_items - .iter() - .any(|vf| vf(VolatileItemKind::StructField(s.clone(), f.clone()))) + if helper + .generator + .volatile_items + .iter() + .any(|vf| vf(VolatileItemKind::StructField(s.clone(), f.clone()))) { "volatile " } else { @@ -379,7 +434,7 @@ impl TestTemplate { }) .map(|(u, f)| { ( - u.ident(), + MapInput::Union(u), f, helper.c_type(u), helper.c_ident(MapInput::UnionField(u, f)), @@ -387,7 +442,16 @@ impl TestTemplate { ) }); - for (id, field, c_ty, c_field, volatile_keyword) in struct_fields.chain(union_fields) { + for (ty, field, c_ty, c_field, volatile_keyword) in struct_fields.chain(union_fields) { + let (escaped_path, id, path) = match ty { + MapInput::Struct(s) => (ast::escape_item_path(s), s.ident(), s.path()), + MapInput::Union(u) => (ast::escape_item_path(u), u.ident(), u.path()), + + _ => unreachable!( + "we consider only records and untagged unions in field \ + size/offset tests" + ), + }; let field_return_type = cdecl::cdecl( &cdecl::ptr( helper.translator.translate_type(&field.ty)?, @@ -404,10 +468,11 @@ impl TestTemplate { })? .into_boxed_str(); let item = TestFieldPtr { - test_name: field_ptr_test_ident(id, field.ident()), + test_name: field_ptr_test_ident(&escaped_path, field.ident()), id: id.into(), c_ty: c_ty?.into(), field: field.clone(), + rust_ty: path.into(), c_field: c_field.into_boxed_str(), volatile_keyword: volatile_keyword.into(), field_return_type, @@ -426,22 +491,23 @@ impl TestTemplate { &mut self, helper: &TranslateHelper, ) -> Result<(), TranslationError> { - let should_skip_fn_test = |ident| { + let should_skip_fn_test = |path| { helper .generator .skip_fn_ptrcheck .as_ref() - .is_some_and(|skip| skip(ident)) + .is_some_and(|skip| skip(path)) }; for func in helper.filtered_ffi_items.foreign_functions() { - if should_skip_fn_test(func.ident()) { + if should_skip_fn_test(func.path()) { continue; } let item = TestForeignFn { - test_name: foreign_fn_test_ident(func.ident()), + test_name: foreign_fn_test_ident(&ast::escape_item_path(func)), id: func.ident().into(), - c_val: helper.c_ident(func).into_boxed_str(), + rust_ty: func.path().into(), + c_val: helper.c_ident(func).into(), }; self.foreign_fn_tests.push(item.clone()); @@ -451,7 +517,8 @@ impl TestTemplate { Ok(()) } - /// Populates tests for foreign statics, keeping track of the names of each test. + /// Populates tests for foreign statics, keeping track of the names of each + /// test. fn populate_foreign_static_tests( &mut self, helper: &TranslateHelper, @@ -460,10 +527,11 @@ impl TestTemplate { let rust_ty = static_.ty.to_token_stream().to_string().into_boxed_str(); let item = TestForeignStatic { - test_name: static_test_ident(static_.ident()), + test_name: static_test_ident(&ast::escape_item_path(static_)), id: static_.ident().into(), c_val: helper.c_ident(static_).into_boxed_str(), rust_ty, + rust_val: static_.path().into(), }; self.foreign_static_tests.push(item.clone()); @@ -477,12 +545,15 @@ impl TestTemplate { /* Many test structures have the following fields: * * - `test_name`: The function name. - * - `id`: An identifier that can be used to create functions related to this type without conflict, - * usually also part of `test_name`. - * - `rust_val`: Identifier for a Rust value, with path qualifications if needed. - * - `rust_ty`: The Rust type of the relevant item, with path qualifications if needed. - * - `c_val`: Identifier for a C value (e.g. `#define`) - * - `c_ty`: The C type of the constant, qualified with `struct` or `union` if needed. + * - `id` : An identifier that can be used to create functions related + * to this type without conflict, usually also part of `test_name`. + * - `rust_val` : Identifier for a Rust value, with path qualifications if + * needed. + * - `rust_ty` : The Rust type of the relevant item, with path + * qualifications if needed. + * - `c_val` : Identifier for a C value (e.g. `#define`) + * - `c_ty` : The C type of the constant, qualified with `struct` or + * `union` if needed. */ #[derive(Clone, Debug)] @@ -525,6 +596,7 @@ pub(crate) struct TestFieldPtr { pub test_name: BoxStr, pub id: BoxStr, pub field: Field, + pub rust_ty: BoxStr, pub c_field: BoxStr, pub c_ty: BoxStr, pub volatile_keyword: BoxStr, @@ -536,6 +608,7 @@ pub(crate) struct TestFieldSizeOffset { pub test_name: BoxStr, pub id: BoxStr, pub field: Field, + pub rust_ty: BoxStr, pub c_field: BoxStr, pub c_ty: BoxStr, } @@ -545,6 +618,7 @@ pub(crate) struct TestRoundtrip { pub test_name: BoxStr, pub id: BoxStr, pub fields: Vec, + pub rust_ty: BoxStr, pub c_ty: BoxStr, pub is_alias: bool, } @@ -553,6 +627,7 @@ pub(crate) struct TestRoundtrip { pub(crate) struct TestForeignFn { pub test_name: BoxStr, pub c_val: BoxStr, + pub rust_ty: BoxStr, pub id: BoxStr, } @@ -562,6 +637,7 @@ pub(crate) struct TestForeignStatic { pub id: BoxStr, pub c_val: BoxStr, pub rust_ty: BoxStr, + pub rust_val: BoxStr, } fn signededness_test_ident(ident: &str) -> BoxStr { @@ -622,65 +698,96 @@ impl<'a> TranslateHelper<'a> { /// Skips entire items such as structs, constants, and aliases from being tested. /// - /// Does not skip specific tests or specific fields. If `skip_private` is true, - /// it will skip tests for all private items. + /// Does not skip specific tests or specific fields. If `skip_private` is `true`, it will skip + /// tests for all private items. fn filter_ffi_items(&mut self) { - let verbose = self.generator.verbose_skip; - - let skipped = self.filtered_ffi_items.aliases.extract_if(.., |alias| { - self.generator - .skips - .iter() - .any(|f| f(&MapInput::CEnumType(alias.ident()))) - }); - - for item in skipped { - if verbose { - eprintln!("Skipping C enum type {}", item.ident()); + fn skipper(items: &mut FfiItems, generator: &TestGenerator) { + let skipped = items.aliases.extract_if(.., |alias| { + // [NOTE]: the point here is to check whether a given alias (as + // an item that may exist in multiple places across the crate) + // has a skip set up using one of those possible paths (which + // differ for an item that exists in multiple places in the + // crate.) + generator.skips.iter().any(|f| { + alias + .with_variations(|it| f(&MapInput::CEnumType(&it.cached_path))) + .into_iter() + .any(convert::identity) + }) + }); + + for item in skipped { + if generator.verbose_skip { + eprintln!("Skipping C enum type {}", Type::path(&item)); + } } - } - let skipped = self - .filtered_ffi_items - .constants - .extract_if(.., |constant| { - self.generator.skips.iter().any(|f| { - f(&MapInput::CEnumType( - &constant.ty.to_token_stream().to_string(), - )) + let skipped = items.constants.extract_if(.., |constant| { + // [NOTE]: ibid. + generator.skips.iter().any(|f| { + constant + .with_variations(|it| { + f(&MapInput::CEnumType(&it.ty.to_token_stream().to_string())) + }) + .into_iter() + .any(convert::identity) }) }); - for item in skipped { - if verbose { - eprintln!("Skipping C enum constant {}", item.ident()); + for item in skipped { + if generator.verbose_skip { + eprintln!("Skipping C enum constant {}", Const::path(&item)); + } } - } - macro_rules! filter { - ($field:ident, $variant:ident, $label:literal) => {{ - let skipped = self.filtered_ffi_items.$field.extract_if(.., |item| { - (self.generator.skip_private && !item.public) - || self - .generator - .skips - .iter() - .any(|f| f(&MapInput::$variant(item))) - }); - for item in skipped { - if verbose { - eprintln!("Skipping {} \"{}\"", $label, item.ident()) + macro_rules! filter { + ($field:ident: $ty:tt, $variant:ident, $label:literal) => {{ + // [NOTE]: ibid. + let skipped = items.$field.extract_if(.., |item| { + // [NOTE]: when checking whether the item is public or + // not, we need not traverse all possible variants of + // the item as imported throughout the crate's module + // tree. The visibility of the item differs across its + // appearances in the module tree, so skipping an item + // because of its visibility in one module should not + // affect the same (possibly public) item in another + // module. + (generator.skip_private && !item.public) + || generator.skips.iter().any(|f| { + item.with_variations(|it| f(&MapInput::$variant(&it))) + .into_iter() + .any(convert::identity) + }) + }); + for item in skipped { + if generator.verbose_skip { + eprintln!("Skipping {} \"{}\"", $label, $ty::path(&item)) + } } - } - }}; + }}; + } + + // [FIXME]: it may be preferable to rename MapInput::Alias to + // MapInput::Type, and so avoid having to include another "argument" + // to the macro. All other item types use the same identifier for + // both the variant in MapInput as well as their own type. Or maybe + // go the other way around, and change Type to Alias. + filter!(aliases: Type, Alias, "alias"); + filter!(constants: Const, Const, "const"); + filter!(structs: Struct, Struct, "struct"); + filter!(unions: Union, Union, "union"); + filter!(foreign_functions: Fn, Fn, "fn"); + filter!(foreign_statics: Static, Static, "static"); + filter!(modules: Module, Module, "module"); + + // [NOTE]: after dropping the modules that should be skipped from + // `items`, we can safely iterate through whichever ones remain. + for module in &mut items.modules { + skipper(&mut module.items, generator); + } } - filter!(aliases, Alias, "alias"); - filter!(constants, Const, "const"); - filter!(structs, Struct, "struct"); - filter!(unions, Union, "union"); - filter!(foreign_functions, Fn, "fn"); - filter!(foreign_statics, Static, "static"); + skipper(&mut self.filtered_ffi_items, self.generator); } /// Returns the equivalent C/Cpp identifier of the Rust item. @@ -692,31 +799,46 @@ impl<'a> TranslateHelper<'a> { pub(crate) fn c_type(&self, item: impl Into>) -> Result { let item: MapInput = item.into(); - let (ident, ty) = match item { - MapInput::Const(c) => (c.ident(), self.translator.translate_type(&c.ty)?), + // [NOTE]: we fetch the whole item path here instead of the identifier + // through `ident`, because `item_path` gets used only for error + // reporting. + let (item_path, ty) = match item { + MapInput::Const(c) => (c.path(), self.translator.translate_type(&c.ty)?), MapInput::StructField(_, f) => (f.ident(), self.translator.translate_type(&f.ty)?), MapInput::UnionField(_, f) => (f.ident(), self.translator.translate_type(&f.ty)?), - MapInput::Static(s) => (s.ident(), self.translator.translate_type(&s.ty)?), - // For functions, their type would be a bare fn signature, which would need to be saved - // inside of `Fn` when parsed. + MapInput::Static(s) => (s.path(), self.translator.translate_type(&s.ty)?), + // For functions, their type would be a bare fn signature, which + // would need to be saved inside of `Fn` when parsed. MapInput::Fn(_) => unimplemented!(), - // For structs/unions/aliases, their type is the same as their identifier. - MapInput::Alias(a) => (a.ident(), cdecl::named(a.ident(), Constness::Mut)), - MapInput::Struct(s) => (s.ident(), cdecl::named(s.ident(), Constness::Mut)), - MapInput::Union(u) => (u.ident(), cdecl::named(u.ident(), Constness::Mut)), - - MapInput::StructType(_) => panic!("MapInput::StructType is not allowed!"), - MapInput::UnionType(_) => panic!("MapInput::UnionType is not allowed!"), - MapInput::CEnumType(_) => panic!("MapInput::CEnumType is not allowed!"), - MapInput::StructFieldType(_, _) => panic!("MapInput::StructFieldType is not allowed!"), - MapInput::UnionFieldType(_, _) => panic!("MapInput::UnionFieldType is not allowed!"), + // For structs/unions/aliases, their type is the same as their + // identifier. + MapInput::Alias(a) => (a.path(), cdecl::named(&a.ident(), Constness::Mut)), + MapInput::Struct(s) => (s.path(), cdecl::named(&s.ident(), Constness::Mut)), + MapInput::Union(u) => (u.path(), cdecl::named(&u.ident(), Constness::Mut)), + + MapInput::StructType(_) => { + panic!("MapInput::StructType is not allowed!") + } + MapInput::UnionType(_) => { + panic!("MapInput::UnionType is not allowed!") + } + MapInput::CEnumType(_) => { + panic!("MapInput::CEnumType is not allowed!") + } + MapInput::StructFieldType(_, _) => { + panic!("MapInput::StructFieldType is not allowed!") + } + MapInput::UnionFieldType(_, _) => { + panic!("MapInput::UnionFieldType is not allowed!") + } MapInput::Type(_) => panic!("MapInput::Type is not allowed!"), + MapInput::Module(_) => panic!("MapInput::Module is not allowed!"), }; let ty = cdecl::cdecl(&ty, "".to_string()).map_err(|_| { TranslationError::new( TranslationErrorKind::InvalidReturn, - ident, + item_path, Span::call_site(), ) })?; @@ -726,3 +848,18 @@ impl<'a> TranslateHelper<'a> { Ok(self.generator.rty_to_cty(item)) } } + +#[test] +fn tmp() { + use syn::visit::Visit; + + let file = + r#"mod t { mod r { extern "C" { fn ctime() -> c_int; fn something() -> c_int; } } }"#; + let mut items = FfiItems::new(); + let file = syn::parse_file(file).unwrap(); + items.visit_file(&file); + let mut generator = TestGenerator::new(); + generator.skip_fn(|it| it.path() == "t::r::ctime"); + let translator = TranslateHelper::new(&items, &generator); + println!("{:#?}", translator.filtered_ffi_items); +} diff --git a/ctest/src/tests.rs b/ctest/src/tests.rs index baaa7ee1bdd83..5e4a09e5a7265 100644 --- a/ctest/src/tests.rs +++ b/ctest/src/tests.rs @@ -1,5 +1,4 @@ use syn::spanned::Spanned; -use syn::visit::Visit; use crate::ffi_items::FfiItems; use crate::translator::{ @@ -68,12 +67,24 @@ fn test_extraction_ffi_items() { let mut ffi_items = FfiItems::new(); ffi_items.visit_file(&ast); - assert_eq!(collect_idents!(ffi_items.aliases()), ["Foo"]); - assert_eq!(collect_idents!(ffi_items.constants()), ["bar"]); + assert!(collect_idents!(ffi_items.aliases()).is_empty()); + assert_eq!( + collect_idents!(ffi_items.modules.first().unwrap().items.aliases()), + ["Foo"] + ); + assert!(collect_idents!(ffi_items.constants()).is_empty()); + assert_eq!( + collect_idents!(ffi_items.modules.first().unwrap().items.constants()), + ["bar"] + ); assert_eq!(collect_idents!(ffi_items.foreign_functions()), ["malloc"]); assert_eq!(collect_idents!(ffi_items.foreign_statics()), ["baz"]); assert_eq!(collect_idents!(ffi_items.structs()), ["Array"]); - assert_eq!(collect_idents!(ffi_items.unions()), ["Word"]); + assert!(collect_idents!(ffi_items.unions()).is_empty()); + assert_eq!( + collect_idents!(ffi_items.modules.first().unwrap().items.unions()), + ["Word"] + ); } #[test] diff --git a/ctest/src/translator.rs b/ctest/src/translator.rs index 9f14fe65154d3..c6a9b133dc057 100644 --- a/ctest/src/translator.rs +++ b/ctest/src/translator.rs @@ -67,17 +67,20 @@ pub(crate) enum TranslationErrorKind { #[error("unsupported type")] UnsupportedType, - /// A reference to a non-primitive type was encountered, which is not supported. + /// A reference to a non-primitive type was encountered, which is not + /// supported. #[error("references to non-primitive types are not allowed")] NonPrimitiveReference, - /// Lifetimes were found in the type or function signature, which are not supported. + /// Lifetimes were found in the type or function signature, which are not + /// supported. #[error("lifetimes cannot be translated")] HasLifetimes, /// A type that is not ffi compatible was found. #[error( - "this type is not guaranteed to have a C compatible layout. See improper_ctypes_definitions lint" + "this type is not guaranteed to have a C compatible layout. See \ + improper_ctypes_definitions lint" )] NotFfiCompatible, @@ -217,7 +220,8 @@ impl<'a> Translator<'a> { if let syn::PathArguments::AngleBracketed(args) = &last.arguments && let syn::GenericArgument::Type(inner_ty) = args.args.first().unwrap() { - // Option is ONLY ffi-safe if it contains a function pointer, or a reference. + // Option is ONLY ffi-safe if it contains a function pointer, or + // a reference. match inner_ty { syn::Type::Reference(_) | syn::Type::FnPtr(_) => { return self.translate_type(inner_ty); @@ -260,13 +264,16 @@ impl<'a> Translator<'a> { /// Determine whether a C type is a signed type. /// - /// For primitive types it checks against a known list of signed types, but for aliases - /// which are the only thing other than primitives that can be signed, it recursively checks - /// the underlying type of the alias. + /// For primitive types it checks against a known list of signed types, but + /// for aliases which are the only thing other than primitives that can be + /// signed, it recursively checks the underlying type of the alias. pub(crate) fn is_signed(&self, ty: &syn::Type) -> bool { match ty { - syn::Type::Path(path) => { - let ident = path.path.segments.last().unwrap().ident.clone(); + syn::Type::Path(syn::TypePath { + path: syn::Path { segments, .. }, + .. + }) => { + let ident = segments.last().unwrap().ident.clone(); if let Some(aliased) = self.ffi_items.aliases().iter().find(|a| ident == a.ident()) { return self.is_signed(&aliased.ty); @@ -289,7 +296,20 @@ impl<'a> Translator<'a> { MapInput::StructType(name) } else if self.ffi_items.contains_union(name) { MapInput::UnionType(name) - } else if self.generator.c_enums.iter().any(|f| f(name)) { + } + // [NOTE]: for each module (which itself is a separate `FfiItems`,) we + // check first if there is some alias that corresponds with the + // passed `name`, after which we can check if the global set of + // aliases-as-C-`enum`s in `TestGenerator` (which applies to all + // modules and uses the full path to items in its routines) + // contains a remapping of the type alias (with its full path.) + else if let Some(ty) = self + .ffi_items + .aliases() + .iter() + .find(|ty| ty.ident() == name) + && self.generator.c_enums.iter().any(|f| f(ty.path())) + { MapInput::CEnumType(name) } else { MapInput::Type(name) @@ -346,9 +366,10 @@ pub(crate) fn translate_primitive_type(ty: &str) -> String { /// Construct a CTy and modify the constness of the inner type. /// -/// Basically, `syn` always gives us the `constness` of the inner type of a pointer. -/// However `cdecl::ptr` wants the `constness` of the pointer. So we just modify -/// the way it is built so that `cdecl::ptr` takes the `constness` of the inner type. +/// Basically, `syn` always gives us the `constness` of the inner type of a +/// pointer. However `cdecl::ptr` wants the `constness` of the pointer. So we +/// just modify the way it is built so that `cdecl::ptr` takes the `constness` +/// of the inner type. pub(crate) fn ptr_with_inner(inner: cdecl::CTy, constness: Constness) -> cdecl::CTy { let mut ty = Box::new(inner); match ty.deref_mut() { @@ -368,8 +389,8 @@ pub(crate) fn ptr_with_inner(inner: cdecl::CTy, constness: Constness) -> cdecl:: /// Translate a simple Rust expression to C. /// -/// This function will just pass the expression as is in most cases. In more complex cases it can -/// convert `Type as u8 + 5` to `(uint8_t)CType + 5`. +/// This function will just pass the expression as is in most cases. In more +/// complex cases it can convert `Type as u8 + 5` to `(uint8_t)CType + 5`. pub(crate) fn translate_expr(expr: &syn::Expr) -> String { match expr { syn::Expr::Index(i) => { diff --git a/ctest/templates/test.c b/ctest/templates/test.c index c9baba3dc1192..f2a706688cf95 100644 --- a/ctest/templates/test.c +++ b/ctest/templates/test.c @@ -123,7 +123,7 @@ ctest_field_ptr__{{ item.id }}__{{ item.field.ident() }}({{ item.c_ty }} *b) { #endif #ifdef __GNUC__ - // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. + // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-qualifiers" #endif diff --git a/ctest/templates/test.rs b/ctest/templates/test.rs index 3a71c73ba3242..697e245c842d4 100644 --- a/ctest/templates/test.rs +++ b/ctest/templates/test.rs @@ -22,6 +22,7 @@ mod generated_tests { #[allow(unused_imports)] use std::mem::{MaybeUninit, offset_of}; + #[allow(unused)] use super::*; pub static FAILED: AtomicBool = AtomicBool::new(false); @@ -30,6 +31,7 @@ mod generated_tests { /// Check that the value returned from the Rust and C side in a certain test is equivalent. /// /// Internally it will remember which checks failed and how many tests have been run. + #[allow(unused)] fn check_same(rust: T, c: T, attr: &str) { if rust != c { eprintln!("bad {attr}: rust: {rust:?} != c {c:?}"); @@ -39,6 +41,7 @@ mod generated_tests { } } + #[allow(unused)] fn check_same_bytes(rust: &[u8], c: &[u8], attr: &str) { if rust == c { NTESTS.fetch_add(1, Ordering::Relaxed); @@ -184,18 +187,18 @@ mod generated_tests { fn ctest_size_of__{{ item.id }}__{{ item.field.ident() }}() -> u64; } - let uninit_ty = MaybeUninit::<{{ item.id }}>::zeroed(); + let uninit_ty = MaybeUninit::<{{ item.rust_ty }}>::zeroed(); let uninit_ty = uninit_ty.as_ptr(); {# /* SAFETY: we assume the field access doesn't wrap */ #} - let ty_ptr = unsafe { &raw const (*uninit_ty).{{ item.field.rust_ident() }} }; + let ty_ptr = unsafe { &raw const (*uninit_ty).{{ item.field.rust_ident() }} }; {# /* SAFETY: we assume that all zeros is a valid bitpattern for `ty_ptr`, otherwise the * test should be skipped. */ #} let val = unsafe { ty_ptr.read_unaligned() }; {# /* SAFETY: FFI call with no preconditions */ #} let ctest_field_offset = unsafe { ctest_offset_of__{{ item.id }}__{{ item.field.ident() }}() }; - check_same(offset_of!({{ item.id }}, {{ item.field.rust_ident() }}) as u64, ctest_field_offset, + check_same(offset_of!({{ item.rust_ty }}, {{ item.field.rust_ident() }}) as u64, ctest_field_offset, "field offset `{{ item.field.rust_ident() }}` of `{{ item.id }}`"); {# /* SAFETY: FFI call with no preconditions */ #} let ctest_field_size = unsafe { ctest_size_of__{{ item.id }}__{{ item.field.ident() }}() }; @@ -213,7 +216,7 @@ mod generated_tests { fn ctest_field_ptr__{{ item.id }}__{{ item.field.ident() }}(a: *const {{ item.id }}) -> *mut u8; } - let uninit_ty = MaybeUninit::<{{ item.id }}>::zeroed(); + let uninit_ty = MaybeUninit::<{{ item.rust_ty }}>::zeroed(); let ty_ptr = uninit_ty.as_ptr(); // SAFETY: We don't read `field_ptr`, only compare the pointer itself. // The assumption is made that this does not wrap the address space. @@ -242,14 +245,14 @@ mod generated_tests { fn roundtrip_padding__{{ item.id }}() -> Vec { if {{ item.fields.len() }} == 0 { {# /* FIXME(ctest): What if it's an alias to a struct/union? */ #} - return vec![!{{ item.is_alias }}; size_of::<{{ item.id }}>()] + return vec![!{{ item.is_alias }}; size_of::<{{ item.rust_ty }}>()] } {# /* If there are no fields, v and bar become unused. */ #} #[allow(unused_mut)] let mut v = Vec::<(usize, usize)>::new(); #[allow(unused_variables)] - let bar = MaybeUninit::<{{ item.id }}>::zeroed(); + let bar = MaybeUninit::<{{ item.rust_ty }}>::zeroed(); #[allow(unused_variables)] let bar = bar.as_ptr(); {%- for field in item.fields +%} @@ -258,7 +261,7 @@ mod generated_tests { let val = unsafe { ty_ptr.read_unaligned() }; let size = size_of_val(&val); - let off = offset_of!({{ item.id }}, {{ field.rust_ident() }}); + let off = offset_of!({{ item.rust_ty }}, {{ field.rust_ident() }}); v.push((off, size)); {%- endfor +%} {# /* This vector contains `true` if the byte is padding and `false` if the byte is not @@ -266,7 +269,7 @@ mod generated_tests { * - padding if we have fields, this means that only the fields will be checked * - no-padding if we have a type alias: if this causes problems the type alias should * be skipped */ #} - let mut is_padding_byte = vec![true; size_of::<{{ item.id }}>()]; + let mut is_padding_byte = vec![true; size_of::<{{ item.rust_ty }}>()]; for (off, size) in &v { for i in 0..*size { is_padding_byte[off + i] = false; @@ -280,7 +283,7 @@ mod generated_tests { * It checks if the size is the same as well as if the padding bytes are all in the * correct place. For this test to be sound, `T` must be valid for any bitpattern. */ #} pub fn {{ item.test_name }}() { - type U = {{ item.id }}; + type U = {{ item.rust_ty }}; {{ ctest_extern }} "C" { fn ctest_size_of__{{ item.id }}() -> u64; fn ctest_roundtrip__{{ item.id }}( @@ -320,7 +323,7 @@ mod generated_tests { return; } - let mut c_value_bytes = vec![0; size_of::<{{ item.id }}>()]; + let mut c_value_bytes = vec![0; size_of::<{{ item.rust_ty }}>()]; let r: U = unsafe { ctest_roundtrip__{{ item.id }}(input, is_padding_byte.as_ptr(), c_value_bytes.as_mut_ptr()) }; @@ -359,7 +362,7 @@ mod generated_tests { fn ctest_foreign_fn__{{ item.id }}() -> unsafe extern "C" fn(); } let actual = unsafe { ctest_foreign_fn__{{ item.id }}() } as u64; - let expected = {{ item.id }} as *const () as u64; + let expected = {{ item.rust_ty }} as *const () as u64; check_same(actual, expected, "`{{ item.id }}` function pointer"); } {%- endfor +%} @@ -371,7 +374,7 @@ mod generated_tests { {{ ctest_extern }} "C" { fn ctest_static__{{ static_.id }}() -> *const {{ static_.rust_ty }}; } - let actual = (&raw const {{ static_.id }}).addr(); + let actual = (&raw const {{ static_.rust_val }}).addr(); let expected = unsafe { ctest_static__{{ static_.id }}().addr() }; diff --git a/ctest/tests/basic.rs b/ctest/tests/basic.rs index 593af94ad13a5..1fb3c75d797ff 100644 --- a/ctest/tests/basic.rs +++ b/ctest/tests/basic.rs @@ -111,13 +111,13 @@ fn test_skip_simple() { let library_path = "simple.out.with-skips.a"; let (mut gen_, out_dir) = default_generator(1, Some("simple.h")).unwrap(); - gen_.skip_const(|c| c.ident() == "B" || c.ident() == "A") + gen_.skip_const(|c| c.path() == "B" || c.path() == "A") .skip_c_enum(|e| e == "Color") - .skip_alias(|a| a.ident() == "Byte" || a.ident() == "gregset_t") - .skip_struct(|s| s.ident() == "Person") - .skip_union(|u| u.ident() == "Word") - .skip_fn(|f| f.ident() == "calloc") - .skip_static(|s| s.ident() == "byte"); + .skip_alias(|a| a.path() == "Byte" || a.path() == "gregset_t") + .skip_struct(|s| s.path() == "Person") + .skip_union(|u| u.path() == "Word") + .skip_fn(|f| f.path() == "calloc") + .skip_static(|s| s.path() == "byte"); check_entrypoint(&mut gen_, out_dir, crate_path, library_path, include_path); } @@ -130,7 +130,7 @@ fn test_map_simple() { let library_path = "simple.out.with-renames.a"; let (mut gen_, out_dir) = default_generator(1, Some("simple.h")).unwrap(); - gen_.rename_constant(|c| (c.ident() == "B").then(|| "C_B".to_string())) + gen_.rename_constant(|c| (c.path() == "B").then(|| "C_B".to_string())) .alias_is_c_enum(|e| e == "Color") .skip_signededness(|ty| ty == "Color"); diff --git a/ctest/tests/input/hierarchy.out.c b/ctest/tests/input/hierarchy.out.c index d34e157ee8f72..48fccdb7eaf41 100644 --- a/ctest/tests/input/hierarchy.out.c +++ b/ctest/tests/input/hierarchy.out.c @@ -29,18 +29,9 @@ typedef void (*ctest_void_func)(void); * This will later be called on the Rust side via FFI. */ -static bool ctest_const_ON_val_static = ON; - -CTEST_EXTERN bool *ctest_const__ON(void) { - return &ctest_const_ON_val_static; -} - /* Query the size and alignment of all types */ -CTEST_EXTERN uint64_t ctest_size_of__in6_addr(void) { return sizeof(in6_addr); } -CTEST_EXTERN uint64_t ctest_align_of__in6_addr(void) { return CTEST_ALIGNOF(in6_addr); } - /* Query the signedness of a type. * @@ -48,11 +39,6 @@ CTEST_EXTERN uint64_t ctest_align_of__in6_addr(void) { return CTEST_ALIGNOF(in6_ * Casting -1 to the aliased type if signed evaluates to `-1 < 0`, if unsigned to `MAX_VALUE < 0` */ -CTEST_EXTERN uint32_t ctest_signededness_of__in6_addr(void) { - in6_addr all_ones = (in6_addr) -1; - return all_ones < 0; -} - /* Query the offsets of fields and their sizes. */ @@ -66,7 +52,7 @@ CTEST_EXTERN uint32_t ctest_signededness_of__in6_addr(void) { #endif #ifdef __GNUC__ - // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. + // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-qualifiers" #endif @@ -79,27 +65,6 @@ CTEST_EXTERN uint32_t ctest_signededness_of__in6_addr(void) { * It checks if the size is the same as well as if the padding bytes are all in the correct place. */ -CTEST_EXTERN in6_addr ctest_roundtrip__in6_addr( - in6_addr value, - const uint8_t is_padding_byte[sizeof(in6_addr)], - uint8_t value_bytes[sizeof(in6_addr)] -) { - int size = (int)sizeof(in6_addr); - - volatile uint8_t* p = (volatile uint8_t*)&value; - int i = 0; - for (i = 0; i < size; ++i) { - - if (is_padding_byte[i]) { continue; } - value_bytes[i] = p[i]; - - uint8_t d = (uint8_t)(255) - (uint8_t)(i % 256); - d = d == 0 ? 42: d; - p[i] = d; - } - return value; -} - #ifdef __GNUC__ // Pop allow for `-Wignored-qualifiers` #pragma GCC diagnostic pop @@ -118,10 +83,6 @@ CTEST_EXTERN in6_addr ctest_roundtrip__in6_addr( /* Query a function's pointer */ -CTEST_EXTERN ctest_void_func ctest_foreign_fn__malloc(void) { - return (ctest_void_func)malloc; -} - #ifdef _MSC_VER // Pop allow for 4191 #pragma warning(default:4191) @@ -129,8 +90,3 @@ CTEST_EXTERN ctest_void_func ctest_foreign_fn__malloc(void) { /* Query pointers to statics */ - -CTEST_EXTERN void *ctest_static__in6addr_any(void) { - - return (void *)&in6addr_any; -} diff --git a/ctest/tests/input/hierarchy.out.rs b/ctest/tests/input/hierarchy.out.rs index 9a9ab6151488a..2ca2e1f2d3d33 100644 --- a/ctest/tests/input/hierarchy.out.rs +++ b/ctest/tests/input/hierarchy.out.rs @@ -18,6 +18,7 @@ mod generated_tests { #[allow(unused_imports)] use std::mem::{MaybeUninit, offset_of}; + #[allow(unused)] use super::*; pub static FAILED: AtomicBool = AtomicBool::new(false); @@ -26,6 +27,7 @@ mod generated_tests { /// Check that the value returned from the Rust and C side in a certain test is equivalent. /// /// Internally it will remember which checks failed and how many tests have been run. + #[allow(unused)] fn check_same(rust: T, c: T, attr: &str) { if rust != c { eprintln!("bad {attr}: rust: {rust:?} != c {c:?}"); @@ -35,6 +37,7 @@ mod generated_tests { } } + #[allow(unused)] fn check_same_bytes(rust: &[u8], c: &[u8], attr: &str) { if rust == c { NTESTS.fetch_add(1, Ordering::Relaxed); @@ -79,46 +82,9 @@ mod generated_tests { * This performs a byte by byte comparison of the constant value. */ - pub fn ctest_const_ON() { - type T = bool; - extern "C" { - fn ctest_const__ON() -> *const T; - } - - - - let r_val: T = ON; - let r_bytes = unsafe { - slice::from_raw_parts(ptr::from_ref(&r_val).cast::(), size_of::()) - }; - - let c_bytes = unsafe { - let c_ptr: *const T = ctest_const__ON(); - slice::from_raw_parts(c_ptr.cast::(), size_of::()) - }; - - check_same_bytes(r_bytes, c_bytes, "`ON` value"); - } - /* Compare the size and alignment of the type in Rust and C, making sure they are the same. */ - pub fn ctest_size_align_in6_addr() { - extern "C" { - fn ctest_size_of__in6_addr() -> u64; - fn ctest_align_of__in6_addr() -> u64; - } - - let rust_size = size_of::() as u64; - let c_size = unsafe { ctest_size_of__in6_addr() }; - - let rust_align = align_of::() as u64; - let c_align = unsafe { ctest_align_of__in6_addr() }; - - check_same(rust_size, c_size, "`in6_addr` size"); - check_same(rust_align, c_align, "`in6_addr` align"); - } - /* Make sure that the signededness of a type alias in Rust and C is the same. * @@ -127,17 +93,6 @@ mod generated_tests { * smaller than 0. */ - pub fn ctest_signededness_in6_addr() { - extern "C" { - fn ctest_signededness_of__in6_addr() -> u32; - } - let all_ones = !(0 as in6_addr); - let all_zeros = 0 as in6_addr; - let c_is_signed = unsafe { ctest_signededness_of__in6_addr() }; - - check_same((all_ones < all_zeros) as u32, c_is_signed, "`in6_addr` signed"); - } - /* Make sure that the offset and size of a field in a struct/union is the same. */ @@ -155,121 +110,9 @@ mod generated_tests { * go through each field and figure out the padding. */ - fn roundtrip_padding__in6_addr() -> Vec { - if 0 == 0 { - - return vec![!true; size_of::()] - } - - - #[allow(unused_mut)] - let mut v = Vec::<(usize, usize)>::new(); - #[allow(unused_variables)] - let bar = MaybeUninit::::zeroed(); - #[allow(unused_variables)] - let bar = bar.as_ptr(); - - let mut is_padding_byte = vec![true; size_of::()]; - for (off, size) in &v { - for i in 0..*size { - is_padding_byte[off + i] = false; - } - } - is_padding_byte - } - - - pub fn ctest_roundtrip_in6_addr() { - type U = in6_addr; - extern "C" { - fn ctest_size_of__in6_addr() -> u64; - fn ctest_roundtrip__in6_addr( - input: MaybeUninit, is_padding_byte: *const bool, value_bytes: *mut u8 - ) -> U; - } - - const SIZE: usize = size_of::(); - - let is_padding_byte = roundtrip_padding__in6_addr(); - let mut expected = vec![0u8; SIZE]; - let mut input = MaybeUninit::::zeroed(); - - let input_ptr = input.as_mut_ptr().cast::(); - - - for i in 0..SIZE { - let c: u8 = (i % 256) as u8; - let c = if c == 0 { 42 } else { c }; - let d: u8 = 255_u8 - (i % 256) as u8; - let d = if d == 0 { 42 } else { d }; - unsafe { - input_ptr.add(i).write_volatile(c); - expected[i] = d; - } - } - - let c_size = unsafe { ctest_size_of__in6_addr() } as usize; - if SIZE != c_size { - FAILED.store(true, Ordering::Relaxed); - eprintln!( - "size of `in6_addr` is {c_size} in C and {SIZE} in Rust\n", - ); - return; - } - - let mut c_value_bytes = vec![0; size_of::()]; - let r: U = unsafe { - ctest_roundtrip__in6_addr(input, is_padding_byte.as_ptr(), c_value_bytes.as_mut_ptr()) - }; - - - for (i, is_padding_byte) in is_padding_byte.iter().enumerate() { - if *is_padding_byte { continue; } - let rust = unsafe { *input_ptr.add(i) }; - let c = c_value_bytes[i]; - if rust != c { - eprintln!("rust[{}] = {} != {} (C): Rust `in6_addr` -> C", i, rust, c); - FAILED.store(true, Ordering::Relaxed); - } - } - - - for (i, is_padding_byte) in is_padding_byte.iter().enumerate() { - if *is_padding_byte { continue; } - let rust = expected[i] as usize; - let c = unsafe { (&raw const r).cast::().add(i).read_volatile() as usize }; - if rust != c { - eprintln!( - "rust [{i}] = {rust} != {c} (C): C `in6_addr` -> Rust", - ); - FAILED.store(true, Ordering::Relaxed); - } - } - } - /* Check if the Rust and C side function pointers point to the same underlying function. */ - pub fn ctest_foreign_fn_malloc() { - extern "C" { - fn ctest_foreign_fn__malloc() -> unsafe extern "C" fn(); - } - let actual = unsafe { ctest_foreign_fn__malloc() } as u64; - let expected = malloc as *const () as u64; - check_same(actual, expected, "`malloc` function pointer"); - } - /* Tests if the pointer to the static variable matches in both Rust and C. */ - - pub fn ctest_static_in6addr_any() { - extern "C" { - fn ctest_static__in6addr_any() -> *const in6_addr; - } - let actual = (&raw const in6addr_any).addr(); - let expected = unsafe { - ctest_static__in6addr_any().addr() - }; - check_same(actual, expected, "`in6addr_any` static"); - } } use generated_tests::*; @@ -291,10 +134,4 @@ fn main() { // FIXME(ctest): Maybe consider running the tests in parallel, since everything is independent // and we already use atomics. fn run_all() { - ctest_const_ON(); - ctest_size_align_in6_addr(); - ctest_signededness_in6_addr(); - ctest_roundtrip_in6_addr(); - ctest_foreign_fn_malloc(); - ctest_static_in6addr_any(); } diff --git a/ctest/tests/input/macro.out.c b/ctest/tests/input/macro.out.c index 3fe5cf1d91b14..d170831c1ebe3 100644 --- a/ctest/tests/input/macro.out.c +++ b/ctest/tests/input/macro.out.c @@ -141,7 +141,7 @@ ctest_field_ptr__VecU16__y(struct VecU16 *b) { #endif #ifdef __GNUC__ - // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. + // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-qualifiers" #endif diff --git a/ctest/tests/input/macro.out.edition-2024.c b/ctest/tests/input/macro.out.edition-2024.c index 3fe5cf1d91b14..d170831c1ebe3 100644 --- a/ctest/tests/input/macro.out.edition-2024.c +++ b/ctest/tests/input/macro.out.edition-2024.c @@ -141,7 +141,7 @@ ctest_field_ptr__VecU16__y(struct VecU16 *b) { #endif #ifdef __GNUC__ - // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. + // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-qualifiers" #endif diff --git a/ctest/tests/input/macro.out.edition-2024.rs b/ctest/tests/input/macro.out.edition-2024.rs index 6970695f02ff0..fe0ee081a42dd 100644 --- a/ctest/tests/input/macro.out.edition-2024.rs +++ b/ctest/tests/input/macro.out.edition-2024.rs @@ -18,6 +18,7 @@ mod generated_tests { #[allow(unused_imports)] use std::mem::{MaybeUninit, offset_of}; + #[allow(unused)] use super::*; pub static FAILED: AtomicBool = AtomicBool::new(false); @@ -26,6 +27,7 @@ mod generated_tests { /// Check that the value returned from the Rust and C side in a certain test is equivalent. /// /// Internally it will remember which checks failed and how many tests have been run. + #[allow(unused)] fn check_same(rust: T, c: T, attr: &str) { if rust != c { eprintln!("bad {attr}: rust: {rust:?} != c {c:?}"); @@ -35,6 +37,7 @@ mod generated_tests { } } + #[allow(unused)] fn check_same_bytes(rust: &[u8], c: &[u8], attr: &str) { if rust == c { NTESTS.fetch_add(1, Ordering::Relaxed); @@ -135,7 +138,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).x }; + let ty_ptr = unsafe { &raw const (*uninit_ty).x }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -159,7 +162,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).y }; + let ty_ptr = unsafe { &raw const (*uninit_ty).y }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -183,7 +186,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).x }; + let ty_ptr = unsafe { &raw const (*uninit_ty).x }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -207,7 +210,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).y }; + let ty_ptr = unsafe { &raw const (*uninit_ty).y }; let val = unsafe { ty_ptr.read_unaligned() }; diff --git a/ctest/tests/input/macro.out.rs b/ctest/tests/input/macro.out.rs index 25555e0c25650..d129f26f45a96 100644 --- a/ctest/tests/input/macro.out.rs +++ b/ctest/tests/input/macro.out.rs @@ -18,6 +18,7 @@ mod generated_tests { #[allow(unused_imports)] use std::mem::{MaybeUninit, offset_of}; + #[allow(unused)] use super::*; pub static FAILED: AtomicBool = AtomicBool::new(false); @@ -26,6 +27,7 @@ mod generated_tests { /// Check that the value returned from the Rust and C side in a certain test is equivalent. /// /// Internally it will remember which checks failed and how many tests have been run. + #[allow(unused)] fn check_same(rust: T, c: T, attr: &str) { if rust != c { eprintln!("bad {attr}: rust: {rust:?} != c {c:?}"); @@ -35,6 +37,7 @@ mod generated_tests { } } + #[allow(unused)] fn check_same_bytes(rust: &[u8], c: &[u8], attr: &str) { if rust == c { NTESTS.fetch_add(1, Ordering::Relaxed); @@ -135,7 +138,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).x }; + let ty_ptr = unsafe { &raw const (*uninit_ty).x }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -159,7 +162,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).y }; + let ty_ptr = unsafe { &raw const (*uninit_ty).y }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -183,7 +186,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).x }; + let ty_ptr = unsafe { &raw const (*uninit_ty).x }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -207,7 +210,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).y }; + let ty_ptr = unsafe { &raw const (*uninit_ty).y }; let val = unsafe { ty_ptr.read_unaligned() }; diff --git a/ctest/tests/input/simple.out.with-renames.c b/ctest/tests/input/simple.out.with-renames.c index 23f0b45bca6b9..86ac54601fd0c 100644 --- a/ctest/tests/input/simple.out.with-renames.c +++ b/ctest/tests/input/simple.out.with-renames.c @@ -230,7 +230,7 @@ ctest_field_ptr__Word__byte(union Word *b) { #endif #ifdef __GNUC__ - // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. + // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-qualifiers" #endif diff --git a/ctest/tests/input/simple.out.with-renames.rs b/ctest/tests/input/simple.out.with-renames.rs index 1736e9941e13b..f332ddaac3adb 100644 --- a/ctest/tests/input/simple.out.with-renames.rs +++ b/ctest/tests/input/simple.out.with-renames.rs @@ -18,6 +18,7 @@ mod generated_tests { #[allow(unused_imports)] use std::mem::{MaybeUninit, offset_of}; + #[allow(unused)] use super::*; pub static FAILED: AtomicBool = AtomicBool::new(false); @@ -26,6 +27,7 @@ mod generated_tests { /// Check that the value returned from the Rust and C side in a certain test is equivalent. /// /// Internally it will remember which checks failed and how many tests have been run. + #[allow(unused)] fn check_same(rust: T, c: T, attr: &str) { if rust != c { eprintln!("bad {attr}: rust: {rust:?} != c {c:?}"); @@ -35,6 +37,7 @@ mod generated_tests { } } + #[allow(unused)] fn check_same_bytes(rust: &[u8], c: &[u8], attr: &str) { if rust == c { NTESTS.fetch_add(1, Ordering::Relaxed); @@ -326,7 +329,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).name }; + let ty_ptr = unsafe { &raw const (*uninit_ty).name }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -350,7 +353,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).age }; + let ty_ptr = unsafe { &raw const (*uninit_ty).age }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -374,7 +377,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).job }; + let ty_ptr = unsafe { &raw const (*uninit_ty).job }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -398,7 +401,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).favorite_color }; + let ty_ptr = unsafe { &raw const (*uninit_ty).favorite_color }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -422,7 +425,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).word }; + let ty_ptr = unsafe { &raw const (*uninit_ty).word }; let val = unsafe { ty_ptr.read_unaligned() }; @@ -446,7 +449,7 @@ mod generated_tests { let uninit_ty = uninit_ty.as_ptr(); - let ty_ptr = unsafe { &raw const (*uninit_ty).byte }; + let ty_ptr = unsafe { &raw const (*uninit_ty).byte }; let val = unsafe { ty_ptr.read_unaligned() }; diff --git a/ctest/tests/input/simple.out.with-skips.c b/ctest/tests/input/simple.out.with-skips.c index bf2ce3cba863c..e9c2167269185 100644 --- a/ctest/tests/input/simple.out.with-skips.c +++ b/ctest/tests/input/simple.out.with-skips.c @@ -60,7 +60,7 @@ CTEST_EXTERN uint32_t ctest_signededness_of__volatile_char(void) { #endif #ifdef __GNUC__ - // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. + // GCC emits a warning with `-Wextra` if we return a typedef to a type marked `volatile`. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wignored-qualifiers" #endif diff --git a/ctest/tests/input/simple.out.with-skips.rs b/ctest/tests/input/simple.out.with-skips.rs index 125e8a564fcf0..71b7508911d12 100644 --- a/ctest/tests/input/simple.out.with-skips.rs +++ b/ctest/tests/input/simple.out.with-skips.rs @@ -18,6 +18,7 @@ mod generated_tests { #[allow(unused_imports)] use std::mem::{MaybeUninit, offset_of}; + #[allow(unused)] use super::*; pub static FAILED: AtomicBool = AtomicBool::new(false); @@ -26,6 +27,7 @@ mod generated_tests { /// Check that the value returned from the Rust and C side in a certain test is equivalent. /// /// Internally it will remember which checks failed and how many tests have been run. + #[allow(unused)] fn check_same(rust: T, c: T, attr: &str) { if rust != c { eprintln!("bad {attr}: rust: {rust:?} != c {c:?}"); @@ -35,6 +37,7 @@ mod generated_tests { } } + #[allow(unused)] fn check_same_bytes(rust: &[u8], c: &[u8], attr: &str) { if rust == c { NTESTS.fetch_add(1, Ordering::Relaxed); diff --git a/libc-test/build/main.rs b/libc-test/build/main.rs index c6b1a271e5221..22939d7558cf0 100755 --- a/libc-test/build/main.rs +++ b/libc-test/build/main.rs @@ -87,13 +87,13 @@ fn ctest_cfg() -> ctest::TestGenerator { cfg.skip_private(true); // Skip anonymous unions/structs. - cfg.skip_union(|u| u.ident().starts_with("__c_anonymous_")); - cfg.skip_struct(|s| s.ident().starts_with("__c_anonymous_")); - cfg.skip_alias(|ty| ty.ident().starts_with("__c_anonymous_")); + cfg.skip_union(|u| u.path().starts_with("__c_anonymous_")); + cfg.skip_struct(|s| s.path().starts_with("__c_anonymous_")); + cfg.skip_alias(|ty| ty.path().starts_with("__c_anonymous_")); // __uint128 is not declared in C, but is an alias we export. // FIXME(1.0): These aliases will eventually be removed. - cfg.skip_alias(|ty| ty.ident() == "__uint128"); + cfg.skip_alias(|ty| ty.path() == "__uint128"); if env::var("LIBC_CI_ZBUILD_STD").is_ok() { *cfg.expansion_cargo_args_mut() = vec!["-Zbuild-std=core,std".into()]; @@ -266,7 +266,7 @@ fn test_apple(t: &Target) { ); cfg.skip_struct(move |s| { - match s.ident() { + match s.path() { // Extern types "DIR" | "FILE" | "fpos_t" | "timezone" | "_opaque_pthread_t" => true, @@ -286,7 +286,7 @@ fn test_apple(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { // FIXME(deprecated): Removed since 12.0.1 / xnu-8019.41.5. See `ttycom.h` at // https://github.com/apple-oss-distributions/xnu/commit/e6231be02a03711ca404e5121a151b24afbff733 "TIOCREMOTE" => true, @@ -306,7 +306,7 @@ fn test_apple(t: &Target) { }); cfg.skip_alias(move |ty| { - match ty.ident() { + match ty.path() { // FIXME(macos): The size is changed in macOS/iOS/... 27. "vm_statistics64_data_t" => apple.unwrap() < (27, 0), _ => false, @@ -315,7 +315,7 @@ fn test_apple(t: &Target) { cfg.skip_fn(move |func| { // skip those that are manually verified - match func.ident() { + match func.path() { // close calls the close_nocancel system call on x86 "close" if x86_64 => true, // FIXME(1.0): std removed libresolv support: https://github.com/rust-lang/rust/pull/102766 @@ -329,7 +329,7 @@ fn test_apple(t: &Target) { }); cfg.skip_struct_field(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // Anonymous ADT fields ("ifreq", "ifr_ifru") => true, ("in6_ifreq", "ifr_ifru") => true, @@ -341,10 +341,10 @@ fn test_apple(t: &Target) { cfg.skip_struct_field_type(move |struct_, field| { // The type of `bfl_u` is an anonymous union - (struct_.ident(), field.ident()) == ("bpf_dltlist", "bfl_u") + (struct_.path(), field.ident()) == ("bpf_dltlist", "bfl_u") }); - cfg.volatile_struct_field(|s, f| s.ident() == "aiocb" && f.ident() == "aio_buf"); + cfg.volatile_struct_field(|s, f| s.path() == "aiocb" && f.ident() == "aio_buf"); cfg.rename_struct_ty(move |ty| { // Just pass all these through, no need for a "struct" prefix @@ -361,13 +361,13 @@ fn test_apple(t: &Target) { cfg.rename_struct_field(|s, f| { match f.ident() { - n if n.ends_with("_nsec") && s.ident().starts_with("stat") => { + n if n.ends_with("_nsec") && s.path().starts_with("stat") => { Some(n.replace("e_nsec", "espec.tv_nsec")) } // FIXME(macos): sigaction actually contains a union with two variants: // a sa_sigaction with type: (*)(int, struct __siginfo *, void *) // a sa_handler with type sig_t - "sa_sigaction" if s.ident() == "sigaction" => Some("sa_handler".to_string()), + "sa_sigaction" if s.path() == "sigaction" => Some("sa_handler".to_string()), _ => None, } }); @@ -494,7 +494,7 @@ fn test_openbsd(t: &Target) { }); cfg.rename_struct_field(|struct_, field_| { - let struct_ = struct_.ident(); + let struct_ = struct_.path(); let replacement = match field_.ident() { "st_birthtime" if struct_.starts_with("stat") => "__st_birthtime".to_string(), "st_birthtime_nsec" if struct_.starts_with("stat") => "__st_birthtimensec".to_string(), @@ -517,7 +517,7 @@ fn test_openbsd(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { // Removed in OpenBSD 7.7 (unused since 1991) "ATF_COM" | "ATF_PERM" | "ATF_PUBL" | "ATF_USETRAILERS" => true, @@ -537,7 +537,7 @@ fn test_openbsd(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // Extern types "DIR" | "FILE" | "fpos_t" | "sem" | "timezone" => true, @@ -546,7 +546,7 @@ fn test_openbsd(t: &Target) { }); cfg.skip_struct_field(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // conflicting with `p_type` macro from . ("Elf32_Phdr", "p_type") => true, ("Elf64_Phdr", "p_type") => true, @@ -641,7 +641,7 @@ fn test_cygwin(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { // FIXME(cygwin): these constants do not exist on Cygwin "ARPOP_REQUEST" | "ARPOP_REPLY" | "ATF_COM" | "ATF_PERM" | "ATF_PUBL" | "ATF_USETRAILERS" => true, @@ -668,7 +668,7 @@ fn test_cygwin(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // Extern types "DIR" | "FILE" | "fpos_t" => true, @@ -680,21 +680,21 @@ fn test_cygwin(t: &Target) { match field.ident() { // Our stat *_nsec fields normally don't actually exist but are part // of a timeval struct - s if s.ends_with("_nsec") && struct_.ident().starts_with("stat") => { + s if s.ends_with("_nsec") && struct_.path().starts_with("stat") => { Some(s.replace("e_nsec", ".tv_nsec")) } // FIXME(cygwin): sigaction actually contains a union with two variants: // a sa_sigaction with type: (*)(int, struct __siginfo *, void *) // a sa_handler with type sig_t - "sa_sigaction" if struct_.ident() == "sigaction" => Some("sa_handler".to_string()), + "sa_sigaction" if struct_.path() == "sigaction" => Some("sa_handler".to_string()), _ => None, } }); cfg.skip_struct_field(|struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // this is actually a union on linux, so we can't represent it well and // just insert some padding. ("ifreq", "ifr_ifru") => true, @@ -706,7 +706,7 @@ fn test_cygwin(t: &Target) { cfg.skip_fn(move |func| { // skip those that are manually verified - match func.ident() { + match func.path() { // There are two versions of the sterror_r function, see // // https://linux.die.net/man/3/strerror_r @@ -813,10 +813,10 @@ fn test_windows(t: &Target) { cfg.rename_fn(move |func| { func.link_name() .map(|l| l.to_string()) - .or(func.ident().to_string().into()) + .or(func.path().to_string().into()) }); - cfg.skip_alias(move |alias| match alias.ident() { + cfg.skip_alias(move |alias| match alias.path() { "SSIZE_T" if !gnu => true, "ssize_t" if !gnu => true, // FIXME(windows): The size and alignment of this type are incorrect @@ -825,7 +825,7 @@ fn test_windows(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // FIXME(windows): The size and alignment of this struct are incorrect "timespec" if gnu && x86_32 => true, // Extern types @@ -835,7 +835,7 @@ fn test_windows(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { // FIXME(windows): API error: // SIG_ERR type is "void (*)(int)", not "int" "SIG_ERR" | @@ -847,7 +847,7 @@ fn test_windows(t: &Target) { } }); - cfg.skip_struct_field(move |s, field| s.ident() == "CONTEXT" && field.ident() == "Fp"); + cfg.skip_struct_field(move |s, field| s.path() == "CONTEXT" && field.ident() == "Fp"); // FIXME(windows): All functions point to the wrong addresses? cfg.skip_fn_ptrcheck(|_| true); @@ -1029,7 +1029,7 @@ fn test_solarish(t: &Target) { headers!(cfg, "sys/lgrp_user_impl.h",); } - cfg.skip_alias(move |ty| match ty.ident() { + cfg.skip_alias(move |ty| match ty.path() { "sighandler_t" => true, _ => false, }); @@ -1047,7 +1047,7 @@ fn test_solarish(t: &Target) { }); cfg.rename_struct_field(move |struct_, field| { - match struct_.ident() { + match struct_.path() { // rust struct was committed with typo for Solaris "door_arg_t" if field.ident() == "dec_num" => Some("desc_num".to_string()), "stat" if field.ident().ends_with("_nsec") => { @@ -1058,7 +1058,7 @@ fn test_solarish(t: &Target) { } }); - cfg.skip_const(move |constant| match constant.ident() { + cfg.skip_const(move |constant| match constant.path() { "DT_FIFO" | "DT_CHR" | "DT_DIR" | "DT_BLK" | "DT_REG" | "DT_LNK" | "DT_SOCK" | "USRQUOTA" | "GRPQUOTA" | "PRIO_MIN" | "PRIO_MAX" => true, @@ -1095,13 +1095,13 @@ fn test_solarish(t: &Target) { cfg.skip_union(|union_| { // the union handling is a mess - if union_.ident().contains("door_desc_t_") { + if union_.path().contains("door_desc_t_") { return true; } false }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // the union handling is a mess x if x.contains("door_desc_t_") => true, @@ -1117,11 +1117,11 @@ fn test_solarish(t: &Target) { cfg.skip_struct_field_type(move |struct_, field| { // aio_buf is "volatile void*" - struct_.ident() == "aiocb" && field.ident() == "aio_buf" + struct_.path() == "aiocb" && field.ident() == "aio_buf" }); cfg.skip_struct_field(move |s, field| { - match (s.ident(), field.ident()) { + match (s.path(), field.ident()) { // C99 sizing on this is tough ("dirent", "d_name") => true, // the union/macro makes this rough @@ -1150,7 +1150,7 @@ fn test_solarish(t: &Target) { cfg.skip_fn(move |func| { // skip those that are manually verified - match func.ident() { + match func.path() { // const-ness only added recently "dladdr" => true, @@ -1342,7 +1342,7 @@ fn test_netbsd(t: &Target) { match field.ident() { // Our stat *_nsec fields normally don't actually exist but are part // of a timeval struct - s if s.ends_with("_nsec") && struct_.ident().starts_with("stat") => { + s if s.ends_with("_nsec") && struct_.path().starts_with("stat") => { Some(s.replace("e_nsec", ".tv_nsec")) } _ => None, @@ -1352,7 +1352,7 @@ fn test_netbsd(t: &Target) { cfg.alias_is_c_enum(|ty| ty == "fae_action"); cfg.skip_alias(move |ty| { - match ty.ident() { + match ty.path() { // FIXME(netbsd): sighandler_t is crazy across platforms "sighandler_t" => true, // Incomplete type in C @@ -1363,7 +1363,7 @@ fn test_netbsd(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // These are tested as part of the linux_fcntl tests since there are // header conflicts when including them with all the other structs. "termios2" => true, @@ -1392,7 +1392,7 @@ fn test_netbsd(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { "SIG_DFL" | "SIG_ERR" | "SIG_IGN" => true, // sighandler_t weirdness // deprecated, obsolete upstream @@ -1414,7 +1414,7 @@ fn test_netbsd(t: &Target) { cfg.skip_fn(move |func| { #[expect(clippy::wildcard_in_or_patterns)] - match func.ident() { + match func.path() { // FIXME(netbsd): Look into setting `_POSIX_C_SOURCE` to enable this "qsort_r" => true, @@ -1434,7 +1434,7 @@ fn test_netbsd(t: &Target) { }); cfg.skip_struct_field_type(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // This is a weird union, don't check the type. ("ifaddrs", "ifa_ifu") => true, // sighandler_t type is super weird @@ -1446,7 +1446,7 @@ fn test_netbsd(t: &Target) { }); cfg.skip_struct_field(|struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // conflicting with `p_type` macro from . ("Elf32_Phdr", "p_type") => true, ("Elf64_Phdr", "p_type") => true, @@ -1471,7 +1471,7 @@ fn test_netbsd(t: &Target) { // Unless otherwise noted, everything in this block was an addition in NetBS 10. if netbsd9 { - cfg.skip_const(move |constant| match constant.ident() { + cfg.skip_const(move |constant| match constant.path() { "EOWNERDEAD" | "ENOTRECOVERABLE" | "F_GETPATH" @@ -1492,7 +1492,7 @@ fn test_netbsd(t: &Target) { _ => false, }); - cfg.skip_struct(move |struct_| match struct_.ident() { + cfg.skip_struct(move |struct_| match struct_.path() { "sockaddr_dl" if !netbsd9 => true, // Last field increased size in 10 x if x.starts_with("ptrace_lwp") => true, // These were packed before NetBSD 10 @@ -1500,13 +1500,13 @@ fn test_netbsd(t: &Target) { _ => false, }); - cfg.skip_fn(move |func| match func.ident() { + cfg.skip_fn(move |func| match func.path() { "reallocarray" | "getentropy" | "ppoll" | "getrandom" => true, x if x.starts_with("timerfd_") => true, _ => false, }); - cfg.skip_struct_field(|struct_, field| match (struct_.ident(), field.ident()) { + cfg.skip_struct_field(|struct_, field| match (struct_.path(), field.ident()) { ("statvfs", "f_mntfromlabel") => true, // added field ("kevent", "udata") => true, // changed type (ABI-compatible) _ => false, @@ -1652,18 +1652,18 @@ fn test_dragonflybsd(t: &Target) { match field.ident() { // Our stat *_nsec fields normally don't actually exist but are part // of a timeval struct - s if s.ends_with("_nsec") && struct_.ident().starts_with("stat") => { + s if s.ends_with("_nsec") && struct_.path().starts_with("stat") => { Some(s.replace("e_nsec", ".tv_nsec")) } // Field is named `type` in C but that is a Rust keyword, // so these fields are translated to `type_` in the bindings. - "type_" if struct_.ident() == "rtprio" => Some("type".to_string()), + "type_" if struct_.path() == "rtprio" => Some("type".to_string()), _ => None, } }); cfg.skip_alias(move |ty| { - match ty.ident() { + match ty.path() { // sighandler_t is crazy across platforms "sighandler_t" => true, _ => false, @@ -1671,7 +1671,7 @@ fn test_dragonflybsd(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // FIXME(dragonflybsd): These are tested as part of the linux_fcntl tests since // there are header conflicts when including them with all the other // structs. @@ -1703,7 +1703,7 @@ fn test_dragonflybsd(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { "SIG_DFL" | "SIG_ERR" | "SIG_IGN" => true, // sighandler_t weirdness // Kernel-only symbols in DragonFly headers. @@ -1763,7 +1763,7 @@ fn test_dragonflybsd(t: &Target) { cfg.skip_fn(move |func| { // skip those that are manually verified - match func.ident() { + match func.path() { "getrlimit" | "getrlimit64" | // non-int in 1st arg "setrlimit" | "setrlimit64" | // non-int in 1st arg "prlimit" | "prlimit64" // non-int in 2nd arg @@ -1783,7 +1783,7 @@ fn test_dragonflybsd(t: &Target) { }); cfg.skip_alias(move |ty| { - match ty.ident() { + match ty.path() { // sighandler_t is crazy across platforms "sighandler_t" => true, // Same as FreeBSD: `kvm_t` is an opaque handle used through @@ -1794,7 +1794,7 @@ fn test_dragonflybsd(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // FIXME(dragonflybsd): These are tested as part of the linux_fcntl tests since // there are header conflicts when including them with all the other // structs. @@ -1807,7 +1807,7 @@ fn test_dragonflybsd(t: &Target) { }); cfg.skip_struct_field(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // this is actually a union on linux, so we can't represent it well and // just insert some padding. ("siginfo_t", "_pad") => true, @@ -1825,7 +1825,7 @@ fn test_dragonflybsd(t: &Target) { }); cfg.skip_struct_field_type(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // This is a weird union, don't check the type. ("ifaddrs", "ifa_ifu") => true, // sighandler_t type is super weird @@ -1937,18 +1937,18 @@ fn test_wasi(t: &Target) { // These have a different and internal type in header files and are only // used here to generate a pointer to them in bindings so skip these tests. - cfg.skip_static(|s| s.ident().starts_with("_CLOCK_")); + cfg.skip_static(|s| s.path().starts_with("_CLOCK_")); match wasi_sdk.1 { WasiVersion::P1 => {} // This was removed in wasip2 target for wasi-sdk-30+, but it's just a // typedef, so ignore it. _ => { - cfg.skip_alias(|s| s.ident() == "__wasi_rights_t"); + cfg.skip_alias(|s| s.path() == "__wasi_rights_t"); } } - cfg.skip_const(|c| match c.ident() { + cfg.skip_const(|c| match c.path() { // These constants aren't yet defined in wasi-libc. // Exposing them is being tracked by https://github.com/WebAssembly/wasi-libc/issues/531. "SO_BROADCAST" | "SO_LINGER" => true, @@ -1957,14 +1957,14 @@ fn test_wasi(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // Extern types "DIR" | "FILE" | "__locale_struct" => true, _ => false, } }); - cfg.skip_fn(|f| match f.ident() { + cfg.skip_fn(|f| match f.path() { // This function doesn't actually exist in libc's header files "__errno_location" => true, @@ -1979,7 +1979,7 @@ fn test_wasi(t: &Target) { // d_name is declared as a flexible array in WASI libc, so it // doesn't support sizeof. - cfg.skip_struct_field(|s, field| s.ident() == "dirent" && field.ident() == "d_name"); + cfg.skip_struct_field(|s, field| s.path() == "dirent" && field.ident() == "d_name"); ctest::generate_test(&mut cfg, "../src/lib.rs", "ctest_output.rs").unwrap(); } @@ -2162,7 +2162,7 @@ fn test_android(t: &Target) { }); cfg.rename_struct_field(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // Our stat *_nsec fields normally don't actually exist but are part // of a timeval struct ("stat" | "statfs" | "statvfs" | "stat64" | "statfs64" | "statvfs64", f) @@ -2179,7 +2179,7 @@ fn test_android(t: &Target) { }); cfg.skip_alias(move |ty| { - match ty.ident() { + match ty.path() { // FIXME(android): `sighandler_t` type is incorrect, see: // https://github.com/rust-lang/libc/issues/1359 "sighandler_t" => true, @@ -2198,7 +2198,7 @@ fn test_android(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // These are tested as part of the linux_fcntl tests since there are // header conflicts when including them with all the other structs. "termios2" => true, @@ -2224,7 +2224,7 @@ fn test_android(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { // The IPV6 constants are tested in the `linux_ipv6.rs` tests: | "IPV6_FLOWINFO" | "IPV6_FLOWLABEL_MGR" @@ -2365,7 +2365,7 @@ fn test_android(t: &Target) { cfg.skip_fn(move |func| { // skip those that are manually verified - match func.ident() { + match func.path() { // FIXME(android): for unknown reasons linker unable to find "fexecve" "fexecve" => true, @@ -2446,7 +2446,7 @@ fn test_android(t: &Target) { }); cfg.skip_struct_field_type(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // This is a weird union, don't check the type. ("ifaddrs", "ifa_ifu") => true, // this one is an anonymous union @@ -2463,7 +2463,7 @@ fn test_android(t: &Target) { }); cfg.skip_struct_field(|struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // conflicting with `p_type` macro from . ("Elf32_Phdr", "p_type") => true, ("Elf64_Phdr", "p_type") => true, @@ -2688,7 +2688,7 @@ fn test_freebsd(t: &Target) { }); cfg.rename_struct_field(|struct_, field_| { - let struct_ = struct_.ident(); + let struct_ = struct_.path(); let replacement = match field_.ident() { // Our stat *_nsec fields normally don't actually exist but are part // of a timeval struct @@ -2709,7 +2709,7 @@ fn test_freebsd(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { // These constants were introduced in FreeBSD 13: "F_ADD_SEALS" | "F_GET_SEALS" | "F_SEAL_SEAL" | "F_SEAL_SHRINK" | "F_SEAL_GROW" | "F_SEAL_WRITE" @@ -2982,7 +2982,7 @@ fn test_freebsd(t: &Target) { }); cfg.skip_alias(move |ty| { - match ty.ident() { + match ty.path() { // the struct "__kvm" is quite tricky to bind so since we only use a pointer to it // for now, it doesn't matter too much... "kvm_t" => true, @@ -2994,7 +2994,7 @@ fn test_freebsd(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // `procstat` is a private struct "procstat" => true, @@ -3040,7 +3040,7 @@ fn test_freebsd(t: &Target) { cfg.skip_fn(move |func| { // skip those that are manually verified - match func.ident() { + match func.path() { // This is introduced in FreeBSD 14.1 "execvpe" => true, @@ -3102,10 +3102,10 @@ fn test_freebsd(t: &Target) { // aio_buf is a volatile void* but since we cannot express that in // Rust types, we have to explicitly tell the checker about it here: - cfg.volatile_struct_field(|s, f| s.ident() == "aiocb" && f.ident() == "aio_buf"); + cfg.volatile_struct_field(|s, f| s.path() == "aiocb" && f.ident() == "aio_buf"); cfg.skip_struct_field(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // FIXME(freebsd): `sa_sigaction` has type `sighandler_t` but that type is // incorrect, see: https://github.com/rust-lang/libc/issues/1359 ("sigaction", "sa_sigaction") => true, @@ -3290,7 +3290,7 @@ fn test_emscripten(t: &Target) { match field.ident() { // Our stat *_nsec fields normally don't actually exist but are part // of a timeval struct - s if s.ends_with("_nsec") && struct_.ident().starts_with("stat") => { + s if s.ends_with("_nsec") && struct_.path().starts_with("stat") => { Some(s.replace("e_nsec", ".tv_nsec")) } _ => None, @@ -3298,7 +3298,7 @@ fn test_emscripten(t: &Target) { }); cfg.skip_alias(move |ty| { - match ty.ident() { + match ty.path() { // sighandler_t is crazy across platforms // FIXME(emscripten): is this necessary? "sighandler_t" => true, @@ -3310,7 +3310,7 @@ fn test_emscripten(t: &Target) { }); cfg.skip_union(move |union_| { - match union_.ident() { + match union_.path() { // FIXME(emscripten): Investigate why the test fails. // Skip for now to unblock CI. "sigval" => true, @@ -3324,7 +3324,7 @@ fn test_emscripten(t: &Target) { }); cfg.skip_alias(|ty| { - match ty.ident() { + match ty.path() { // LFS64 types have been removed in Emscripten 3.1.44 // https://github.com/emscripten-core/emscripten/pull/19812 ty => ty.ends_with("64") || ty.ends_with("64_t"), @@ -3332,7 +3332,7 @@ fn test_emscripten(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // FIXME(emscripten): Investigate why the test fails. // Skip for now to unblock CI. "pthread_condattr_t" => true, @@ -3351,7 +3351,7 @@ fn test_emscripten(t: &Target) { }); cfg.skip_fn(move |func| { - match func.ident() { + match func.path() { // Emscripten does not support fork/exec/wait or any kind of multi-process support // https://github.com/emscripten-core/emscripten/blob/3.1.68/tools/system_libs.py#L1100 "execv" | "execve" | "execvp" | "execvpe" | "fexecve" | "wait4" => true, @@ -3372,7 +3372,7 @@ fn test_emscripten(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { // FIXME(emscripten): emscripten uses different constants to constructs these n if n.contains("__SIZEOF_PTHREAD") => true, @@ -3416,7 +3416,7 @@ fn test_emscripten(t: &Target) { }); cfg.skip_struct_field_type(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // This is a weird union, don't check the type. ("ifaddrs", "ifa_ifu") => true, // sighandler_t type is super weird @@ -3426,7 +3426,7 @@ fn test_emscripten(t: &Target) { }); cfg.skip_struct_field(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // _sigev_un is an anonymous union ("sigevent", "_sigev_un") => true, // this is actually a union on linux, so we can't represent it well and @@ -3611,7 +3611,7 @@ fn test_neutrino(t: &Target) { _ => None, }); - cfg.volatile_struct_field(|s, f| match (s.ident(), f.ident()) { + cfg.volatile_struct_field(|s, f| match (s.path(), f.ident()) { ("aiocb", "aio_buf") => true, ("qtime_entry", "nsec_tod_adjust") => true, ("qtime_entry", "nsec") => true, @@ -3621,7 +3621,7 @@ fn test_neutrino(t: &Target) { }); cfg.skip_alias(move |ty| { - match ty.ident() { + match ty.path() { // FIXME(sighandler): `sighandler_t` type is incorrect, see: // https://github.com/rust-lang/libc/issues/1359 "sighandler_t" => true, @@ -3634,7 +3634,7 @@ fn test_neutrino(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { "Elf64_Phdr" | "Elf32_Phdr" => true, // union @@ -3648,7 +3648,7 @@ fn test_neutrino(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { // These signal "functions" are actually integer values that are casted to a fn ptr // This causes the compiler to err because of "illegal cast of int to ptr". "SIG_DFL" => true, @@ -3670,7 +3670,7 @@ fn test_neutrino(t: &Target) { cfg.skip_fn(move |func| { // skip those that are manually verified - match func.ident() { + match func.path() { // wrong signature "signal" => true, @@ -3704,12 +3704,12 @@ fn test_neutrino(t: &Target) { cfg.skip_struct_field_type(move |struct_, field| { // Anonymous structures - struct_.ident() == "_idle_hook" && field.ident() == "time" + struct_.path() == "_idle_hook" && field.ident() == "time" }); cfg.skip_struct_field(|struct_, field| { matches!( - (struct_.ident(), field.ident()), + (struct_.path(), field.ident()), ("__sched_param", "reserved") | ("sched_param", "reserved") | ("sigevent", "__padding1") // ensure alignment @@ -3719,7 +3719,7 @@ fn test_neutrino(t: &Target) { ) }); - cfg.skip_static(move |static_| static_.ident() == "__dso_handle"); + cfg.skip_static(move |static_| static_.path() == "__dso_handle"); ctest::generate_test(&mut cfg, "../src/lib.rs", "ctest_output.rs").unwrap(); } @@ -3819,7 +3819,7 @@ fn test_vxworks(t: &Target) { "net/if.h", ); // FIXME(vxworks) - cfg.skip_const(move |constant| match constant.ident() { + cfg.skip_const(move |constant| match constant.path() { // sighandler_t weirdness "SIG_DFL" | "SIG_ERR" | "SIG_IGN" // These are not defined in VxWorks @@ -3831,13 +3831,13 @@ fn test_vxworks(t: &Target) { _ => false, }); // FIXME(vxworks) - cfg.skip_alias(move |ty| match ty.ident() { + cfg.skip_alias(move |ty| match ty.path() { "stat64" | "sighandler_t" | "off64_t" => true, _ => false, }); cfg.skip_struct_field_type( - move |struct_, field| match (struct_.ident(), field.ident()) { + move |struct_, field| match (struct_.path(), field.ident()) { ("siginfo_t", "si_value") | ("stat", "st_size") // sighandler_t type is super weird @@ -3857,7 +3857,7 @@ fn test_vxworks(t: &Target) { }); // FIXME(vxworks) - cfg.skip_fn(move |func| match func.ident() { + cfg.skip_fn(move |func| match func.path() { // sighandler_t "signal" // This is used a realpath and not _realpath @@ -3868,7 +3868,7 @@ fn test_vxworks(t: &Target) { }); // Not defined in vxworks. Just a crate specific union type. - cfg.skip_union(move |u| u.ident() == "sa_u_t"); + cfg.skip_union(move |u| u.path() == "sa_u_t"); ctest::generate_test(&mut cfg, "../src/lib.rs", "ctest_output.rs").unwrap(); } @@ -4261,7 +4261,7 @@ fn test_linux(t: &Target) { }); cfg.rename_struct_field(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // Our stat *_nsec fields normally don't actually exist but are part // of a timeval struct - this is fixed in musl_v1_2 ("stat" | "statfs" | "statvfs" | "stat64" | "statfs64" | "statvfs64", f) @@ -4283,7 +4283,7 @@ fn test_linux(t: &Target) { }); cfg.skip_alias(move |alias| { - let ty = alias.ident(); + let ty = alias.path(); // FIXME(musl): very recent additions to musl, not yet released. // also apparently some glibc versions if ty == "Elf32_Relr" || ty == "Elf64_Relr" { @@ -4328,7 +4328,7 @@ fn test_linux(t: &Target) { }); cfg.skip_struct(move |struct_| { - let ty = struct_.ident(); + let ty = struct_.path(); // LFS64 types have been removed in musl 1.2.4+ if musl && (ty.ends_with("64") || ty.ends_with("64_t")) { @@ -4470,7 +4470,7 @@ fn test_linux(t: &Target) { }); cfg.skip_const(move |constant| { - let name = constant.ident(); + let name = constant.path(); // FIXME(linux): Requires newer kernel headers than CI has. These uapi/linux/mount.h // constants (OPEN_TREE_NAMESPACE landed in v7.0, FSCONFIG_CMD_CREATE_EXCL in v6.6) @@ -4895,7 +4895,7 @@ fn test_linux(t: &Target) { }); cfg.skip_fn(move |function| { - let name = function.ident(); + let name = function.path(); // skip those that are manually verified match name { // There are two versions of the sterror_r function, see @@ -5006,7 +5006,7 @@ fn test_linux(t: &Target) { }); cfg.skip_struct_field_type(move |union_, field| { - match (union_.ident(), field.ident()) { + match (union_.path(), field.ident()) { // This is a weird union, don't check the type. ("ifaddrs", "ifa_ifu") => true, // sighandler_t type is super weird @@ -5035,10 +5035,10 @@ fn test_linux(t: &Target) { } }); - cfg.volatile_struct_field(|s, f| s.ident() == "aiocb" && f.ident() == "aio_buf"); + cfg.volatile_struct_field(|s, f| s.path() == "aiocb" && f.ident() == "aio_buf"); cfg.skip_struct_field(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.path()) { // musl names this __dummy1 but it's still there ("glob_t", "gl_flags") if musl => true, // musl seems to define this as an *anonymous* bitfield @@ -5174,10 +5174,10 @@ fn test_linux(t: &Target) { "B3500000", "B38400", "B4000000", "B460800", "B4800", "B50", "B500000", "B57600", "B576000", "B600", "B614400", "B7200", "B75", "B76800", "B921600", "B9600", ] - .contains(&s.ident()) + .contains(&s.path()) }); if mips || sparc { - cfg.skip_const(|s| s.ident() == "NCCS"); + cfg.skip_const(|s| s.path() == "NCCS"); } // old symbols, so tests fail if glibc is too new // note: `skip_fn_ptrcheck` overrides the previous function @@ -5196,7 +5196,7 @@ fn test_linux(t: &Target) { }); // old structs, so tests fail if glibc is too new if mips || sparc { - cfg.skip_struct(|s| s.ident() == "termios"); + cfg.skip_struct(|s| s.path() == "termios"); } } @@ -5229,7 +5229,7 @@ fn test_linux_like_apis(t: &Target) { .skip_const(|_| true) .skip_struct(|_| true) .skip_union(|_| true) - .skip_fn(|function| function.ident() != "strerror_r"); + .skip_fn(|function| function.path() != "strerror_r"); ctest::generate_test(&mut cfg, "../src/lib.rs", "linux_strerror_r.rs").unwrap(); } @@ -5252,7 +5252,7 @@ fn test_linux_like_apis(t: &Target) { .skip_struct(|_| true) .skip_union(|_| true) .skip_fn(|_| true) - .skip_const(move |constant| !fnctl_constants.contains(&constant.ident())); + .skip_const(move |constant| !fnctl_constants.contains(&constant.path())); config_gnu_bits(t, &mut cfg); if musl { @@ -5277,9 +5277,9 @@ fn test_linux_like_apis(t: &Target) { .skip_alias(|_| true) .skip_static(|_| true) .skip_fn(|_| true) - .skip_const(move |constant| !termios_constants.contains(&constant.ident())) + .skip_const(move |constant| !termios_constants.contains(&constant.path())) .skip_union(|_| true) - .skip_struct(|s| s.ident() != "termios2") + .skip_struct(|s| s.path() != "termios2") .rename_type(move |ty| match ty { "Ioctl" if gnu => Some("unsigned long".to_string()), "Ioctl" => Some("int".to_string()), @@ -5304,7 +5304,7 @@ fn test_linux_like_apis(t: &Target) { .skip_fn(|_| true) .skip_struct(|_| true) .skip_union(|_| true) - .skip_const(move |constant| !ipv6_constants.contains(&constant.ident())); + .skip_const(move |constant| !ipv6_constants.contains(&constant.path())); config_gnu_bits(t, &mut cfg); headers!(cfg, "linux/in6.h",); @@ -5325,8 +5325,8 @@ fn test_linux_like_apis(t: &Target) { .skip_const(|_| true) .skip_union(|_| true) .rename_struct_ty(move |ty| Some(ty.to_string())) - .skip_struct(move |struct_| !elf_structs.contains(&struct_.ident())) - .skip_alias(move |alias| !elf_structs.contains(&alias.ident())); + .skip_struct(move |struct_| !elf_structs.contains(&struct_.path())) + .skip_alias(move |alias| !elf_structs.contains(&alias.path())); config_gnu_bits(t, &mut cfg); @@ -5344,7 +5344,7 @@ fn test_linux_like_apis(t: &Target) { .skip_struct(|_| true) .skip_union(|_| true) .skip_alias(|_| true) - .skip_const(move |constant| constant.ident() != "ARPHRD_CAN"); + .skip_const(move |constant| constant.path() != "ARPHRD_CAN"); ctest::generate_test(&mut cfg, "../src/lib.rs", "linux_if_arp.rs").unwrap(); } @@ -5497,7 +5497,7 @@ fn test_haiku(t: &Target) { ); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // FIXME(haiku): locale_t does not exist on Haiku "locale_t" => true, // FIXME(haiku): rusage has a different layout on Haiku @@ -5523,7 +5523,7 @@ fn test_haiku(t: &Target) { }); cfg.skip_alias(move |ty| { - match ty.ident() { + match ty.path() { // FIXME(haiku): locale_t does not exist on Haiku "locale_t" => true, // These cause errors, to be reviewed in the future @@ -5538,7 +5538,7 @@ fn test_haiku(t: &Target) { cfg.skip_fn(move |func| { // skip those that are manually verified - match func.ident() { + match func.path() { // FIXME(haiku): does not exist on haiku "open_wmemstream" => true, "mlockall" | "munlockall" => true, @@ -5562,7 +5562,7 @@ fn test_haiku(t: &Target) { }); cfg.skip_const(move |constant| { - match constant.ident() { + match constant.path() { // FIXME(haiku): these constants do not exist on Haiku "DT_UNKNOWN" | "DT_FIFO" | "DT_CHR" | "DT_DIR" | "DT_BLK" | "DT_REG" | "DT_LNK" | "DT_SOCK" => true, @@ -5588,7 +5588,7 @@ fn test_haiku(t: &Target) { }); cfg.skip_struct_field(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // FIXME(time): the stat struct actually has timespec members, whereas // the current representation has these unpacked. ("stat", "st_atime") => true, @@ -5659,7 +5659,7 @@ fn test_haiku(t: &Target) { }); cfg.rename_struct_field(move |struct_, field| { - let struct_ = struct_.ident(); + let struct_ = struct_.path(); match field.ident() { // Field is named `type` in C but that is a Rust keyword, // so these fields are translated to `type_` in the bindings. @@ -5777,7 +5777,7 @@ fn test_aix(t: &Target) { "wchar.h", ); - cfg.skip_alias(move |ty| match ty.ident() { + cfg.skip_alias(move |ty| match ty.path() { // AIX does not define type 'sighandler_t'. "sighandler_t" => true, @@ -5794,7 +5794,7 @@ fn test_aix(t: &Target) { _ => None, }); - cfg.skip_const(move |constant| match constant.ident() { + cfg.skip_const(move |constant| match constant.path() { // Skip 'sighandler_t' assignments. "SIG_DFL" | "SIG_ERR" | "SIG_IGN" => true, @@ -5821,7 +5821,7 @@ fn test_aix(t: &Target) { }); cfg.skip_struct(move |struct_| { - match struct_.ident() { + match struct_.path() { // 'struct fpreg_t' is not defined in AIX headers. It is created to // allow type 'double' to be used in signal contexts. "fpreg_t" => true, @@ -5838,7 +5838,7 @@ fn test_aix(t: &Target) { }); cfg.skip_union(|union_| { - match union_.ident() { + match union_.path() { // '__poll_ctl_ext_u' and '__pollfd_ext_u' are for unnamed unions. "__poll_ctl_ext_u" => true, "__pollfd_ext_u" => true, @@ -5856,7 +5856,7 @@ fn test_aix(t: &Target) { }); cfg.skip_struct_field_type(move |struct_, field| { - match (struct_.ident(), field.ident()) { + match (struct_.path(), field.ident()) { // AIX does not define 'sighandler_t'. ("sigaction", "sa_sigaction") => true, @@ -5878,7 +5878,7 @@ fn test_aix(t: &Target) { }); cfg.skip_struct_field(move |s, field| { - match s.ident() { + match s.path() { // The field 'u' is actually a unnamed union in the AIX header. "poll_ctl_ext" if field.ident() == "u" => true, "pollfd_ext_t" if field.ident() == "u" => true, @@ -5891,7 +5891,7 @@ fn test_aix(t: &Target) { }); cfg.skip_fn(move |func| { - match func.ident() { + match func.path() { // 'sighandler_t' is not defined on AIX. "signal" => true, @@ -5954,7 +5954,7 @@ fn test_aix(t: &Target) { } }); - cfg.volatile_struct_field(|s, f| match (s.ident(), f.ident()) { + cfg.volatile_struct_field(|s, f| match (s.path(), f.ident()) { // 'aio_buf' is of type 'volatile void**' but since we cannot // express that in Rust types, we have to explicitly tell the // checker about it here. @@ -5995,7 +5995,7 @@ fn test_qurt(t: &Target) { // QuRT doesn't have all the standard unix types/structs cfg.skip_struct(|s| { - match s.ident() { + match s.path() { // These are compatibility stubs in libc, not from QuRT headers "stat" | "tm" | "timespec" | "timeval" | "itimerspec" | "dirent" | "DIR" | "termios" | "rlimit" | "rusage" | "flock" | "div_t" | "ldiv_t" | "lldiv_t" => true, @@ -6004,7 +6004,7 @@ fn test_qurt(t: &Target) { }); cfg.skip_alias(|ty| { - match ty.ident() { + match ty.path() { // Skip types not defined in QuRT POSIX headers "intptr_t" | "uintptr_t" | "ptrdiff_t" | "size_t" | "ssize_t" | "time_t" | "suseconds_t" | "useconds_t" | "timer_t" | "dev_t" | "ino_t" | "mode_t" @@ -6026,7 +6026,7 @@ fn test_qurt(t: &Target) { }); cfg.skip_const(|c| { - let name = c.ident(); + let name = c.path(); match name { // Skip constants not from QuRT POSIX headers "EOK" | "PAGESIZE" | "PAGE_SIZE" | "L_tmpnam" | "TMP_MAX" | "FOPEN_MAX" => true, @@ -6129,7 +6129,7 @@ fn test_qurt(t: &Target) { }); cfg.skip_fn(|func| { - let name = func.ident(); + let name = func.path(); match name { // Skip functions not from QuRT POSIX headers we're testing "strlen" | "strcpy" | "strncpy" | "strcat" | "strncat" | "strcmp" | "strncmp" @@ -6196,7 +6196,7 @@ fn test_qurt(t: &Target) { // Skip field checks for opaque types cfg.skip_struct_field(|s, f| { // pthread_attr_t bitfield can't be checked directly - s.ident() == "pthread_attr_t" && f.ident() == "__bitfield" + s.path() == "pthread_attr_t" && f.ident() == "__bitfield" }); cfg.skip_roundtrip(|_| true); diff --git a/notes/Demo.idr b/notes/Demo.idr new file mode 100644 index 0000000000000..e27c57684eaf0 --- /dev/null +++ b/notes/Demo.idr @@ -0,0 +1,350 @@ +module Demo + +import Data.List +import Data.Vect + +%default total + +namespace UseTree + public export + data UseTree : Type where Path : String -> UseTree -> UseTree + Name : String -> UseTree + Glob : UseTree + Group : List UseTree -> UseTree + +ModulePath : Nat -> Type +ModulePath n = Vect n String + +namespace FfiItems + public export + record FfiItems (n : Nat) where + constructor MkFfiItems + path : Vect n String + ident : String + items : List String + mods : List (FfiItems (S n)) + uses : List UseTree + +record ResolvedItem where + constructor MkResolved + items : List String + mods : List (FfiItems depth) + uses : List UseTree + +data Resolution : Type where + Resolved : UseTree -> ResolvedItem -> Resolution + Unresolved : UseTree -> Resolution + +record StatefulItems (n : Nat) where + constructor MkState + state : FfiItems Z + current : FfiItems n + +-- [NOTE]: these types are only used as proof witnesses and will not get ported +-- over to Rust. Instead, a new, refined type will be used in Rust to express +-- the constraints imposed by the below dependent types. +namespace Ungrouped + public export + data Ungrouped : UseTree -> Type where Name : Ungrouped (Name _) + Glob : Ungrouped Glob + Path : {auto prf : Ungrouped t} + -> Ungrouped (Path _ t) + + public export + data UngroupedItems : FfiItems _ -> Type where + Empty : UngroupedItems (MkFfiItems _ _ _ _ []) + Witness : {auto prfs : UngroupedItems (MkFfiItems mp mid is ms ts)} + -> {auto prf : Ungrouped t} + -> UngroupedItems (MkFfiItems mp mid is ms (t :: ts)) + +empty : ModulePath n -> String -> FfiItems n +empty p id = MkFfiItems p id [] [] [] + +-- [NOTE]: this deconstructs a module that is proven to contain no group +-- imports, into its list of imports (also proven to not contain group imports.) +-- The inverse of this is done by the `re` function. +ex : (i : FfiItems _ ** UngroupedItems i) -> List (u : UseTree ** Ungrouped u) +ex ((MkFfiItems _ _ _ _ []) ** _) = + [] +ex a@(MkFfiItems mp mid is ms (h :: t) ** Witness {prfs} {prf}) = + let na = ((MkFfiItems mp mid is ms t) ** prfs) in + (h ** prf) :: (ex $ assert_smaller a na) + +-- [NOTE]: this builds up a proof tree by deconstructing the already proven +-- trees of group-clean imports. The goal is to explain to the type-checker that +-- these imports will make up a module that is guaranteed to be clean of +-- group imports. The inverse of this is done by the `ex` function. +re : List (u : UseTree ** Ungrouped u) + -> FfiItems n + -> (i : FfiItems n ** UngroupedItems i) +re [] (MkFfiItems mp mid is ms _) = + (MkFfiItems mp mid is ms [] ** Empty) +re ((h ** hw) :: t) i = + let (MkFfiItems mp mid is ms us ** w) = re t i in + (MkFfiItems mp mid is ms (h :: us) ** Witness {prfs = w} {prf = hw}) + +normalize : FfiItems n -> (i : FfiItems n ** UngroupedItems i) +normalize it@(MkFfiItems { uses = us, _ }) = foldr f [] us |> re <| it where + -- [NOTE]: this function requires asserting to the totality checker that the + -- trees rooted at group imports are always bound to be smaller than the trees + -- rooted one level above. This is because the shape of the `UseTree` type in + -- group imports stops "growing" when it finds lists. The items of these lists + -- (themselves trees) could then be found to be potentially larger than the + -- top-level parent group. This is a contradiction because a path's tail's + -- group import is always smaller than the sum of the path's tail and the + -- path's head. + f : UseTree + -> List (u : UseTree ** Ungrouped u) + -> List (u : UseTree ** Ungrouped u) + f = (++) . f' where + f' : UseTree -> List (u : UseTree ** Ungrouped u) + f' (Name id) = [ (Name id ** Name) ] + f' Glob = [ (Glob ** Glob) ] + f' (Path id t) = f' t |> flip map <| \(t ** _) => ( Path id t + ** Path) + f' o@(Group l) = join <| l |> flip map <| \t => f' $ + assert_smaller o t + +findItem : String -> FfiItems n -> Maybe (Either String (FfiItems (S n))) +findItem id (MkFfiItems { items = is, mods = ms, _ }) = + find c $ map Left is ++ map Right ms where + c : Either String (FfiItems (S n)) -> Bool + c (Left s) = s == id + c (Right (MkFfiItems { ident = s, _ })) = s == id + +samePath : ModulePath _ -> ModulePath _ -> Bool +samePath [] [] = True +samePath (h1 :: t1) (h2 :: t2) = h1 == h2 && samePath t1 t2 +samePath _ _ = False + +covering +findParent : StatefulItems (S n) -> Maybe (FfiItems n) +findParent (MkState { state = st, current = MkFfiItems { path = cmp, _ } }) = + f st where r : FfiItems _ -> Maybe (FfiItems n) -> Maybe (FfiItems n) + f : FfiItems _ -> Maybe (FfiItems n) + r it Nothing = f it + r _ p = p + f it@(MkFfiItems mp _ _ ms _) = + case cmp |> samePath . reverse . tail . reverse <| mp of + True => believe_me $ Just it + False => foldr r Nothing ms + +namespace ResolveRoot + public export + resolveReexport : UseTree + -> (t : UseTree) + -> {auto 0 prf : Ungrouped t} + -> StatefulItems _ + -> List Resolution + resolveReexport oid (Name id) (MkState { current = it, _ }) + = case findItem id it of + Just (Left i) => + [ oid |> Resolved <| MkResolved { items = [ i ] + , mods = [ ] + , uses = [ ] + , depth = Z } ] + Just (Right m) => + [ oid |> Resolved <| MkResolved { items = [ ] + , mods = [ m ] + , uses = [ ] } ] + Nothing => + [ Unresolved oid ] + resolveReexport oid Glob (MkState { current = (MkFfiItems { items = is + , mods = ms + , uses = us + , _ }) + , _ }) + = [ Resolved oid $ MkResolved { items = is, mods = ms, uses = us } ] + resolveReexport oid (Path id t) {prf = Path {prf}} (MkState { state = st + , current = it }) + = case findItem id it of + Just (Right m) => ResolveRoot.resolveReexport oid t $ + MkState { state = st, current = m } + Just _ => [ Unresolved oid ] + Nothing => [ Unresolved oid ] + +namespace ResolveNode + public export partial + resolveReexport : UseTree + -> (t : UseTree) + -> {auto 0 prf : Ungrouped t} + -> StatefulItems (S _) + -> List Resolution + resolveReexport oid (Name id) st@(MkState { current = it, _ }) + = case id == "super" of + True => case findParent st of + Just m => + [ oid |> Resolved <| MkResolved { items = [ ] + , mods = [ m ] + , uses = [ ] } ] + Nothing => + idris_crash "[TODO]" + False => case findItem id it of + Just (Left i) => + [ oid |> Resolved <| MkResolved { items = [ i ] + , mods = [ ] + , uses = [ ] + , depth = Z } ] + Just (Right m) => + [ oid |> Resolved <| MkResolved { items = [ ] + , mods = [ m ] + , uses = [ ] } ] + Nothing => + [ Unresolved oid ] + resolveReexport oid Glob (MkState { current = (MkFfiItems { items = is + , mods = ms + , uses = us + , _ }) + , _ }) + = [ Resolved oid $ MkResolved { items = is, mods = ms, uses = us } ] + resolveReexport oid (Path id t) {prf = Path {prf}} + ist@(MkState { state = st, current = it }) + = case id == "super" of + True => case findParent ist of + Just m => + case length . path $ m of + Z => + ResolveRoot.resolveReexport oid t $ + MkState { state = st, current = m } + o@(S _) => + let nst : StatefulItems o + nst = believe_me $ MkState { state = st + , current = m } + in ResolveNode.resolveReexport oid t nst + Nothing => idris_crash "[TODO]" + False => case findItem id it of + Just (Right m) => ResolveNode.resolveReexport oid t $ + MkState { state = st + , current = m } + Just _ => [ Unresolved oid ] + Nothing => [ Unresolved oid ] + +partial +resolveOne : { n : _ } + -> (i : FfiItems n ** UngroupedItems i) + -> FfiItems Z + -> List Resolution +resolveOne a@(it ** _) st = join . (map $ f) . ex $ a where + p : StatefulItems n + f : (u : UseTree ** Ungrouped u) -> List Resolution + p = MkState st it + f (u ** _) = case n of Z => ResolveRoot.resolveReexport u u p + S _ => ResolveNode.resolveReexport u u p + + + +covering +merge : (i : FfiItems n ** UngroupedItems i) + -> List Resolution + -> (i : FfiItems n ** UngroupedItems i) +merge it@(_ ** _) [] = it +merge it@(_ ** _) ((Unresolved _) :: t) = merge it t +merge it@(iit ** _) ((Resolved oid (MkResolved is ms _)) :: t) = merge nit t + where + covering + transform : FfiItems _ + -> (FfiItems m, List (FfiItems (S m))) + -> (FfiItems m, List (FfiItems (S m))) + transform it (st, its) = let np := st |> snoc . path <| ident it in + ( st + , { path := np + , mods := mods it |> + foldr transform (empty np $ ident it, []) |> snd } it :: its ) + + covering + nit : (i : FfiItems n ** UngroupedItems i) + nit = + let nus = (deleteBy f oid) . ex $ it + in { items $= (++ is) + , mods := foldr transform (iit, []) ms |> snd } iit |> re nus + where + f : UseTree -> (u : UseTree ** Ungrouped u) -> Bool + f (Name id1) (Name id2 ** _) = + id1 == id2 + f Glob (Glob ** _) = + True + f (Path id1 t1) (Path id2 t2 ** Path {prf}) = + id1 == id2 && f t1 (t2 ** prf) + f _ _ = + False + +covering +updateMod : FfiItems n -> FfiItems Z -> FfiItems Z +updateMod it@(MkFfiItems { path = itp, _ }) st@(MkFfiItems { path = stp, _ }) = + case samePath stp itp of True => believe_me it + False => { mods $= foldr f [] } st + where f : FfiItems m -> List (FfiItems m) -> List (FfiItems m) + f fit@(MkFfiItems { path = fitp, _ }) fits = + case samePath fitp itp of True => believe_me it :: fits + False => { mods $= foldr f [] } fit :: fits + +partial +resolveDriver : { n : _ } -> StatefulItems n -> StatefulItems n +resolveDriver (MkState { state = st, current = it }) = + let (nst, nms) := (foldr r (st, [])) . mods $ it + nit := normalize . { mods := nms } $ it + in f nit nst where + r : FfiItems (S n) + -> (FfiItems Z, List (FfiItems (S n))) + -> (FfiItems Z, List (FfiItems (S n))) + r it (st, its) = + let (MkState { state = nst, current = nit }) := + resolveDriver $ MkState { state = st, current = it } + in (nst, nit :: its) + + f : (i : FfiItems n ** UngroupedItems i) -> FfiItems Z -> StatefulItems n + f a@(it ** _) st = + let na@(nit ** _) := st |> resolveOne a |> merge a + nst := updateMod nit st + in case (List.length . uses $ it) == (List.length . uses $ nit) of + True => MkState { state = nst, current = nit } + False => f na nst + +partial +resolve : FfiItems Z -> FfiItems Z +resolve it = MkState it it |> resolveDriver |> current + +-- [NOTE]: the following tests comprise only test data. To test it out, the +-- Idris REPL is required. + +-- [NOTE]: this test is for the resolution function `resolve`. +-- test1 : FfiItems +-- test1 = let bar := { items := [ "Foo" ] } . (empty "foo::bar") $ "bar" +-- foo := { mods := [ bar ] } . (empty "foo") $ "foo" +-- in { mods := [ foo ] +-- , uses := [ Path "bar" Glob, Path "foo" Glob ] } . (empty "") $ "" + +-- [NOTE]: this test is for the normalization function `normalize`. +-- test2 : FfiItems +-- test2 = +-- let g := [ Name "TypeId", Name "Any" ] +-- in { uses := [ Path "std" (Path "any" (Group g)) ] } . (empty "") $ "" + +-- [NOTE]: this test is for the normalization function `normalize`. +-- test3 : FfiItems +-- test3 = +-- let g := [ Path "foo" (Path "bar" (Group [ Name "Ty" +-- , Path "test" (Name "foobar") ])) +-- , Name "TypeId" +-- , Name "Any" ] +-- in { uses := [ Path "std" (Path "any" (Group g)) ] } . (empty "") $ "" + +-- [NOTE]: this test is for the resolution function `resolve`. +-- test4 : FfiItems +-- test4 = +-- let uses := [ Path "std" (Path "os" (Path "raw" (Name "c_void"))) +-- , Path "level1" Glob ] +-- id := "level1" +-- level1 := { items := [ "Foo", "bar", "Word" ] } . (empty id) $ id +-- mods := [ level1 ] +-- items := [ "Array", "baz", "malloc" ] +-- in { items := items, mods := mods, uses := uses } . (empty "") $ "" + +-- [NOTE]: this test is for the resolution function `resolve`. +-- test5 : FfiItems +-- test5 = +-- let foobar := { items := [ "Foo" ] } . (empty "foo::bar::foobar") $ "foobar" +-- bar := { mods := [ foobar ] } . (empty "foo::bar") $ "bar" +-- foo := { mods := [ bar ] } . (empty "foo") $ "foo" +-- in { uses := [ Path "foobar" Glob, Path "bar" Glob, Path "foo" (Name "bar") ] +-- , mods := [ foo ] } . (empty "") $ "" diff --git a/notes/main.typ b/notes/main.typ new file mode 100644 index 0000000000000..36e4d0c261730 --- /dev/null +++ b/notes/main.typ @@ -0,0 +1,326 @@ +#import "@local/scratchpad:0.1.4": * + +#show: template.with(title: [Notes on extending `ctest`]) + +#title() + += On the design and implementation of the rename proposition +The function is known to be needed both in the merging function and in the +rename case of the single-reexport resolution proposition. The proposition can +assume true a closure, and imply another proposition from a module to a module. + +The closure would be passed the path to some item, and it would be expected to +return another path. The new path would then be used to replace that of the item +while performing a traversal of the assumed module tree. + +Both values may be passed by value to the proposition, and the closure may take +as well a fully-owned path, and return a fully-owned path. The proof would be +shaped as a single-run of the closure over the root, and a map of the children. + +The path of the module would first be passed to the closure. Then a new module +would be produced, where the path would correspond with that returned by the +clsoure. The map over the child items would repeat the same things. + +Note the traversal would be required to go through all items in the subtree +rooted at the passed module. This implies that there ought be multiple maps for +each of the containers used for each type of item. + +Recusion would take place when the container being handled would correspond with +that of child modules to the currently traversed module. For each of those, the +same logic would follow, but additionally the mapping would take place anew. + +This hints at the possibility to abstract away the module-pass in a single inner +function that would take the module and the closure, and yield the updated +module tree rooted at the passed module. + +This is exactly what the whole renaming proposition would do. This implies that +the logic should only consist of performing the update over the module path, +then repeating the same thing over the child items, recursing with the modules. + +The only detail of the proof that remains to be specified is the behavior post +recursing with each of the child modules. The returned module would be assigned +instead to the prior module. This means there is no need to rename the child. + +The renaming of the module's path will already take place within the next call +stack frame. There is one thing that remains to be specified; The module path +exists not only in the corresponding field, but also in a cached form. + +The cached form would also need updating after the path were changed. This would +seem to be readily possible with the proposition that is also used at parse-time +to yield the same paths serving as a stringified cache of the data structure. + +It would seem the proof is complete; There is something odd with it, though. Let +there be a review of the path manipulation strategy that would be used when +handling renames in the single-reexport resolution proposition. + +Renaming the last segment of all paths appearing in a given module subtree would +need to change the last segment of that tree's root, followed by a change of all +subsequent child items' path segments corresponding to the ancestor's old ident. + +This could be accomplished by capturing in the closure the original path of the +ancestor module. Then whatever followed from the path passed to the clsoure post +extracting this path would be reappended to the modified ancestor's path. + +So to picture an example; Consider path foo::bar. The module tree rooted at +module bar should be renamed to baz. This would mean that all items whose paths +started with foo::baz in that tree would first extract the path after this. + +For example, if there is some item with path foo::bar::foobar::Foo, the closure +would extract the path after the original ancestor's foo::bar path. This would +correspond with foobar::Foo. + +Then this would be appended to the result of modifying the root module's path to +foo::baz. The end result would be path foo::baz::foobar::Foo. Of course, it +could be that there is no path after the root path, like the root module itself. + +In those cases, the append opeeration would yield only the modified root path. +Notably, the Rust runtime has no way of knowing that the path being extracted +from the path passed to the closure is prepended by the root module's path. + +This will require some potential unreachable! calls, but that should be +acceptable. Once the proposition for handing use-trees with renames in them is +discussed, the details of the proof will be provided. + +Something that remains to be discussed from the proofing of the path +manipulation proposition is that there exists a potential issue with the way it +handles the traits implemented by the passed closure. + +The trat implementation of the passed closure is also required to be Clone +because otherwise there are issues with using it in the item-list-mapping +closures. This constraint seems not feasible lest closures implement Clone. + +It would seem closures do implement Clone. Constraining the closure trait any +further with an immutable and recallable trait seems neither feasible. The FnMut +trait is not required, and FnOnce can only be assumed to be a one-off thing. + +The only worrying thing about the path manipulation strategy, as well as all +other propositions whose proofs use recursiveness, is that the Rust runtime +ensures not unboundedness of recursion in most cases. + +This consideration will be left for later. Current endeavors are rather focused +on the renaming function to complete case handling during single-reexport +resolution, and on the merging function. These are the only two things left. + +The renaming case may now more easily proceed by using the path manipulation +proposition. Looking back on it, the refined type for use statements seems like +it could benefit from making the glob injection be a const functor. + +Collapsing the contents of a glob seems feasible because it contains no +information of use. Now back to the rename case. In this instance, it is known +that the rename case holds both a source identifier and a rename. + +The first thing to do here would be a lookup of the source identifier. If found, +then the returned item (if not a module) would suffer in-place modification of +its path by creating a new (non-module) item. + +If the returned item were to be a module, then the renamed identifier alongside +the yield module would have to be used with the path manipulation proposition. +Let the proof be written up to this point. + +Further discussion is merited on the side of renaming non-module items. The path +of the item should be changed; One can be sure that post-parsing there are no +items whose paths are empty. + +Beyond that, the last segment of the path can be readily removed from the +original path segments. Then the renamed identifier can be pushed anew into the +punctuated list of segments. + +This process is mechanical enough for each item type that it could potentially +be automated by means of an MBE. The path field is always available, no matter +the item; All but the final operation is then item-independent. + +The last required action is to reconstruct the item with the updated path, and +the updated cache for the path. This should be fairly simple to accomplish. The +only case left to address is that of modules. + +When matching against a module in the rename case, the proof should instead use +the path manipulation proposition. The closure assumed true in that proposition +should capture the original path of the subtree root module. + +Then for each path passed to the closure, it should extract whatever is left of +the path after the root set of segments, and prepend to that the updated path to +the root module. This goes through potentially caching the updated path. + +This idea is still fuzzy; How does one find the base path of the passed root +module in the path of each item? This can be accomplished by first taking the +length of the base path, and then removing that suffix from the path segments. + +The length can be obtained from the ancestor module outside of the closure. The +type in syn for representing path segments does not allow removal from the +front. One simple alternative is to perform random-access insertion. + +Firstly, the length of the ancestor module's path gets stored outside the +closure. Then an iterator from this length to the length of the path passed in +the closure fetches each segment pertaining only to the item's path. + +Then this gets gathered into a vector, the elements of which are iterated over +to be inserted into the updated and cached base path of the root module living +outside the closure. + +An even better approach may be to get first the difference of the length of the +item's path with respect to the module's path, and then pop from the back as +many times as that difference indicates. + +Then this can be gathered in a vector, to be reversed prior to being iterated +over and added to a new copy of the cached and updated version of the root +module's path (again, living outside the closure.) + +Or a simpler idea would be to iterate over all segments of the item's path, +skipping first through the length of the ancestor module's path length. Then the +remaining segments can be gathered into a vector. + +The vector would then be folded with a seed value consisting of a copy of the +cached new ancestor path, to which all previoulsy collected, item-specific path +segments would be appended. It would seem the proposition for handling +single-use-statement resolution has had its proof finished. The only thing left +is to provide proof for the merging proposition. In this case, this is known to +iterate through the resolution items, ignoring those that are marked unresolved. +For each resolved item, the associated use statement should be removed from the +module. Beyond that, merging should integrate the items in the container carried +for this one injection of the resolution coproduct, into the destination module. +This, as mentioned during elaboration of the path manipulation function, +requires renaming all paths in both items and module subtrees rooted at the +items to merge, such that everything but their last path segment is replaced +with the destination's module path segment. In other words, merging of resolved +items requires considering the case for non-module items, and the case for +modules. Something similar was already proofed in the rename case of the +single-use-statement resolution proposition. Maybe this is worth abstracting, or +maybe not. Either way, it seems clear items should have everything but their +trailing path segment replaced with the path of the module into which to merge. +Modules need to be fed into the path manipulation proposition, with a closure +that, again, extracts the last segment of the path, and prepends to it the path +of the module into which to merge. + += Idris community tutorial +Before starting with the Idris community tutorial, it would be best to finish +the attempt at providing a proof tha the vector splitting proposition is +correct. This was part of an initial attempt during early hours with Idris. + +The proposition requires there being a vector that implies another proposition. +This other proposition assumes a natural and implies a pair of a vector with +some TBD length, and some vector with some other TBD length. + +The splitting logic would dictate that the splitting index must fall between 0 +and the length of the vector. Constraining that type is likely possible through +finite sets, but it may prove more instructive to do so by hand. + +The proposition constraining some natural number to exist within a range should +be indexed by a natural. If one assumes an inductive definition for the type, +there is one vacuously true case; Natural zero is always in bounds. + +Thus, one case of the proposition constructs a finite set from the zeroth +natural. One other case follows then; Provided an existing finite set, there +exists another finite set representing the successor to the existing finite set. + +This would dictate that some finite set indexed by 2 can be deconstructed into a +finite set that was constructed from a finite set indexed by 1, and that was +itself constructed from a finite indexed by 2. + +Would this proposition be enough to reflect that there exists no natural, that +upon being dependently paired with a finite set, would not surpass its index? +This can be readily tested by producing a dependent natural with a finite set. + +It would seem the above design for finite sets constrains the natural to be a +specific number, and not to fall within a specific range. Consider the case of a +natural 2, which should fall within the finite set bounded from above at 3. + +The construction of the finite set follows the same construction as that of a +natural. The successor to some existing finite set builds from the prior finite +set. This allows not one to retrieve the finite set that would limit a range. + +Construction of the proof requires being capable of constructing the type of the +finite set. For some natural, if a finite set represents an upper bound, the +building of the finite set must allow representing the natural. + +Except it may very well be that the test case itself is flawed. The type of the +dependent pair specifically uses the dependant natural as the index to the +dependee finite set. This implies the natural must be the upper bound itself. + +A better test case may be found if instead the finite set is used to constrain a +natural that exists in another context. An ideal example would seem to be a +length-indexed vector whose implicit for the length were bounded by the set. + +It would seem a more precise test case may go through the proofing of an +indexing propostion for a length-indexed vector. This would consider a finite +set that represented some non-zero natural, and a vector with that length. + +The finite set would have be indexed by the successor to some natural, and the +vector length would have to also be the successor the same natural. This would +ensure the finite set represents at least numbers greater than zero. + +The proofing of this may not be as simple, though. The basecase would seem to +correspond with a proposition with a finite set for the successor of zero. This +would yield the only element in the vector. + +This could be assumed to be the inductive hypothesis. The only other case would +be for the finite set to represent the successor of some other finite set, and +thus for the vector to hold more than one element. + +In the proof, this would pattern match with the finite set as mentioned above, +and with the vector as a destructuring of its head from its tail. The only issue +to this is that this would always fetch the first element of the vector. + +The only possible cases to consider would be those. It seems the finite set as +conceived represents not a bounded natural, but the idea of a bounded natural. +One can not destructure the set into any one natural in the bounded range. + +For the indexing to work, the natural indicating the index of the element in the +vector would have to be bounded. The natural would thus require existing as a +dependent pair alongside a proposition indexed by the limit and natural. + +This proposition to limit a natural knowing the upper bound seems expressed by +the finite set. The problem with the finite set is that the natural is not +accessbile; Only the limit is known. + +A better proposition would have to build on both ideas; It would have to be +indexed by a limit and a natural. The latter would indicate the natural in +question to bound, while the former would indicate the limit of the natural. + +This seems a lot like a less-than-or-equal binary relation. It is indexed by two +naturals, and indicates the lhs is strictly smaller than or equal to the rhs. +This idea seems exactly like what vector indexing needs. + +One would wonder then what is the point of finite sets, then. It would seem +finite sets alone do not provide much value; std implements a bunch of utilities +for roundtripping between a finite set and a natural. + +These utilites deal with integer overloads between naturals and integers, and +then between naturals and finite sets. This sometimes implies use of type system +subversion through belive_me. This is unrelated to the topic at hand. + +It would seem providing a finite set then, so long as there existed a way of +converting from a natural to a finite set, is acceptable. This would mean that +the finite set exists within the limits of the vector. + +The role of finite sets is then only to provide some bound in the type signature +of a proposition. The actual work of transforming the integer into a natural in +the range denoted by the finite set is delegated to some other function. + +This would mean that the finite set is always useless outside of the guarantees +it provides at the type level. There's always need for a supporting +infrastructure to hold up the transformation between a natural and the set. + +This then means that a custom finite set is not enough for none of the indexing +nor the vector splitting routine. There seems to be an interface for converting +between integers and some other type, which is used by std's finite sets. + +Implementing integer to finite set conversion seems not to be a simple endeavor. +The integer cannot be merely any integer, and neither can it be cast first into +a natural, and pattern-matched to be used in constructing a finite set. + +The finite set is already defined in the propositions for the Num interface as +providing a bound for some natural. This means integer overloading could fail. +Most notably, a negative integer would not be good enough. + +One should only be capable of constructing a finite set off of an unsigned +integer literal. One could introduce such a constraint prior to the propositions +in the Num interface, and use the constrained integer for overloading. + +It would seem setting up constraints prior to introducing the interface +constraint is possible. With that in mind, it then becomes possible to establish +that provided a certain integer n, this integer will only become a finite set if +another proposition determines that this is possible. This other proposition +would imply a type that would serve as proposition requirement and would either +have to be proof-searched or otherwise manually provided. In theory, a proof +search seems feasible because this exists for the sole purpose of handling +overloaded literals. diff --git a/notes/new-algo.md b/notes/new-algo.md new file mode 100644 index 0000000000000..5db3002917dea --- /dev/null +++ b/notes/new-algo.md @@ -0,0 +1,1726 @@ +The new algorithm needs to support bidirectional use statements. Current +support is based around one invariant. This holds only for forward use +statements. The depth-first approach exploits the invariant. The problem +comes from the possibility for leaf modules to contain use statements. +These were previously also considered. But previously these were +guaranteed to be unresolvable import statements. These refer to use +statements that target items from third-party crates. But bidirectional +imports means there are also backward use statements. These may refer to +some item in an ancestor module. And these are resolvable use +statements. The one thing relying on this the most is the single-module +resolution proposition. It bottoms out on the list of child modules. It +resoves leaf modules first. Then it bubbles back up. Ancestor module +then get resolved with the guarantee of a fully resolved subtree. This +needs to change. But prior notes revealed order matters not. To start +resolution from the crate root does not improve things. The ideal +resolution process is fairly clear. One import gets resolved. If the +target gets some changes within it then the source of the import +observes those changes. This may seem like it calls for nifty pointer +tricks. But a pure solution is preferred. One must specify what it means +to "observe" the changes in the target. Currently target merging implies +use statement removal. That means the whole thing is settled once a +target is found. But it does not have to be this way. A target may be +merged without need for use statement removal. The pass system observes +the number of use statements before and after a pass. This should in +theory prevent potential infinite loops. But that only holds under one +condition. Single-module resolution currently never goes through the +same module again. To "observe" the changes in the use statement target +requires another pass over the use statement source module. And yet +preservation of the use-statement implies no changes to the list of use +statements. So the pass system concludes the remaining use statements +are unresolvable. It then moves on to another module. This implies the +overarching flow of resolution through the module tree has to be thought +anew. It is insufficient for a given number of passes over a module to +all happen consecutively. But keeping track of the tree nodes that +potentially require another pass is a hassle. It is best if the +top-level call to resolve the module tree from the crate root is +changed. It should loop over all modules in the tree instead of over a +single module. It should be made into a potentially non-terminating +loop. A check would be performed after one full module pass were +performed. This would be very similar to the one currently used in the +pass system. But it would instead recursively check the lists of use +statements of all modules in the tree. If they all compared the same +then it would be safe to say the algorithm may now terminate. The point +here would be to exploit deduplication. An abstract example may be in +order. Consider the algorithm is at the crate root. It finds a +dependency-free reexport. This means the reexport can be immediately +resolved. And it chooses to resolve the reexport. This means it will be +produced as a resolution item. This item will then be merged into the +module in which the use statement appeared at. But here's the difference +with the prior revision of the algorithm. The use statement is not +removed. It is instead kept as-is. The termination condition in the +single module resolution proposition is then removed. A single pass is +the only pass ever performed on a given module. It is within one global +module tree pass. The termination check is then moved to the top-level +resolution proposition. This is the one that originally called into the +former with the crate root. But now it has been made into an "infinite" +loop. The termination condition stops being the number of use statements +pre- and post-pass. It becomes a full comparison of the module tree +before and after the global resolution pass. This should reveal whether +some module has had its list of items modified. The trick here would be +to perform deduplication everytime a single-module pass is done. This +would ensure one can merge one use statement over and over again without +tricking this check into believing there have been changes. The details +of this are still fuzzy. Further specification is required. Consider the +prior case of the crate root again. The use statement is resolved. +Assume the crate root has only one child module. Assume as well this +child module is a leaf module. Suppose this child module contains a use +statement pointing to the crate root. The module resolves this use +statement by adding all items from the root module. That will merge anew +its own items. But deduplication will remove them during merging. The +module-wide pass system will then check for changes in the tree. It will +see there were changes. So it will trigger another pass. This will +repeat the same steps. But the use statement in the crate root will +already cause deduplication to remove both the child module and its own +items from the child module. The same thing will happen in the child +module. This seems trivial. But it may very well be the solution. More +thought has to be put into test cases. There are likely edge cases that +this does not consider. The current data structures are a potentially +subpar fit for the task. That should be the task of benchmarks. A recap +of the plan is in order. The idea is to perform module-wide passes. +These happen at the top-level resolution proposition. The termination +check should also be moved there. The check should not base itself off +of a diff of use statements. The check is over the entire tree. This +implies need for a copy of the module tree. That would have happened +anyway. That is because of the pure nature of the computations. +Discussion about other areas is in order. The single-module resolution +proposition should change. It should now be simpler. The pass system +does not exist anymore there. A single resolution should be made. Though +this should continue to be followed up by use of the merge proposition. +The other areas of the current revision can be left as-is. Path +manipulation will come in very handy. The current merging strategy is +key to the new algorithm. That should likely be left unchanged. A more +complex test case must be thought up. The test cases should change. +They've all had the same shape thus far. Consider the case of an +absolute use statement. Suppose it is found in some arbitrarily deep +module. Then suppose every other module has a reexport to it. The first +pass would import the module over to all other modules. This pass would +also resolve the absolute use statement. Some modules would already +observe this. These would be the descendants to the former. A second +pass would then be performed. There have been changes in the first pass. +That is why a second pass would be in order. The second pass would go +through the same use statements anew. This is where deduplication comes +in. The same use statements would be resolved anew. Deduplication would +remove duplicate items. The import to the denoted module would be kept. +These are key. An observation can be made about this algorithm. The +amount of work per module-wide pass is fixed in some ways. The number of +use statements considered is always the same. Dependent use statements +will not be resolved. They will once their dependency exists no more. +That is the one way in which work varies across passes. This can be +modeled in terms of a set of sentences. These sentences await +satisfiability. Satisfiability of some sentence depends on +satisfiability of some other sentence. Dependent sentences are defined +as such. At least one sentence must be dependency-free. This triggers +resolution of other sentences. There is an exception here. Some +sentences are unresolvable. And yet they appear as dependent. These +correspond with use statements from third-party crates. The algorithm +relies not on these. Unresolvable sentences are eventually skipped. The +algorithm has no notion of unresolvable sentences. It only ever +determines dependency of sentences. It does not determine which are the +dependencies. A dependent use statement can not be immediately resolved. +Some name in its path does not currently exist. At one point it may +exist. It is unknown whether that point is reachable. The diff +comparison determines if it may be reachable. An observable change in +the tree indicates further resolution is in order. No observable change +indicates no further resolution is possible. The diff thus determines if +some dependent sentence is satisfiable. It does not determine which +sentence is satisfiable. The algorithm always goes through all +sentences. It attempts to satisfy each sentence in turn. Some may yield +observable change. Some may not. Termination follows when there is no +observable change at all. Discussion is needed of the new machinery this +algorithm needs. Resolution of bidirectional use statements needs access +to the entire module tree. The stateful module in the +reexport-resolution proposition is not enough. There is need for the +entire module tree. This should reflect the most up-to-date snapshot. +That can be found after each single-module resolution. This was already +explored in the latest revision of the Idris proof. The depth-first +approach in the single-module resolution proposition must be kept. There +is now need for this to carry state. The state should consist of the +module tree snapshot. The seed should be the same as the module to +resolve. This means the initial module is the crate root. So should the +stateful module tree be. Then the single-module resolution proposition +can use this alongside the single-reexport resolution proposition. The +latter will need to add another piece of state. The state is now +composed of the parent module and the entire module tree. Note the +module tree state need not change across each reexport resolution. It +changes once the merging proposition is triggered. That will need +another auxiliary proposition. It should observe the changes in a given +subtree. Then it should effect those in a new crate module tree. The +details of the single-reexport resolution proof need discussion. This +may now support path keywords. These include: crate, super and self. A +review of the Rust reference may be in order. These are the only +keywords that must be considered. The syn use tree does not include +special support for any of these. That likely means they are to be used +as identifiers. The syn documentation on identifiers mentions that. +Keywords will be parsed as identifiers. A manually-constructed +identifier may not be a keyword. The logic is then simplified. The +basecase for names needs modification. It should check whether the +identifier refers to a keyword. This also affects the rename case. The +check will need to consult the syn documentation. Maybe there is already +a utility proposition for keywords. There is none. The string conversion +utility will be used instead. This should yield reasonable results. The +case is of a single identifier. There should be no need for +sanitization. This is a good opportunity for implementation. A change to +the above cases will not affect the prior revision of the algorithm. The +self keyword requires no module tree state. It may be readily +implemented. The current proof takes into account solely a lookup into +the container. The revised version should check for keywords. The only +feasible keyword to implement at present is the self keyword. This +should come before the lookup in the item container. Recall the current +stateful module corresponds exactly to the self module. A resolution +follows immediately if the self module is brought over. The question +becomes the format of the item container packed in the resolution item. +Should it be a virutal container? Yes. self imports no immediate set of +items. It only brings over a module. The simplest next keyword to parse +is the crate keyword. That keyword is meant to be the source of an +absolute import path. Is that supposed to mean anything to the +algorithm? Yes. The place where it is checked can not be any place. self +also can not be checked only in the name case. It also has to be checked +in the rename case. The case for paths also needs to consider it. That +makes it notable in all cases. Sanitification is not necessary. +Well-formed input is expected. The case for renames need some more +attention. The current logic for renames could be reused. It is the same +for both the lookup and the self case. The lookup first finds a module. +Then it decides to modify the path of the module. Then it manipulates +the paths of all items in the subtree rooted at that module. The idea is +the same. It may be feasible to have a closure do this. It would get +passed an arbitrary module to rename. The case of a path requires +further discussion. The self keyword may appear at the start of the +path. It may also appear at the end of a path. The latter matches +against the name basecase during reexport resolution. Well-formed input +does not contain a self module in the middle of a path. Or does it? It +does not. A diagnostic reports just this. The self keyword may only +appear as a starting or ending segment in a path. Well-formed input is +guaranteed here. That should precede the lookups. The self keyword +indicates to carry through with the same stateful module. This will not +use the utility macro. That questions the approach used in that macro. +Though one thing is clear. The lookup should go after the self check. +The new state is now conditionally determined. It can be the same old +state if the path head is the self keyword. It can be the result of the +lookup if the path head is not the self keyword. Discussion is merited +to support other keywords. That check in paths will eventually consider +also the super and crate keywords. The check will have to be expanded. +Though the same strategy should be feasible. The new state is +conditionally defined. It is the current module in the self case. It is +the parent module in the super case. It is the module tree in the crate +case. This makes it obvious there is need for the module tree as state. +The crate case would make both pieces of state become the module tree. +The next step seems to be to add support for the new state. This now +comprises the crate-wide module tree. The first thing that needs +modification is the top-level loop. This should be removed from the +single-module resolution proof. Though there is no obvious proposition +for global resolution. It seems like the place to put this is the +parsing entrypoint for the item container. That would be subpar. A +better approach may be to abstract that away into another proposition. +That proposition would perform the global passes. That should leave the +parsing entry point unmodified. The global resolution proposition should +never be called recursively. That should ensure one can reason about the +crate module as the only consumer of the proposition. The proof should +call into the proposition that performs one pass over the entire module +tree. The implication of that is another module. This is guaranteed to +be the post-pass state of the crate root. Then the diffing step should +come. This is a diff between the pre-pass and post-pass states of the +whole module tree. This comparison proposition needs discussion. It +should compare the shape of the tree. Maybe an automatically derived +PartialEq implementation will do. It will not. That would also take into +consideration the current_module field used at parse-time. Something +similar would be convenient. The proposition for equality would then +"write itself." A comparison of aliases would do. Then would follow a +comparison of records. And then a comparison of all other fields. Except +for the parse-time utility field. That should do the trick. This assumes +there are PartialEq implementations for the types used for items. That +does not necessarily hold. Though those should be feasible. Those +implementations seem done now. A comparison of a tree should trigger a +recursive comparison of the entire tree. Though note this is exposed as +a separate proposition. The driver seems to be now ready. The next thing +should be to change the state passed to the single-pass proposition. +This needs to now also pass the module tree in whole. That should be +seeded with the same value as is currently used for the crate root. This +awards a dedicated proposition. It should gather within it the state as +a subtree and the whole-crate module tree. This should also make up the +implication of a single resolution pass. Then it follows that there +exists a new state after each child module is processed. This calls for +a fold instead of a map. The fold implies a product of identity +functors. The pair is comprised of an updated state and a resolved list +of child modules. The updated state itself reflects the state carried +across passes. The resolved list represents the previously mapped list +of child modules. There is something unnecessary here. There is no need +to have a duplicate list of updated child modules. It is enough that the +stateful pair carries the subtree and the state. Except it is not. The +subtree in the stateful pair refers not to the module from which the +child modules spawned. There is need to reconstruct a new module. This +should hold the resolved child modules. Then comes resolution of the +module itself. This should hold this newly constructed module. The other +half of the pair should hold the stateful module tree from the fold over +the list of child modules. The proposition for single-module resolution +should now take a stateful pair. The list of resolutions need not carry +with it an updated whole-crate state. That changes only after a pass +over a given module is done. Discussion of the single-reexport +resolution proposition will be delayed for later. The next step follows +back in the single-pass proposition. The proof now has a set of +resolution items. It needs to merge those with the module currently at +hand. Then it needs to deduplicate the items. The last thing is to +update the stateful module tree. Merging already has a dedicated +proposition. Deduplication also has a dedicated proposition. Whole-crate +module tree updates do not. That will require more work than the +equality proposition. The goal is to find a subtree in the crate tree. +Recursive updates are not necessary. The subtree need only update the +data of the subtree's root. That will be justified now. The simplest +case is a leaf node. These are resolved first. That is thanks to the +depth-first approach taken in the single-pass proposition. The +whole-crate module tree is updated here as well. The update would find +the leaf module. Then it would update its items. The leaf has no child +modules. So the only updated items are non-modules. The whole-crate +module now contains an updated leaf. Say it now goes to this leaf's +parent module. Suppose this parent module contains one other child +module. Further suppose that this other child is also a leaf. The +whole-crate module tree would be updated anew. Then it would reflect +both resolved child modules. Then comes the time to resolve the parent +module. One could attempt to resolve it and its children. There is no +need for the latter. There may be one exception to this. The module may +now have new child modules. That does not change one basic fact. The +module ought be found in the whole-crate tree. Then it needs to have its +items replaced with the updated module's items. That should get the job +done every time. A recap is in order. The proposition should assume one +module. Then it should assume another module. One is the crate root. The +other is the subtree. The subtree could also be the crate root. It is +best if a pass pair is passed instead. Though it should not imply a pass +pair. It should only imply a module. That should be the updated +whole-crate module tree. There is a problem in this approach. The +proposition will be used inductively in its proof. That means a pass +pair would be misleading. One of the projections refers to the +whole-crate tree. The other does not. The proposition will be used with +some other module in the whole-crate tree. The proof is not entirely +clear. The destination module's path has to be checked with the source +module's path. A match means the latter's proof must be implied. A +mismatch means the former's children must be matched. This should be +done through a fold transform. Recall only one module matches. The fold +should be prone to termination. What should the fold's seed be? It is +known the source module will surely be found. That is not a guarantee +the type system understands. A dummy value should do. The handiest one +is the source module. This is incorrect. Recall the proposition performs +walks on the module tree. One such walk could hit a dead-end. Such a +dead-end would manifest itself in a leaf module. That should imply a +coproduct. Its injections should correspond with the identity functor +and an injective const functor. That should be the seed of the fold over +the list of child modules. This means the whole proposition should imply +the above coproduct. The proof is wrong. One should not fold over the +list of child modules. That would mean some modules are downright lost. +The simplest example is a one-level deep module tree. Suppose there are +an arbitrary number n of nodes. Assume n is greater than 1. Further +suppose that the update subtree is on of these child modules. The +current proof would discard both the root module and the sibling module. +The proof should map over the list of child modules. Then it should +imply the mismatched module with the updated children. One of those +children modules will have matched the source module. That should do it +for the single-pass proposition. The next thing should be the +single-reexport proposition. Each case will be discussed in turn. The +name basecase can only ever refer to self and super. The self case is +covered. A similar macro could be elaborated for the super case. Is +super really possible in the name basecase? It can not. Can the crate +keyword appear alone? Not the case. The only one that can appear as a +basecase is the self keyword. The crate keyword can only appear at the +start of a path. The super keyword can only appear in the rename case. +It is otherwise a segment head in a larger path. The rename case for a +super keyword should access the module right above the current one. That +means there is need for an auxiliary proposition. That proposition +should assume the whole-crate module tree. It should also assume a path. +It should then look for that path in the module tree. Then there should +be need for another auxiliary proposition. This should get the parent +path to some module. It should assume a module subtree. It should imply +a path. That will then be used in the assumption for the prior +proposition. The proof for the parent path is fairly simple. It should +take a certain number of segments from the assumed module's path. The +number should correspond with one fewer segment than the path contains. +The proposition to fetch a given module should first match on the path +of the subtree with the target path. A mismatch indicates it should fold +over the module's child paths. The fold should be prone to termination. +There is need for a seed value. One knows the module can be found +somewhere in the crate graph. That is the main use of the proposition. +This could be generalized into implying a coproduct. One injection would +correspond with an injective const functor. The other would correspond +with the identity functor. The assumption that the module really does +exist can be left to call-site. The proof for all propositions in the +revised algorithm seems complete. To humanly verify everything in it, +let use review the workings on the revised algorithm. The algorithm is +based off of the idea that repeated resolution of the module tree will +eventually converge to a snapshot of the tree that is unchanged after +performing another resolution pass. To that extent, the algorithm driver +consists solely of a use of the resolution proposition, and a recursive +comparison of the resulting tree with the pre-pass tree. If such +comparison happens to imply boolean truth, then one may safely assume +that another pass will not reveal further information. Termination thus +follows. Now onto the resolution proposition. The proof for this one is +ever so slightly more intricate; It really only has more steps. The +assumptions consist solely of a pair of a module subtree and of the +whole-crate module tree. The former is the one from which use +declarations are meant to be sourced from. The latter serves in both +potentially resolving some of those use declarations, and in updating +itself with the result of those resolutions. Such update only takes +place after the target module is itself updated. This one's update is +the merging proposition. That takes in the module where the use +declarations were sourced from, and the potentially empty set of +resolution items yield from the single-module resolution proposition. +Let that on hold for a moment. The proof for the pass proposition has +not been explained in full. This proposition will first recurse through +all child modules of the module being currently processed. That should +not be necessary to do first, but it is the more intricate part of the +proposition's proof, so we're best off getting it out of the way sooner +rather than later. The nifty part about this is that it is not enough to +simply have each of the child modules recursively resolved; Recall each +call provides proof of the stateful pair of a subtree and the +whole-crate module tree. In other words, there is need for a fold over +the list of child modules, such that the whole-crate part of the +stateful pair is aware of the result of resolving each of the child +modules. This is because, as mentioned above, each resolution pass ends +with an "updating" of the whole-crate element of the stateful pair +(though still a pure computation) carried across recursive calls to the +resolution proposition. This fold is seeded by an unlabeled product +comprising the stateful pair and the list of resolved child modules, +which starts empty. For each child module, it first calls into the same +resolution proposition as we are in right now. The input pair here is +the whole-crate module tree as received from the fold and the child +module in question, also as received from the fold. The list of child +modules is then updated with the resolved subtree implied from that use +of the single-pass proposition. The updated pair in the fold is then +comprised of this updated subtree (the child module), and of the +whole-crate module tree that was implied in the above use of the +single-pass resolution proposition. And that just about makes the more +complex part of the single-pass resolution proposition's proof. The next +steps just build up a new module with the updated child modules, and +feed that into the proposition that actually attempts to resolve the +list of use declarations in a given module, alongside the snapshot of +the whole-crate module tree obtained from the above fold. The +proposition that attempts to resolve the use declarations in a given +module is fairly simple in that it simply maps over all use declarations +in the passed subtree. The map transform is the single-use declaration +resolution proposition, which we shall explain later on. Now back to the +single-pass resolution proposition's proof. Once the list of use +declarations has yield a potentially non-empty list of resolution items, +these are to be merged back into the module where the use declarations +were sourced from. That is handed off to the merging proposition. This +proposition goes through a two-stage pipeline; It first gets rid of the +unresolved use declarations in the pass, and then goes through each of +the resolved ones to extract the set of items that the use declaration +referred to. These then get their paths modified such that they are +correctly integrated into the module into which to merge, and then the +resulting lists get appended to the existing lists of items in the +module into which to merge. That merits some further discussion, but +that will be left for later. Now back again to the single-pass +resolution proposition. We now have an updated subtree, which makes up +the first half of the stateful pair used across passes, and the only +thing left is to update the whole-crate module tree. To achieve that, we +delegate to another proposition that attempts to find some module a into +another module b, and, if found, integrates module a into module b's +subtree. Module a could very well be module b itself, in which case the +proposition would imply module a anew. Otherwise, if module a is within +some arbitrarily deep layer of the subtree rooted at module b, that one +subtree will be found and replaced with module a (which itself is a +subtree, of course.) The tree-update proposition's proof is fairly +simple in nature. It checks if the path of module b matches that of +module a and implies the latter if such is the case; Otherwise, it +attempts to map over the list of child modules in module b, where the +map transform is the same tree-update proposition. That either yields +the module untouched (if its path did not match that of module a,) or +otherwise yields module a. This is actually quite nice in that it will +halt the search+replace as soon as the module in question is found. +Granted, the whole list of child items will be processed, but that does +not seem like a huge deal. There's likely more pressing performance +bottlenecks. The updated whole-tree module tree, back up at the +single-pass resolution proposition, is now ready, and with that, the +proof is done. The implied stateful pair is comprised of the latest +snapshot of the subtree being resolved, and of a snapshot of the +whole-crate module tree that is aware of this updated subtree (for it +has just been integrated into it.) The only core proposition that +remains to be reviewed is the single-use declaration proposition. The +proof for this one is, at least at a high-level, simple to grasp. It +recursively munches through the use declaration's path. The ever so +slightly more complex parts of it are concerned with handling cases such +as a use declaration that introduces renamed items. Then there's also +the three keywords Rust accepts in paths, which either (1) introduce an +item by its crate-relative (absolute) path, or (2) introduce some +ancestor module through a super-relative path, or (3) introduce some +descendant module, if not themselves, through a self-relative path. +Those will all be explained shortly. We shall start by deconstructing +the proof for each of the cases that a use declaration may represent. A +use declaration could be a single identifier. That identifier, as parsed +by syn, could be one of the super, self or crate keywords, or could +refer to some genuine identifier that was not a Rust keyword. +Thankfully, syn parsing takes place after macro expansion, which itself +takes place after a cargo-check run (or it should, in principle,) and +use-declaration resolution takes place after syn parsing. This means we +can trust all input is well-formed input; That also means we can discard +the possibility for a single identifier use declaration to refer to the +crate keyword. No matter whether the use declaration really was made out +of a single segment, or if we got to this basecase after munching +through some larger (multi-segment) use declaration, the crate keyword +simply cannot appear at the tail end of a use declaration. The crate +keyword, as per the Rust Reference, may only appear as part of the start +of a use declaration's path. The path may not necessarily be a +multi-path segment referring to some other item, so long as the use +declaration introduces a renamed identifier for the crate keyword. That +means the crate keyword can only logically appear as either the head of +a longer path, or as a part of a renamed use declaration. That only +leaves the self and super keywords for consideration. These need some +further explanation, as the muncher proof must be ready for technically +malformed input. According to the Rust Reference, the self keyword in a +use declaration may only appear at the start or end of a use declaration +not containing any groups. The latter case, much like with the crate +keyword, requires the use declaration to introduce a renamed identifier +for the self (current) module. The nifty part about this is that these +rules apply to use declarations that do not contain a group at its tail +end. If a use declaration does contain a group, the self keyword may +appear bare (without renames) at some top-level, nested use declaration +introduced in the group. The ctest-specific parsing logic performs +"normalization" of paths in use declarations, such that if they contain +some group, this group is deconstructed into a set of use declarations. +This, in turn, means that we could very well stumble upon some use +declaration with a trailing self keyword that is not part of a nested +use declaration (of course, within a group.) Long story short; We need +to handle both the self case and the super case in the name basecase of +the single-use resolution proposition even though these would get +considered malformed input by the compiler. Let's get going then. Let us +first discuss which of the keywords or the case of a regular identifier +should come first. The former is fairly simple to check for as the +Display implementation of syn's Ident lets you pry into the "raw" token +that was parsed. That only leaves the latter case, which happens to also +be fairly straightforward. A use declaration refers to some name in the +module tree, and for all we know in this instance, that name must be +found in the current subtree of the stateful pair we keep across passes. +If the original use declaration was actually made out of a larger +fraction of path segments, it is not our responsibility (nor do we even +have the information) to track whether the stateful subtree is, indeed, +the module where we'll find the identifier in question. We decide to +handle first the case of keywords, though this would not matter much; +Rust raw identifiers do not allow any one of the above three keywords to +be used for items. The case of the self keyword means the current +stateful subtree ought be implied as-is. The case of super implies a +lookup of the parent module is in order. That is where two other +anciliary propositions come in; One to determine the parent path to some +module, and another one to search for a module path provided a module +subtree. The former's proof is fairly straightforward; A module path (or +any item's path, for that matter) is made out of segments, the last of +which is the module's identifier. It thus holds that the parent path is +the path resulting of destructuring those segments and discarding the +last one. Functional programming languages have an operation exactly for +this, but we have to make do with an iterator that takes the first n - 1 +segments of the path, where n is the number of segments in the path. Of +note is the fact that the proposition will yield a path with no segments +if the assumed module happens to live in the crate root itself. That is +because absolute paths, which are the ones stored in each parsed item, +are crate-unprepended. This means all items that appear in the crate +root have single-segment absolute paths, instead of two-segment paths +with crate prepended to them. This has thus far not caused any issues, +so it has been left untouched from initial work on the whole ctest +extension (back when the parsing facilities were extended.) The implied +path of this proposition then becomes the first assumption of the +module-fetching proposition. Its second assumption is the subtree in +which to search for that path. The proof is, again, fairly simple; The +path is matched against the passed subtree. If it matches, then we have +an answer. Otherwise, we lazily seek through the list of child modules +in search of the target path by inductively using the same proposition. +Much like the tree-update proposition's proof, whole-crate traversal +will only take place if either (1) the target path happens to match the +very last module in a left-to-right DFS, or (2) the target path matches +that of no module in the assumed subtree. This latter case merits some +further discussion. We have a notion that this proposition only ever +gets used by providing as subtree in the very first call the whole-crate +module tree. The search performed in the proof could end up yielding an +const functor injective to the unit, which would mean the search failed +to find a module with that path. We could unwrap the coproduct holding +the injection with the above const functor, but that's just assuming +risks in the proposition's proof that are best left to call-site. So +this proposition actually implies a coproduct with two injections; An +identity functor, and an injective const functor. Now back to the +single-use resolution proposition. This had been left in the super case, +where we had been getting ready a lookup of the parent module to the +current stateful subtree. That used the two propositions just reviewed +to first fetch the parent path of the current stateful module, such that +it could attempt to find that path in the whole-crate module tree. That +just about makes up the keywords subcase in the name basecase of the +overarching proposition. The next subcase is that of regular +identifiers; These need to be looked up in the stateful subtree. But +that's about it. The lookup uses the search proposition associated to an +item container, which itself is fairly straightforward. It first gathers +into a single container all items that are not use declarations, to then +sort that container, perform a binary search on it, and then index the +container with all gathered items. This smells like a performance +bottleneck. It may be a good idea to just perform a lazy linear-cost +traversal of the container with all gathered items, as we end up +incurring unnecessary costs for sorting and searching. That seems to get +the job done just fine. It is also more obvious what the proof is doing +at first glance. Now back at the single-use resolution proposition. We +had left this one at the point where the matched identifier was being +looked up as an item in the stateful subtree. If no item's identifier +matched the passed identifier, then we have no further way of attempting +to resolve the use declaration; It's unresolved. Otherwise, the matched +item (resulting from the look up) is packed in an item container. It may +now be a good idea to make a quick review of the basecases considered in +the single-use resolution proposition's proof. There are a total of +three; One is the single-name basecase we just explained above, another +one is the glob basecase, and the other one is the rename basecase. The +only recursive path through the proof is the path case, where the tail +end of the path makes up another use declaration to munch through. The +glob basecase is the simplest of them all; The item container within the +stateful subtree ought be returned as-is. Note that the item container +packed with resolved resolution items will be deconstructed in the +merging proposition's proof, and each of its fields will extend the same +field in the item container of the module into which to merge. A glob +import brings into scope all items from the module making up the head +segment of a path, so there is need to return the items of the stateful +subtree and not an item container wrapping it. This contrasts with a +path that refers to a specific module or set of modules, which must +itself be wrapped in an item container, such that items of the module(s) +themselves do not get merged, but rather the module in its entirety. The +next and last basecase in the proof is the rename case. This is +essentially the same as the name case, except we have to rewrite a bunch +of item paths. Due to their similarity, we shall skip over the entire +details of the proof, and only discuss the details of renaming. These +are fairly simple to explain for non-module items, but will require +introducing an anciliary proposition for module items. The former only +require replacing the last segment of their paths with the renamed +identifier. Modules, on the other hand, need to take into consideration +the subtree rooted in them. A module's path dictates the path of both +its items and the child modules contained within it. The child modules +and its item are also affected by the path of the parent module. This +goes on and on until we hit the leaf modules. Renaming a module thus +requires both changing its path and changing the path of all items in +the subtree rooted at that module. This is also used when rewriting item +paths in the merging proposition's proof, so it was thought best to +abstract the whole thing into a path manipulation proposition. This is +the same as that devised for the first revision of the algorithm, so we +won't go into the details of the proposition's proof. It's best if we +focus on the closure that is passed to the proposition in the rename +case of the single-use resolution proposition. The first cases to +consider are the keywords. In this instance, it is completely legal to +find any one of the three keywords that are expected in use declartion +paths. The treatment itself is the same as that in the regular name +basecase of the proposition. The only difference is in the fact each of +the implied modules (the current stateful subtree, or the parent to the +statful subtree, or the crate-root module) require having their paths +and the paths of all items in the tree rooted at them changed. The +renaming consists of first renaming the module in question by changing +the last segment of its path, and then using the path manipulation +proposition mentioned above. The closure passed to the path manipulation +proposition discards as many segments off of of the item's path as the +new length of the renamed tree root contains, and yields the rest. These +are then prepended the new root module's path. An example is the path +std::any getting renamed to std::dynamic. Items such as std::any::TypeId +discard as many segments from their left as the length of the tree root +(here std::any) contains. That leaves std::any::TypeId's path with a +single segment, TypeId. This segment then gets prepended the renamed +path of the tree root, such that it ends up as std::dynamic::TypeId. Of +note is that the path manipulation proposition will apply the transform +to both the items in the given module tree, as well as the module tree +root itself. The above strategy still works when applied to the tree +root, because all segments of its path will be discarded, and the new +path would be prepended to an empty path, yielding the new path anew. +This works thanks to the fact path segments make up a semigroup and a +monoid functor. The neutral element is a list of empty segments, and the +append operation is intuitive enough. The details of the implementation +have been changed to reflect a less verbose approach that does not +repeat the same path manipulation logic for the top-level module. This +applies in much the same way to other non-module items, which implies +one need not repeat the same logic for both non-modules and modules. +This now lives as a macro within the scope of the renamed case of the +single-use resolution proposition. With that, a review of this case +seems done, and the only case left is the inductive path case. This one +behaves in much the same way as one would expect from a functional +approach to a right fold proof. The head of the path is extracted, and +first matched against the set of keywords we conceive possible in use +declarations. Each of these will imply a new stateful subtree that will +be paired up anew with the stateful whole-crate module tree when +inductively using the use-declaration proposition with the tail end of +the path. The self case is, again, fairly straightforward; The same +stateful subtree as carried thus far is implied as the stateful subtree +for the rest of the path. The super case requires performing the same +parent module look up as explained in prior cases of the single-use +resolution proposition. The crate case simply yields the stateful +whole-crate module tree, which makes the next use of this proposition +have as stateful pair the same elements (for the subtree is now +structurally equal to the whole-crate module tree.) The subcase for +items is, again, much the same as in other cases of the single-use +resolution proposition. A look up is performed on the stateful subtree +and only if a module is found does use-declaration munching continue. +The case of any other item is made to crash the whole thing because that +is simply impossible; Except this could potentially be incorrect. ctest +parses all use declarations in a crate. This brings with it both +resolvable and unresolvable use declarations. Thus far, it had been +assumed that the latter could only consist of use declarations that +refer to items from third-party crates. But this may not be entirely +correct. Users have the entire, unrestricted power of the Rust module +system in their hands. They could very well have use declarations that +at first seem like they are resolvable, for they refer to an item we can +find in the crate root, but that refer to a glob import of the variants +of an enumeration. The unconceivable subsubcase explained above would +make the whole thing crash instead of implying an unresolved path. On +second thought, there is no way the look up yields an item and the item +turns out to be an enumeration because enumerations don't even get +parsed in the first place. This implies there is just no way an item +gets returned from the path lookup without it being a module. Still, +it's best if we imply an unresolved resolution instead of crashing, +considering the effects of such implication simply mean no other item +will get merged into the module where the use declarations are currently +being sourced from. That should do it for the single-use resolution +proposition, which means the review is technically done. A larger +surface area of the code has been made pure, which should hopefully +avoid issues unrelated to the core logic. Before testing things out, it +would be best to attempt to reason through the high level idea of the +algorithm. We're trying to resolve use declarations, which here means +that as many use declarations as possible should be resolvable. To that +extent, the algorith performs crate-wide resolution passes until the +state of the post-pass crate module tree is structurally equal to that +of the pre-pass state. There is no difference between procedures across +passes, and unlike the first revision of the algorithm, the logic +depends not on the removal of use declarations after each pass. Each +pass first bottoms out on the module tree, and actually starts resolving +use declarations from the leaf modules up. After a complete pass over +the use declrations of a given module are finished, the resolution items +are merged into that source module. Then this module is integrated into +the crate root that is used to perform resolution of absolute paths and +super-relative paths during use declaration resolution. This repeats +itself with all modules in the crate's module tree, each one receving an +updated snapshot of the whole-crate module tree, which is used when +attempting to resolve its own set of use declarations. This idea works +because no matter how may times a single use declaration is resolved, +the set of items that are effectively merged into the module where the +use declaration was sourced from always has its list of items +deduplicated by their paths. Path rewriting happens during merging, +right before the items become part of the module into which to merge. It +seems like the logic is solid. Time to test things out. It would seem +the latest test worked out just fine. Now onto testing out the new +functionality. Resolution enters an infinite loop when testing with a +super-relative use declaration. The test ought be first minimized to +something that really only causes the loop. So the issue does not happen +in a very simplified case. That means there must be some edge case that +is causing non-termination of the iterations over the resolution pass. +The failing test had multiple cycles in the graph where use declarations +are edges, modules are nodes and the direction of an edge is given by +the module where the referred-to item(s) are. I think I can start +guessing why the test fails, and it may very well be due to cyclic use +declarations. Suppose we have a contrived example of a crate root with +one child module, where the child module contains a super-relative glob +import, and the crate root contains an glob import of the child module's +items. As per the current algorithm, the child module will be resolved +first. The use declaration will be resolvable, and all items in the +crate root will be imported over to the child module. This means the +child module will contain itself. Then comes the root module. The one +and only use declaration within it is also going to be resolved quite +easily. After one pass, diffing notices there has been progress and so +it decides to perform another pass. The issue comes from the fact we +just imported into the list of child modules of both the crate root and +the leaf module the dual module with its list of use declarations. In +the next resolution pass, there will be more use declarations to resolve +than there were before, including twice as many as there were before. +The invariant about the number of use declarations being fixed is thus +broken. This also hints at the fact deduplication is not good enough. +Two items with the same paths getting removed at random does not satisfy +the needs of the algorithm. For non-module items, this gets the job done +just fine. For modules, though, it becomes a necessity to keep, among +duplicate modules, the one module that currently has the largest subtree +rooted at it. This should hold because use declaration resolution always +expands or leaves unmodified a module tree. This discussion has thus +revealed that there is need to modify two things; The handling of merged +modules, and the deduplication proposition. During mering, the subtrees +rooted at the modules to merge should be stripped of all use +declarations. That is likely to require an anciliary proposition. The +removal of use declarations at merge-time is done. It happens right +after the modules to merge get their paths updated, and right before +they are about to become part of the module into which to merge. Now +comes deduplication, where modules need special attention. Sorting them +is likely going to be necessary, so that is going to be left in. Though +sorting based on the path will not be enough. The path should continue +being the primary way of sorting, but upon hitting some two modules +whose paths turn out to compare equal, we should further compare the +sizes of the subtrees, and invert the resulting ordering. This is so +that the largest subtree appears first, and we can use the same +comparison closure for the deduplication proposition (which keeps only +the first item of a run of consecutive items.) The logic seems done but +it may be potentially faulty. The comparison logic first considers the +paths of the modules, after which it recursively compares the subtrees +rooted at them. The issue may come from the fact that subtree recursion +calls back again into the proposition for the top-level modules. This +means that if some two modules have the same path, and yet +post-resolution one of them got a new child module, but this one's path +happened to compare lexicographically smaller than some other +pre-existing module, then the module we'd stick with first would end up +being the older snapshot. Something else that doesn't quite strike me as +correct is the comparison used to determine equality among modules +during deduplication. This can not be the same as the one used when +sorting the list of modules, because we need all modules whose paths +compare equal to be considered for removal; Only the first one among +such set, which is the first thanks to the other comparison logic, will +remain. One place where there could be issues concerning use +declarations is the single-use resolution proposition. The case for glob +imports brings over everything from the target module, but this should, +in theory, not cause any issues. Even if the resolution item packs use +declarations, these are ignored during merging so it should be safe to +leave the logic as-is. It may be worth to add a comment noting that to +the single-use resolution proposition. But before that, testing is in +order. The same reduced failing test should be run anew. It fails again. +Something is causing it to enter an infinite loop. Debugging this would +be great. Maybe there's some information out there on rust-lldb. That +seems like a timesink, so it would be best if we first tried to think +about the problem some more. Debugging will not work without spending a +significant amount of time looking into it. Best to try and reason +through the program logic. The use declaration in the leaf module will +be resolved first, and that will import over all items from the +whole-crate module tree. The merging would yield a child module that +contains a child module of its own referring to itself. The difference +here is that this nested self should not contain use statements. This +hints at the fact, which has just been confirmed, that the failing test +can be further reduced not just to the case of a set of cyclic use +declarations, but to a single super-relative use declaration. The +current failing test has a single module beyond the crate root, where +the former contains a use declaration that glob imports the crate root's +items. The correct course here would be for the leaf module to import +itself and remove its one use declaration before moving on. Then this +would be integrated into the whole-crate module tree and resolution +would move on to the crate root. At this point, resolution should find +there is nothing to do in the crate root. That should finish the first +pass. The diffing should have decided that another pass is in order, as +it should have noticed that there is a subtree size mismatch between the +pre-pass and post-pass states. The infinite loop did not take place in +the absence of cyclic use declarations before the deduplication routine +got tweaked, which likely hints at the fact the underlying issue may be +in the refactoring. A test to sort a bunch of modules should be devised. +This may reveal the infinite loop is taking place during sorting. The +test does not fail when basing one off of purely item paths. That hints +at the fact the issue may be in the secondary sorting condition. Sorting +by the subtree structure also seems to work. That does imply the issue +may not be in the sorting routine at all, though maybe these tests are +too contrived compared with the scenario faced in the test. It could +also be that the comparison function does not implement a total order, +and that causes issues when comparing sizes. One alternative to +performing module deduplication this way is to, during merging, attempt +to find all modules about to be merged in the module into which to +merge, then remove those off of the latter, and then merge the modules +packed as part of the resolution item. That would avoid the need for +deduplicating modules after merging. Either way, it seems best not to +get rid of the current infrastructure until it is proven incorrect. The +first thing to attempt should be a test of the module merging +proposition, which may be the source of issues as it is the one place +where use declarations are removed. The test should exercise two things; +For one, the module into which to merge should end up with duplicate +modules. But the use declarations of the modules to merge should not be +brought over to the former. It would seem use declarations do get +properly cleaned up. Control flow isn't even entering the path case of +the use declaration, which may prompt at the fact the issue is way +before that. It would seem the issue is in the fact stdout is not being +emitted by the test harness. There's probably some option to fix this. +The whole thing works only in the child module, and at that point it +just keeps on going on an infinite loop. The issue has been identified. +The problem lies in the perspective the rest of the crate has of a given +subtree after a resolution pass. Suppose we have a one-level deep module +tree with two modules; A crate root and a leaf module. Assume the leaf +module contains a glob import of all items in the crate root. After the +first pass on the leaf module, the use declaration resolved to the leaf +module itself, which nests it in the module tree. At this point, the +module tree contains a two-level deep module hierarchy, with the crate +root, a node module and a leaf module. The leaf module contains no use +declarations; The crate root contains no use declarations and the node +module continues to have one use declaration. The current revision of +the algorithm will still resolve the use declaration, which this time +contains a snapshot of a one-level deep subtree rooted at the leaf +module. This happens to be "larger" as per our comparison proposition +than the 0-depth module tree containing solely the nested leaf module. +The natural consequence to this is that the node module keeps the larger +snapshot and the resolution driver determines it should go for another +pass. The next pass goes through the same motions; Namely, it resolves +the use declaration anew, and finds the "largest" subtree to keep in the +node module is the now three-level deep subtree rooted at it (which is +represented in the current snapshot of the whole-crate module tree.) +This keeps repeating endlessly because each time the driver diffs the +pre-pass and post-pass states, it notices there are changes, so it +decides to go for another round. The way the algorithm was initially +conceived failed to realize that each subsequent resolution pass to the +one where all use declarations converged will still differ from the last +resolution pass. In other words, the algorithm was assuming that each +new resolution would see the module tree's state as it was after the +pass where convergence took place. The issue with this assumption is +that one can not determine when is it that convergence took place. Let +us describe the ideal result in the above example; The glob import in +the leaf module should have been resolved as it currently does, but the +next resolution pass should have performed resolution with respect to +the original whole-crate module-tree, and not to the state of the +whole-crate module tree after the first resolution pass was done. The +solution could be in performing subsequent resolution passes with the +initial state of the crate's module tree. The issue then would be that +no resolution pass would provide updated information to subsequent +resolution passes. Even the first pass would be futile; In the above +example, the leaf module would get the items from the crate root, then +another pass would run after diffing with the pre-pass module tree with +the post-pass state. This second pass would attempt to resolve the same +use declaration, and this would actually turn out well. The initial +snapshot of the leaf module without itself nested would be removed +during deduplication, leaving the larger one-level deep node module in +its stead. For this to work out, the whole-crate module tree would have +to be updated back at the resolution driver only to perform the diff. +And yet each subsequent resolution would be fed the initial whole-crate +module tree. Does even rustc allow this? It does. It also allows +arbitrarily nested modules within the module with the super-relative +glob import, and subsequent references to their items through the name +of the module (which is only made visible thanks to the import.) Maybe +resolution of super imports can be delayed? Nay, because a single +super-relative glob import is enough for non-termination. No matter the +state of the crate, super-relative imports that affect the list of child +modules in the source module seem to be the root cause of this issue. +This can likely also be extended to absolute imports that refer to some +ancestor module. Maybe the issue can be solved by identifying paths in +the module graph. Two examples come to mind that could be solved by this +strategy. Suppose the module graph consists of a two-level deep +hierarchy with a crate root, a node module and a leaf module. Assume the +crate root contains a glob import of the items in the leaf module. Then +assume the leaf module contains a glob import of the items in the node +module. If we think of the use declarations as directed edges, we can +draw a path from the crate root all the way up to the node module. +Resolution should thus take place by walking the reverse path; The +super-relative module should be resolved first, and only then should the +use declaration in the crate root be resolved. Let us explain the second +example; Suppose we are resolving the same module hierarchy as presented +above. This time, the crate root contains a glob import of the leaf +module's items, and the node module contains a glob import of the crate +root's items. Then again, we find that a path is drawn where, if walked +in reverse, the crate root import would be resolved first, and only then +would the super-relative import be resolved. Indeed, it seems use +declaration resolution reduces to the problem of finding connected +components and walking paths in reverse. But there's two things here +that aren't entirely clear; How would the ideal situation proceed when +posed with multiple possible paths in the walk, and how would cycles be +handled? Say in the second example above the crate root had another glob +import of the node module's items. This would mean there would be two +possible paths to be walked from the node module. Both would have their +first stop at the crate root, but then they could follow up on either +one of the leaf module or the node module. Let us assume cycles are +discarded upon detection. Then only one path could be walked. But should +cycles really be discarded? The idea currently seems to be as follows. + +* Build a directed graph where the nodes are the modules, and the edges + are the use declarations, such that the direction of the edges is + given by the module where the target item(s) of the use declaration + lives in. + +* Detect all connected components in the graph. This should be fairly + straightforward. + +* For each connected component, compute all paths and subsequently sort + the paths by whether they aren't cycles or not. The non-cyclic paths + should come first. + +* For each component, and for each computed path in the sorted list + obtained above, perform a reverse walk of the path, resolving the use + declarations in the order they are found in this reverse walk. Remove + any resolved use declaration. + +This leaves open the question about initially unresolvable use +declarations, which require information introduced by another use +declaration to even attempt resolution. It would seem the list of graph +paths would have to be updated each time a graph path were resolved in +full. To recap, the algorithm will need to, after parse-time is done and +before resolution starts, build a graph off of the use declarations and +modules. Because the nodes are the modules, and the use declarations are +the edges, we can likely base ourselves off of the same tree. But this +discussion is not one we should be having now. The priority is to think +through the handling of cyclic paths. How would ideal handling of this +look like? Let us consider a cyclic example. Suppose the crate root +contains both a child module and a glob import of the child's items. +Suppose as well that the child module contains a glob import of the +crate root's items. How would the ideal solution look like? The crate +root should end up with the child's items but not with itself nested. +This whole approach may not be the right way of thinking about it. +Apparently, Rust allows the same arbitrary nesting behavior that was +thought inadequate in the second revision of the algorithm. But it does +not loop infinitely when resolving. In the same example as presented +above, rustc will gladly allow the user to refer to some arbitrarily +self-nested child module, which is exactly the behavior of repeated +resolution in the second revision of the algorithm. For future +reference, this is illustrated in the following code snippet. + +```rust +mod foo { + pub struct Foo; + + mod bar { + use super::*; + + const FOO: bar::bar::bar::Foo = bar::bar::Foo; + } +} +``` + +The fact rustc does hit termination even though it shouldn't terminate +resolution is likely because they treat either one of paths or modules +as codata. In other words, to rustc, the subtree rooted at a given +module is possibly infinite. Of course, allowing codata requires knowing +you're handling codata, as otherwise you can kiss goodbye to +termination. This means the second revision of the algorithm is still +feasible, so long as these potentially infinite trees can be detected. +This prompts at the fact efforts should now focus on integrating another +algorithm for codata detection. Maybe we can skim the literature to try +and find out if there's something for this. But it's best if we give it +a shot ourselves first. The one point at which we know for sure that +termination took place is back at the resolution driver. This is because +before that, we only really know we are resolving "some" module in the +single-pass resolution proposition. Granted, one could compare the +subtree module tree with the whole crate module tree, and do whatever +then, but we do just this once the call unwinds back to the driver, so +it seem like a waste to not profit the comparison that already takes +place in the driver. Back at the resolution driver, the heuristic for +termination will have to be improved. An additional pass must be made +after each resolution pass that checks if some module contains itself. +If it does, then we know for sure that whatever is under the subtree +rooted at that module can be referred to through multiple paths that do +not officially appear anywhere. This does mean items will need a new +marker field that indicates whether their paths are propositionally +equal to paths with each segment before the last repeated an arbitrary +amount of times. But maybe this can generalized to any item. See, by the +time we get to parsing and resolving use declarations, we know for sure +that input is well-formed because it would not have passed cargo-check +otherwise. That does mean every path by which an item is referred to is +technically correct. Thus, any item that can be referred to by one of +these infinitely nested paths can be safely assumed to also be skippable +in ctest through one of these infinite paths. In other words, we can +allow all items to be referred to by these infinite paths, and just +about call it. Granted, if somebody used some such path for an item that +does not allow it in the skips of the crate being tested, ctest would +give it carte blanche. Maybe it's a good idea to ask Mr. Gross about +this. But it would be best to have a working implementation that +showcases the behavior first. To recap, the new pass that ought be added +to the resolution driver should follow these steps: + +1. Traverse the whole-crate module tree anew, checking among the list of + child modules to a given module whether there exists some module for + which there is another child module with the same path. + +1. If there is, flag the module so as to know that the next resolution + pass, once it has resolved the module in question, should not attempt + to replace the nested self with some newly resolved self. + +1. Imply the whole-crate module tree before the recursive comparison + with the pre-pass state. The diffing should happen after the nested + self has been flagged, as otherwise we risk non-termination. + +It's unclear how would the diffing step ignore the nested self module. +Is it supposed to be wary of the "flag" that these nested self modules +are marked with? Likely not. The idea is lacking in some respect; The +point of resolution is to eventually converge to a stable state where +resolving any use declaration anew would not produce changes on the +whole-crate module tree. What would the ideal case in a case such as the +currently failing test look like? Let us include the code snippet of the +test here. + +```rust +pub struct Foo; + +mod foo { + use super::*; +} +``` + +The tests fail at the point where the driver diffs the pre-pass state +corresponding to the first pass, with the post-pass state after the +second pass. The former contains the crate root, the foo module and a +the foo module nested anew within it. The post-pass state contains the +crate root, the foo module, then the foo module nested within it, and +then the foo module nested again. But this is not the point where it +should really detect that the foo module should not be nested any +further. It actually is, though. The effects of the first pass are +required to understand that the items in the subtree rooted at a given +module can be referred to by an infinite number of paths. It is once the +same module is nested a third time that resolution should remove the +second level of nesting, essentially "normalizing" a potentially +infinite subtree. This can happen during deduplication before returning +to the resolution driver, where the "larger" subtree is scanned for a +third ocurrence of the top-level module within a direct line of +descendant modules. The idea is to perform a tree traversal that, for +each module, checks the next two descendant modules and ensures they are +not the same module. If they are, then the whole subtree rooted at the +the third ocurrence of the module in question should be removed. Then +the resolution driver can perform the diffing as it currently does, and +the recursive comparison should consider the pre-pass and post-pass +states to be structurally equal. Before getting into the weeds of it, +though, it may be a better idea to think about edge cases where this +will remove some nested self module that should not be removed. A +simpler edge case may be an absolute path to the crate root followed by +a rename. Say we have a crate root and a single child module, where the +latter contains a fine-grained renamed import of the crate root. After +the first pass, the child module contains the crate root as its own +child module. After the second pass, the resolved crate root is "larger" +than the prior crate root, so the child module gets the "new" crate +root. The solution to this problem would notice that the same nested +crate root exists within the larger subtree, and should favor removal of +the nested crate root. Then it should flag all items that appear before +the now removed self-module as being addressable through an "infinite" +set of nested paths that repeat the non-removed segments of the path. To +further illustrate this idea, consider the following example, which is +much like the one in the currently failing test, but reduces the +complexity of the type of use declaration to a renamed fine-grained +import. + +```rust +mod foo { + use crate as root; + + pub struct Foo; + + const FOO: root::foo::root::foo::root::foo::Foo = Foo; +} +``` + +The crate root is imported through a renamed identifier, which on each +subsequent pass allows further nestedness of the same crate root within +module foo. rustc understands that there is potential non-termination if +the use declaration resolution does not halt after the crate root is +found in the subtree rooted at the crate root itself. That seems to be +the pattern that identifies module subtrees as codata; Given some module +a, another snapshot of module a may be found within the subtree rooted +at module a. Though this seems to collide with having a set of child +modules each contain an import referring to the crate root. Or not. If +really "viewed" as a tree, a different copy of the crate root exists on +each child module that has imported the crate root. A "linear" scan over +all items in a given module, here the crate root, would attempt to +remove the nested crate root off of each of the child modules containing +it as a consequence of resolving the super-relative imports. That is +undesired, because what is really needed here is to keep the first +nested crate root, but not any subsequently nested crate roots. In other +words, if a given tree traversal stumbles upon the same module where the +traversal started on, and then further down the tree finds the same +module again, it should only remove this latter ocurrence and not the +first nested ocurrence. But it should "mark" all items in the module +where the first nested ocurrence appeared at as being addressable by an +"infinite" path consisting of repeated occurrences of the path up to the +point where the removed module appeared. On this same topic, how are +paths for the same item reexported through different modules connected +to each other? The whole point of having different paths available for a +given item is that a skip over the path in a reexport will also skip +over the original item, as otherwise it would not make much sense to +have the reexporting module's test not include the item, but the source +module include the item. That doesn't void the need for use declaration +resolution, but does mean that the current approach is effectively +useless at getting at this. Resolution should not import an item over to +the module where the use declaration takes it into account; It should +only add to that item's set of addressable paths the path of the target +module. Doing that without resorting to an anciliary data structure +seems doable until one looks into initially unresolvable use +declarations. Those have a dependency on one or more other use +declarations, which makes resolution as it is currently conceived +necessary. But maybe both ideas can be merged. All items could have +their path fields modified to consist of a collection of possible paths, +which would be expanded each time a use declaration were resolved. This +would require expanding the set of use declarations in both the source +item and the newly imported item. That does mean the source module of +the item (not the use declaration) would have to be kept as part of the +resolution state in the single-use resolution proposition. But that's +already getting into the details of the implementation; Those are best +left for later. The goal here would be to have the set of addressable +paths of the item in both the source and target modules consider all +paths by which that one item can be referred to. This would allow +fine-grained skipping, which does tno take into account more than one +item at a time; It does perform deterministic traversals, so one could +trust a given item could be the one to hold all addressable paths for +itself, but that seems like unnecessary complexity at a potentially +higher cost. Though this would also require changes to the skipping +logic. The current skips are all bunched up as wrappers over each type +of item, and exposed to the closures that the user feeds to the test +generator. The simplest way to accomplish this would be to add +documentation to the skipping functions mentioning that the user can +access the set of paths by which an item can be referred to through some +public method on the types used for each item. The current API to +provide access to a single path would be removed in favor of providing +access to a set of possible paths associated with the term. So to recap, +the following changes are necessary to potentially have use declaration +resolution be ready for use in other areas of ctest. + +* Modifying the path field of each item type to hold a collection of + paths instead of a singular path. During parsing, the parsed path + would make up the singleton list of paths considered for a given item. + +* Changing the merging proposition to stop renaming the paths of items + to be merged, in favor of adding the renamed path to the list of + addressable paths of the item. + +* Changing the merging proposition to traverse the whole-crate module + tree and add to the source item the renamed path obtained above. This + will need changing the first assumption of the proposition to instead + be a stateful pair of the subtree where the use declarations were + sourced from, and the whole-crate module tree to perform a search and + an update on. + +* Changing the deduplication proposition to add another step after + deduplicating the modules in the item container. This new step should + traverse the subtree and check that the root module of this subtree + does not appear more than once in a path down the tree. + +The first three points seem clear. The last point is going to need more +discussion. Suppose we have a crate root with one child module, and the +child contains another child module. Then assume the crate root contains +an import of all items in the leaf module, and another import of all +items in the node module. Then assume the node module has a glob import +of all items in the crate root. Initially, the crate root can not +resolve the use declaration that points to the leaf module because it +has no knowledge of it; It first needs to bring over all items from the +node module. The algorithm performs DFS so it reaches the node module +before it gets to the crate root. That will attempt to resolve the crate +root, which will add the crate root to the list of child modules to the +node module. Then the crate root will resolve its own node module import +to bring over a child module that contains a nested crate root. That +should be the end state of the first resolution pass. Before starting +the second pass, deduplication takes place, but will currently find +nothing to deduplicate. The issue comes from having two duplicates down +two different paths in the subtree rooted at the crate root. One +corresponds with the crate root brought over from the node module, and +the other corresponds with the crate root that is now a child of the +node module. To illustrate this, consider the following diagram. + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> node[Node module] + node ==> child[Leaf module] + root -.-> child + root --> node + node --> root +``` + +The thicker directed edge indicates a connection in the graph, while the +thinner lines indicate a reexport of the target node's modules. The +crate root additionally has a dotted reexport to the child module, which +indicates it is not resolvable during the first resolution pass. That +one reexport depends on the glob import of the node module's items to +the crate root. We'll describe now the states during the first pass. The +algorithm performs DFS, so it will first reach the child module and find +there's no reexports. At that point, it will move back to the node +module, where it finds there's a reexport of all items in the crate +root. This brings over the node module and the child module to the node +module's list of child modules. That can been seen in the following +graph. + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> node[Node module] + + node ==> child[Leaf module] + node ==> oldnode[Node module] + + subgraph old[" "] + oldnode ==> oldchild[Leaf module] + end + + root -.-> child + root --> node + node --> root +``` + +The imported-over items, during deduplication, also remove the use +declarations in the imported modules. This means the nested node module +(the only item in the crate root) gets the use declaration to the crate +root removed. Still during the first pass, the algorithm reaches the +crate root, and attempts to resolve the reexport of all items in the +node module. This should bring over the child module to the crate root, +and thus make the reexport of the child's items now resolvable. The +crate root also ends up with the node module that contains itself +nested. The end state of the first pass is as follows. + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> node[Node module] + root ==> newchild[Leaf module] + root ==> newnode[Node module] + + subgraph new[" "] + newchild + newnode + newnodechild + end + + newnode ==> newnodechild[Leaf module] + + node ==> child[Leaf module] + node ==> oldnode[Node module] + + subgraph old[" "] + oldnode ==> oldchild[Leaf module] + end + + root --> newchild + root --> node + node --> root +``` + +The above graph makes it obvious that, during the second resolution +pass, the reexport in the node module of all items in the crate root +would bring over the child module and node module that are the children +of the crate root, but those would be deduplicated with the ones in the +node module. The one imported item that would not be deduplicated is the +node module, which would be again nested in the node module. That would +prompt non-termination, as the same pattern would repeat anew the next +resolution pass. It would seem the natural solution is thus to add some +further logic to the super case of the single-use resolution function. +Super-relative use statements would first fetch the parent module, then +they would strip from the parent module the child module where the use +declaration currently lives at. In theory, some arbitrary chain of +super-relative path segments can only appear at the start of a use +declaration's path, so this should be extensible to the stateful subtree +that we carry through resolution passes. Except the module that would +have to have the codata pattern added to its set of items' possible +paths would have to be the module where the use declaration is sourced +from. That means that part of the strategy will have to be delayed until +later on, potentially at merge time. This last thing isn't entirely +clear just yet. It may be a good idea to first reconsider the above +example and reproduce the graph without having super-relative imports +bring over the module where the use declaration appears at. Suppose then +that we are posed with the same crate module tree, where a crate root +has one child module, and this child module has another child module. +The crate root contains a glob import of the items in the node module +and a glob import of the items in the leaf module. The latter one is not +resolvable during the first pass because it requires having the leaf +module accessible as a direct descendat of the crate root. This only +happens when the crate root resolves its glob import of the node +module's items (among which we find the leaf module.) + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> node[Node module] + node ==> child[Leaf module] + root -.-> child + root --> node + node --> root +``` + +During the first resolution pass, the first module that will attempt +resolution is the node module. This follows the same depth-first +approach as has already been mentioned above. The node module this time +ends up with no new items, as the only item in the crate root is the +node module itself, which we are now skipping. The next step of the +resolution pass goes through resolving the glob-import in the crate root +of the node module. That should import over the contents of the node +module, which here means the leaf module. The end state after the first +resolution pass would look as follows. + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> node[Node module] + root ==> newchild[Leaf module] + node ==> child[Leaf module] + subgraph new[" "] + newchild + end + root --> newchild + root --> node + node --> root +``` + +After the first pass, the resolution driver would conclude that +resolution must continue, as there have been changes between the pre- +and post-pass states of the whole-crate module-tree. During the second +resolution pass, the same depth-first order as before would be applied. +Resolution of the node module's use declaration would be attempted, and +that would import all items from the crate root but the stateful +subtree, which here means the node root. That would bring over a +duplicate of the leaf module to the list of child modules to the node +module. During deduplication in the single-module resolution pass, the +node module would have that duplicate leaf module cleansed. This would +then mean that the final state of the second resolution pass would be +equivalent to that of the first resolution pass. This would prompt the +resolution driver to terminate. The graph does not take into account +importing items, but the second resolution pass would also be capable of +resolving the leaf module glob import found in the crate root. That +would import all items in the leaf module over to the crate root, which +would effectively make the resolution driver perform another pass. That +would then bring over the leaf module's items to the node module, +because of the glob import of the crate root's items found in the node +module. But beyond that, no other use declaration in the third pass +should change the module tree. The glob import of the node module's +items in the crate root would end up deduplicating the leaf module's +items and the leaf module itself. The glob import of the leaf module's +items would also end up deduplicating this module's items after having +already been added to the crate root in the second resolution pass. Now, +how does the above graph, which represents the end state of the +whole-crate module tree post-resolution, look like in comparison with +the ideal case? In the ideal case, one may refer to the items in the +node module with an arbitrarily nested chain of the node module's +identifier. As an example, take the following code snippet, which is +accepted by rustc and replicates the crate module tree considered in +this running example. + +```rust +mod foo { + pub use super::*; + + pub struct Foo; + + mod bar { + pub struct Bar; + } +} + +pub use self::{bar::*, foo::*} +``` + +The above code would allow referring to both the Foo and Bar records +through some arbitrarily nested foo::foo::... path. This does mean the +codata pattern has do be expanded to all items in the subtrees rooted at +the child modules of the module where the super-relative use declaration +lives at. And I would say that's about it. Except that's not it. Notice +how the codata pattern can also manifest itself through the imports of +the node module's items at the crate root. If there were some function +baz at the crate root that referred to record Foo by some path +foo::foo::...::Foo, rustc would accept it. The reason for that is that +the crate root is also meant to get a copy of the items in the node +module with the codata pattern added to their list of addressable paths. +That should be doable in the deduplication function, which now just +orders and deduplicates non-modules by their paths. It will have to be +refactored into deduplicating those items by the number of addresssable +paths they contain. The more addressable paths an item has, the more +up-to-date is their state is in the module tree. This should hold +because we can say there's an invariant about an item's list of +addressable paths. These will only ever be a singleton list or a larger +list, and the list itself will never shrink. So as a quick recap for +future me implementing this: + +* There is need to change all parsed items' types to have their path + field indicate a set of addressable paths instead of only a single + path. + +* There is need for the single-use resolution function to remove the use + declaration's module from the list of child modules of the parent + module fetched in the super subcase of both name and path cases. + +* There is need for the deduplication function to change the sorting + logic to instead take into account both the cached path, and in the + case of equivalent cached paths, the length of the addressable paths. + + The latter should have those items with the longest lists appear first + post-sorting. This means it will have to adopt the same + reverse-ordering strategy as the special case when deduplicating + modules. + +There are some concerns about the first point, though. Consider the +following example, and accompnying graph. + +```rust +mod foo { + mod bar { + use super::super::*; + } +} + +pub struct Foo; +``` + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> node[Node module] + node ==> leaf[Leaf module] + leaf --> root +``` + +In this instance, one would find that the first resolution step will +attempt to resolve the use declaration in the leaf module. For that +matter, there are no other use declarations, so that really is the only +possible use declaration we can resolve. The issue here would be that +the super-relative module would not be capable of finding the leaf +module among the immediate descendants of the crate root. In other +words, the strategy should not just attempt to remove the module where +the use declaration is sourced at, but rather traverse the subtree of +the target module once resolved, and attempt to find the module where +the use declaration was sourced at. After the first pass, the end result +of this strategy would look as follows. + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> node[Node module] + node ==> leaf[Leaf module] + leaf ==> node2[Node module] + leaf --> root + subgraph graph2[" "] + node2 + end +``` + +The second resolution pass would end up doing the same thing, which +would prompt the resolution driver to terminate. Just to be sure, +though, the second pass would, again, attempt to resolve the use +declaration pointing towards the items of the crate root. This would +bring over the node module and whatever items lived in the crate root +(in the above example, record Foo.) But that should be about it, because +the leaf module would be removed off of the imported node module before +the resolution item were yield in the single-use resolution function. +Without getting into the details of the implementation, it seems there's +at least one thing that doesn't seem quite right here. What would be the +codata pattern of all items' list of addressable paths under the leaf +module? Expanding the code snippet with example uses may be in order. + +```rust +mod foo { + mod bar { + use super::super::*; + + const FOO: foo::bar::foo::bar::Foo = Foo; + } +} + +pub struct Foo; +``` + +The codata pattern is in the two-segment fragment foo::bar. That means +all items now living in the leaf module, among which we find record Foo, +are meant to have the codata pattern associate the upward path of some +ancestor modules except for the module where the item is sourced from. +Here, this means that the items brought over to the lea module would +have to be capable of being addressed by their absolute paths, except +for the first segment of the path, which in this instance is "invisible" +because it is the crate root. In other words, the codata pattern +detection and addition to the list of addressable paths of the item must +happen after the item is renamed to reflect the path of its new parent +module. That doesn't currently happen nor can it possibly take place in +the single-use resolution function because we do not keep track of the +module where the use declaration was sourced from. That would mean that +specific piece of logic would have to be moved to the merging function. +This does change the plan outlined in the bullet points above. + +* Change the path field of all parsed items into accepting a set of + addressable paths instead of a single path. + +* Change the merge function to perform another step after renaming and + before extending the module into which to merge. This step should + consist of looking up each module to merge in search of the module + into which to merge within the subtree rooted at them. + + This would ensure that the module into which to merge is not found + nested within the subtree it is a root of. + +This strategy has some fairly large flaws as it stands. We considered +changing the super subcase of the name and path cases of the single-use +resolution function, but that is not a great idea. See, that function +takes into account some stateful pair of a subtree and the whole-crate +module tree, as well as some use declaration. For all we know, once we +unwind from the leaf module to the node module after munching through +the first super segment, we will also remove the node module off of the +crate root after stumbling upon the second super segment. That would +return back to the leaf module a crate root that does not contain the +node module. The resulting module tree is incorrect. One solution to +this would be to extensively refactor the case of super-relative +segments. Instead of individually munching through each segment, and +yielding the immediate parent before moving on to the next segment, all +super segments could be munched at once. Basically, it would go like +this. + +* Have some auxiliary function that was used as soon as a super-relative + import were detected. This function would eagerly munch through all + super segments of the path, but only the super segments. As it went on + munching, it would count the number of super segments, and yield this + number upon hitting the basecase. The basecases here would be hit once + there were either no more segments left in the path, or the next + segment is a non-super-relative segment. + +* The returned number would indicate the amount of times we would have + to fetch the parent module of the current stateful subtree. This would + yield the target module after all super-relative segments are + processed. + + This target module can then be used in the module fetching function to + obtain the "last" ancestor module the use declaration points at. This + module would then be rid of the current stateful subtree by attempting + to look up the latter in the former. This is the key part in avoiding + having some module nested within itself. + +So that's the non-termination problem solved. The next problem is about +dealing with the codata pattern. This we could potentially also solve by +integrating it into the above scheme. We know that the items brought +over from the use declaration have to be addressable by a nested path. +But that is only the first part of the story. Thus far, we've been +considering fairly couth though still pathological cases. What about a +super-relative import that, after going up to some ancestor module, goes +back down into some unrelated branch of the whole-crate module tree? +Consider the following code snippet example, as well as the accompanying +graph. + +```rust +mod foo { + mod bar { + use super::super::baz::*; + } +} + +mod baz { + pub struct Baz; +} +``` + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> foo & baz + foo ==> bar + bar --> baz +``` + +In this case, we would want the module tree post-resolution to look as +follows. + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> foo & baz + foo ==> bar + bar ==> baz2[baz] + bar --> baz2[baz] + subgraph n[" "] + baz2 + end +``` + +The module resolution strategy would reach module bar, find the use +declaration, and assuming the eager super munching strategy were used, +it would first fetch the crate root, then it would strip bar off of the +subtree in the crate root, and then it would continue munching through +the rest of the super-relative use declaration, which points at the baz +module (a child of the crate root.) The next case that would be hit in +the single-use resolution function would be the name basecase, which +would refer to module baz. This would then be looked up in the stateful +subtree carried across passes, and it would be successfully resolved. +All items from baz would be imported over to the bar module in the end. + +Further attempts at resolution would go through the same motions. +Namely, and just in case, the crate root would be fetched anew, the bar +module would be wiped off of its subtree, and the baz module would be +found in the next call to the single-use resolution function. Then it +would bring over all the items from module baz, which would be +deduplicated with those already brought over from the first resolution +pass. + +Now, how would this work with a super-relative glob import in the baz +module? Suppose the baz module always comes second in the resolutions +made of the child modules to the crate root. This would imply that the +foo module would be processed first, and thus the bar module's use +declaration would be processed first as well. The code snippet for this +example and the accompanying graph would look as follows. + +```rust +mod foo { + mod bar { + use super::super::baz::*; + + const FOO: baz::baz::baz::baz::Foo = Foo; + } +} + +pub struct Foo; + +mod baz { + pub use super::*; + + pub const BAZ: baz::baz::baz::baz::Foo = Foo; +} +``` + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> foo + root ==> baz + foo ==> bar + bar --> baz + baz --> root +``` + +In this case, we would want to have the end graph look as follows. + +```mermaid +--- +config: + layout: elk +--- +graph + root[Crate root] ==> foo + root ==> baz + foo ==> bar + bar ==> foo3[foo] + baz ==> foo2[foo] + foo2 ==> bar2[bar] + bar --> baz + baz --> root + subgraph n[" "] + foo2 + bar2 + end + subgraph n2[" "] + foo3 + end +``` + +Resolution would follow first by going down the path where the bar +module lies at. This would attempt to resolve the super-relative +declaration that points towards the items of the baz module. That would +yield no items whatsoever, because the baz module is still empty. Then +resolution would reach the baz module, where it would attempt to resolve +the crate root. It would resolve the super-relative module, then get rid +of the baz module in the subtree rooted at the crate root, and bring +over all remaining items. We're only considering modules in the graph, +so that would bring over the foo module with the bar module nested +within it. The use declaration in the baz module would bring over the +foo module with the baz module within it, and, of course, all cleansed +of use declarations. + +That would just about finish the first resolution pass, after which the +resolution driver would conclude that another pass would be in order. + +The second pass would go through the bar module's use declaration again, +but this time we would find that it would not remove itself off of the +foo module child that is part of the items in baz. In other words, the +graph for the ideal end-state would not be reached; The bar module would +end up with itself nested, which means we actually haven't solved just +yet the non-termination issue. Or does it? Recall how the proposed +super-relative resolution proceeds. We munch through all super segments +of the use declaration, and upon hitting the "last" ancestor module that +the use declaration points at, we traverse the entire subtree rooted at +that module to remove all occurrences of the stateful subtree in the +single-use resolution function. This would mean that both the "original" +bar and the one imported in the baz module as part of its use +declaration would be removed. Then, single-use resolution would follow +up with the last non-super segment of the path, which would point at +module baz and then at all items in module baz. This would yield only +module foo, because at this point we've been following resolution using +as stateful subtree the crate root without any ocurrences of module bar. + +This does imply there is no need to change anything from the strategy +followed thus far. So long as the super munching actually works, then we +should be gold even in cases where the super-relative use declaration +doesn't immediately glob-import all items from the ancestor module. diff --git a/notes/pr-answer.md b/notes/pr-answer.md new file mode 100644 index 0000000000000..317fac08536a9 --- /dev/null +++ b/notes/pr-answer.md @@ -0,0 +1,14 @@ +>Like what? The rules for how `Send` and `Sync` are automatically +>implemented are pretty clear, and docs already indicate whether a type +>implements these or not. + +Sure it may be in the docs, but the section of rustdoc on blanket impls +seems to me to not be nearly as often looked at as other impls. + +It would thus seem that libc users would likely not realize until they +got a compile error. Granted, that already serves as a "fix," so it may +just be enough. + +The usage guidelines could mention that we may sweep in some such +changes across non-breaking SemVer releases. Then folks would be more +actively on the look out of types becoming `!Send`. diff --git a/notes/pr-revised.md b/notes/pr-revised.md new file mode 100644 index 0000000000000..75991311bbaad --- /dev/null +++ b/notes/pr-revised.md @@ -0,0 +1,382 @@ +## Description + +Add support for (currently) parsing and filtering Rust modules found in +the target crate to test. This is supposed to provide scaffolding for +multiple open feature requests that depend on ctest having support for +modules. + +I decided to open this before finishing up work on generating proper +tests on both C and Rust sides, because I am not sure if I should just +let the user deal with including a cohesive set of modules that do not +cause item resolution conflicts, or if I should instead generate +separate tests for each parsed module. + +**Edit**: the above paragraph really goes to say that I am not sure +whether I should generate multiple test templates while recursing +through the modules in `TestGenerator::generate_files`, or if I should +instead recurse through them in `TestTemplate::new` (with the passed +`FfiItems`) and return a collection of test templates for each parsed +module that has not been skipped. + +**Edit 2**: I decided to go for generating multiple test templates for +each parsed module. + +**Edit 3**: I completely forgot about `use` statements, so most of this +is currently useless. + +**Edit 4**: `use` statements have now an initial solution. This still +has limitations. Relevant changes are documented in Section Update 3. A +better algorithm is in the works. + +cc @tgross35 @mbyx + +### TODO + +- [ ] Look into parsing `use` statements, as those are the one things + missing from getting modules to work. + +- [ ] Update the docs to the public API of `TestGenerator` to mention + that the name remapping ought return the C identifiers without + worrying about potential item resolution conflicts. + +- [x] Look through the `populate_roundtrip_tests` function, as that one + I skipped to go first to `populate_field_ptr_tests` because of the + order these tests appear in the test template files. + +- [ ] Look into properly resolving paths to the types of type aliases, + constants and statics. These are handled poorly now. An example is + the check for arrays that is made in + `template::TestTemplate::populate_roundtrip_tests`, which does not + consider whether the type of the alias is itself an alias to be + resolved recursively (and potentially be an array.) + +### Update 1 + +I have changed the way we parse item identifiers, and more specifically, +their absolute paths without `crate` at the start (e.g. +`foo::bar::ctime()` instead of `crate::foo::bar::ctime()`.) + +This should make filtering continue working as-is right now for items, +but allow filtering on items in nested submodules by using an +appropriate string matching something like the above example. + +### Update 2 + +I think the initial set of changes is done. I have gone down the route +of potentially generating multiple templates for each parsed module, +such that more than a single test is generated. I have not yet +implemented the multi-template generation stuff, but that should be +fairly straightforward. What I believe to be done is the logic for +generating test to the right Rust paths while also keeping the single +segment paths on the C side of things. + +I am currently working through the tests, to see what is missing and +broken from the prior `ctest` interface. At this point, I am only trying +to work through the prior tests to see if they continue working and +generating the same set of tests (barring nested modules.) + +### Update 3 + +This is an update on the last few weeks' worth of work. Time was spent +on a module resolution algorithm. This is necessary for correct item +paths. An example of this need is `libc` itself. + +The current module tree shape has reexports at the crate root. This +means items should be "referrable" by that path (i.e. their identifiers +alone.) But the current strategy (in this patch) gets their source path. + +This does not apply to upstream `ctest`. `ctest` currently assumes items +are reexported at the crate root. It only picks up on item identifiers +during parsing. But module support needs import resolution. + +A summary of related changes in this patchset follows. This does not +currently include all changes in this PR. + +#### Changes to `FfiItems` parsing + +The `FfiItems` type needed to change its use of `syn`'s visitor hooks. +These are not completely useful with the addition of modules. This also +uncovered what is believed to be a bug in upstream `ctest`. + +The main concern here are non-module scope items. The `syn` visitor +hooks will visit every single matching item. This means items that only +exist within a function are also parsed. + +Such an item can not be used outside of the function. Other non-module +scopes also have this limitation (e.g. module-level `const` blocks.) +Upstream `ctest` will thus attempt to generate tsets for these as well. + +One could argue downstream users can set up skips for these. But that is +only a workaround. This also applies to `use`-statement scanning. A +`use` statement can exist within a function. But those should not be +resolved. + +Bottom line: The `syn` visitor hooks had to be removed. Though it would +be more accurate to say that they were repurposed. Those trait methods +now live as free functions. + +The solution went through use of the module hooks. These are the only +ones we can safely use. A module may not be declared within a function +scope. And `ctest` requires only the items that appear within modules. + +The current approach wraps the crate root in a module. This is because +the crate root is not parsed as a module. So the hook for modules is not +run in those cases. Then the actual parsing is done in the hook. + +Each of the items `ctest` needs are "manually" parsed. This calls into +the previous hooks. These are now free functions. This is where they are +now used. This deserves further explanation. + +The crate root is wrapped in a virtual module. This is to trigger the +module visitor hook. The virtual module needs to have a special +identifier. This is detected during parsing. + +The crate root gets all parsed items added to it. Any other module gets +a new child module with all parsed items added to it. This builds up the +module tree hierarchy as one would expect. + +But all of this needs to exist within a wrapper. Filling in an +`FfiItems` is no more about calling into `syn`'s file hook. It needs to +first wrap the items in the file in a module of their own. + +This is what eventually gets parsed. The item container is then +unwrapped from the virtual module. Note the module is a `syn` module +type. This needs another transformation for the resolution algorithm. + +There is one last thing to note about parsing. `use` statements may come +as groups (paths wrapped in curly brackets.) This is not convenient to +handle afterwards. These get "normalized" during parsing. + +Normalization does one thing alone. It munches through a `use` +statement. This may eventually stumble upon a group. That gets flattened +into a set of individual `use` statements. A single `use` statement then +yields a list. + +One note about terminology must be made here. `use` statements may have +arbitrary paths as their associated objects. They may also have a single +name. These are all referred to from here on as paths. + +#### The resolution algorithm + +The algorithm for resolution will now be explained. This follows a +fairly straightforward process. There is a single entrypoint routine. It +takes a module and yields another module. + +This is meant to be called from the afore mentioned wrapper. It is +called with another virtual module for the root. Recall the crate root +is initially only an item container. + +The new virtual module uses `ctest`'s own `Module` type. This is used to +keep track of the path to the module. This will be needed during +`use`-statement resolution. + +Resolution starts by bottomming out on the list of child modules. This +is to ensure it takes place in a depth-first manner. Then an adaptive +pass system is triggered over a single module. + +The current approach requires resolution to be depth-first. This is +because of the current limitations the algorithm has. The current +resolution strategy can only resolve "forward" `use` statements. + +A "forward" `use` statement is defined as a one that brings into scope +items from a descendant module. Its dual brings into scope items from +some ascendant module. + +These observations are made with respect to the module in which the +`use` statement appears. + +Note this does not exclude absolute `use` statements. A `use` statement +is determined as one of the above based off of the target item. If the +target item is a descendant then that is a "forward" `use` statement. + +Current work is focused on support for bidirectional imports. The rest +of the explanation will focus on the initial revision of the algorithm. +This supports only `use` statements that refer to descendant items. + +This justifies the need for a depth-first approach. A leaf module in the +crate's module tree will have no resolvable `use` statements. This is +also a good time to define "resolvable" and "unresolvable" `use` +statements. + +A "resolvable" `use` statement is said to be resolvable in some finite +number of resolution passes. Its dual is impossible to resolve in any +number of resolution passes. + +One example of the latter are `use` statements from third-party crates. +These will never resolve. The algorithm could attempt to inject the +crates into the module tree. But a simpler solution was found. + +That shall be explained later. + +The pass system explanation will now be resumed. A single resolution +pass is given by two steps. The first one is the production of a list of +resolution items. The second one merges these into the module at hand. + +The former resolves all `use` statements within a given module. This +yields a list of resolution items. Each resolution item corresponds with +one `use` statement. It can be "resolved" or "unresolved." + +A `use` statement is said to be "resolved" if the path it denotes has +yield a concrete item. Its dual refers to an item path that was not +found with respect to the module where the `use` statement appears at. + +Consider the following example. + +```rust +use foo::*; + +use bar::*; + +use std::any::*; + +mod bar { + mod foo { + pub struct Foo; + } +} +``` + +Resolution of the first `use` statement depends on the second. The first +`use` statement will yiled an unresolved item in the first pass. Only +upon the second `use` statement resolving will the former also resolve. + +Though note both statements in the above example are said to be +resolvable. A definition for this is given further above. The third +`use` statement is said to be unresolvable. + +A single pass maps the list of `use` statements. It uses the +`use`-statement resolution routine as map transform. This routine +munches through a given `use` statement's path. + +Each path of the segment consists of an identifier. This routine carries +with it a module. This serves as state. This state is initially seeded +by the module where the `use` statement appears at. + +Each identifier in the path gets looked up in this stateful module. A +match means the routine may continue munching. A mismatch means the +routine must yield an unresolved item. + +A munch that reaches the basecase denotes a resolved item. This is +produced much like an unresolved item. This bubbles up to the single +module resolution routine. That one mapped the list of `use` statements. + +This eventually yields a potentially non-empty list of resolution items. +These now need to be merged into the top-level module. This is the +module where the `use` statements that sourced the resolutions lived at. + +This now enters the second stage of a given resolution pass. Merging +iterates through the list of resolution items. Resolved items pack the +item the `use` statement referred to. This deserves further explanation. + +A `use` statement can refer to one of a non-glob or a glob. Non-globs +may be further specified as non-modules or modules. A glob packs in the +resolved item an item container with all items from the parent module. + +A non-glob packs in the resolved item an item container with itself only +as the contents. Notice how a non-module is small. A module carries with +it the subtree rooted at it in the crate's module tree. + +A resolved glob has a very similar topology to a module. It also packs +the subtree rooted at it in the crate's module tree. But it contains no +information on the "subroot" where that subtree was sourced from. + +Now back to the merging routine. Each resolved item also carries with it +the source `use` statement. This is looked up in the module into which +to merge. This is then is removed from the module into which to merge. + +The last step is to extend the module into which to merge. This is the +actual "merging." The packed item container is deconstructed into its +component items. These then extend the lists of items in the target. + +That finishes a single resolution pass. The next thing it does is go +back to the top-level resolution routine. That now determines whether +another pass is in order. + +This is determined by diffing the number of `use` statements. The +operands are the pre-pass module and the post-pass module. Passes over a +single module terminate if this comparison yields no differences. + +This is also what allows unresolvable `use`-statements not to break the +algorithm. These will not change once resolvable `use`-statements are +fully resolved. That is the sign that the pass system catches on. + +This is then repeated depth-first across all modules. The root module is +processed last. This one has at that point the entire subtree resolved. +It may need more than one pass. But it will resolve resovable imports. + +#### Extra notes + +A few details were glossed over the above explanation. These are not +essential to understanding the algorithm. Chief among them is the path +manipulation routine and the rename case of `use`-statement resolution. + +The latter will be explained first. A `use` statement in Rust can have a +trailing renamed identifier. This identifier needs to also be reflected +in `ctest`'s view of the crate's module tree. + +An item's path is often straightforward to rename. Modification of the +last segment of a path is all it takes to rename a non-module. But a +module is also a subtree for other items and modules. + +A module thus needs to recursively rename all items within the tree. +That is the purpose of the path manipulation routine. It modifies a +given module subtree with an externally-sourced closure. + +Use of this routine in the rename case is straightforward. The case of +modules first modifies the root's path. Then each item builds its path +anew. The base path becomes the root's new path. Other segments remain. + +The path manipulation routine is also used in the merge routine. Here +the logic is slightly more intricate. Path manipulation takes the module +in which a tree is rooted at. But a resolution item packs a container. + +Recall item containers differ from modules. Items containers exist +within modules. So path manipulation during merging requires a virtual +module. This itself requires special dummy values. + +A dummy path segment for the virtual module must be non-empty. This is +because the passed closure assumes so. This holds for all descendant +items. But there is another subtlety to this path manipulation call. + +The passed subtree contains not the modules. These need further +consideration. A module to be merged should not have the entire path of +descendant items modified. + +Only segments prior to the root's identifier should be removed. This +collides with the above path manipulation call. So modules are handled +separately. They do follow the same logic as the rename case above. + +One more thing happens in a pass. A merged module's items are checked +for duplicates. This can happen from overlapping `use`-statements. The +type for an item container implements that operation. + +Consider this example. + +```rust +use foo::*; + +use bar::*; + +mod foo { + pub struct Foo; + + mod bar { + pub struct Foo; + } +} +``` + +The crate root imports would bring two identical `Foo`s into scope. +Cases like these are in need of deduplication. + +Performance has not been taken into consideration. Certain data +structures could be improved. One example would be the use of +associative arrays for items. This would avoid deduplication. + +## Checklist + +- [ ] Relevant tests in `ctest/tests` and `ctest/src/tests.rs` have been + updated +- [ ] Tested locally (`LIBC_BLESS=1 cargo test -p ctest`); + +@rustbot label +stable-nominated diff --git a/notes/pr-write-up.md b/notes/pr-write-up.md new file mode 100644 index 0000000000000..23b0e170e71a7 --- /dev/null +++ b/notes/pr-write-up.md @@ -0,0 +1,297 @@ +### Update 3 + +This is an update on the last few weeks' worth of work. Time was spent +on a module resolution algorithm. This is necessary for correct item +paths. An example of this need is `libc` itself. + +The current module tree shape has reexports at the crate root. This +means items should be "referrable" by that path (i.e. their identifiers +alone.) But the current strategy (in this patch) gets their source path. + +This does not apply to upstream `ctest`. `ctest` currently assumes items +are reexported at the crate root. It only picks up on item identifiers +during parsing. But module support needs import resolution. + +A summary of related changes in this patchset follows. This does not +currently include all changes in this PR. + +#### Changes to `FfiItems` parsing + +The `FfiItems` type needed to change its use of `syn`'s visitor hooks. +These are not completely useful with the addition of modules. This also +uncovered what is believed to be a bug in upstream `ctest`. + +The main concern here are non-module scope items. The `syn` visitor +hooks will visit every single matching item. This means items that only +exist within a function are also parsed. + +Such an item can not be used outside of the function. Other non-module +scopes also have this limitation (e.g. module-level `const` blocks.) +Upstream `ctest` will thus attempt to generate tsets for these as well. + +One could argue downstream users can set up skips for these. But that is +only a workaround. This also applies to `use`-statement scanning. A +`use` statement can exist within a function. But those should not be +resolved. + +Bottom line: The `syn` visitor hooks had to be removed. Though it would +be more accurate to say that they were repurposed. Those trait methods +now live as free functions. + +The solution went through use of the module hooks. These are the only +ones we can safely use. A module may not be declared within a function +scope. And `ctest` requires only the items that appear within modules. + +The current approach wraps the crate root in a module. This is because +the crate root is not parsed as a module. So the hook for modules is not +run in those cases. Then the actual parsing is done in the hook. + +Each of the items `ctest` needs are "manually" parsed. This calls into +the previous hooks. These are now free functions. This is where they are +now used. This deserves further explanation. + +The crate root is wrapped in a virtual module. This is to trigger the +module visitor hook. The virtual module needs to have a special +identifier. This is detected during parsing. + +The crate root gets all parsed items added to it. Any other module gets +a new child module with all parsed items added to it. This builds up the +module tree hierarchy as one would expect. + +But all of this needs to exist within a wrapper. Filling in an +`FfiItems` is no more about calling into `syn`'s file hook. It needs to +first wrap the items in the file in a module of their own. + +This is what eventually gets parsed. The item container is then +unwrapped from the virtual module. Note the module is a `syn` module +type. This needs another transformation for the resolution algorithm. + +There is one last thing to note about parsing. `use` statements may come +as groups (paths wrapped in curly brackets.) This is not convenient to +handle afterwards. These get "normalized" during parsing. + +Normalization does one thing alone. It munches through a `use` +statement. This may eventually stumble upon a group. That gets flattened +into a set of individual `use` statements. A single `use` statement then +yields a list. + +One note about terminology must be made here. `use` statements may have +arbitrary paths as their associated objects. They may also have a single +name. These are all referred to from here on as paths. + +#### The resolution algorithm + +The algorithm for resolution will now be explained. This follows a +fairly straightforward process. There is a single entrypoint routine. It +takes a module and yields another module. + +This is meant to be called from the afore mentioned wrapper. It is +called with another virtual module for the root. Recall the crate root +is initially only an item container. + +The new virtual module uses `ctest`'s own `Module` type. This is used to +keep track of the path to the module. This will be needed during +`use`-statement resolution. + +Resolution starts by bottomming out on the list of child modules. This +is to ensure it takes place in a depth-first manner. Then an adaptive +pass system is triggered over a single module. + +The current approach requires resolution to be depth-first. This is +because of the current limitations the algorithm has. The current +resolution strategy can only resolve "forward" `use` statements. + +A "forward" `use` statement is defined as a one that brings into scope +items from a descendant module. Its dual brings into scope items from +some ascendant module. + +These observations are made with respect to the module in which the +`use` statement appears. + +Note this does not exclude absolute `use` statements. A `use` statement +is determined as one of the above based off of the target item. If the +target item is a descendant then that is a "forward" `use` statement. + +Current work is focused on support for bidirectional imports. The rest +of the explanation will focus on the initial revision of the algorithm. +This supports only `use` statements that refer to descendant items. + +This justifies the need for a depth-first approach. A leaf module in the +crate's module tree will have no resolvable `use` statements. This is +also a good time to define "resolvable" and "unresolvable" `use` +statements. + +A "resolvable" `use` statement is said to be resolvable in some finite +number of resolution passes. Its dual is impossible to resolve in any +number of resolution passes. + +One example of the latter are `use` statements from third-party crates. +These will never resolve. The algorithm could attempt to inject the +crates into the module tree. But a simpler solution was found. + +That shall be explained later. + +The pass system explanation will now be resumed. A single resolution +pass is given by two steps. The first one is the production of a list of +resolution items. The second one merges these into the module at hand. + +The former resolves all `use` statements within a given module. This +yields a list of resolution items. Each resolution item corresponds with +one `use` statement. It can be "resolved" or "unresolved." + +A `use` statement is said to be "resolved" if the path it denotes has +yield a concrete item. Its dual refers to an item path that was not +found with respect to the module where the `use` statement appears at. + +Consider the following example. + +```rust +use foo::*; + +use bar::*; + +use std::any::*; + +mod bar { + mod foo { + pub struct Foo; + } +} +``` + +Resolution of the first `use` statement depends on the second. The first +`use` statement will yiled an unresolved item in the first pass. Only +upon the second `use` statement resolving will the former also resolve. + +Though note both statements in the above example are said to be +resolvable. A definition for this is given further above. The third +`use` statement is said to be unresolvable. + +A single pass maps the list of `use` statements. It uses the +`use`-statement resolution routine as map transform. This routine +munches through a given `use` statement's path. + +Each path of the segment consists of an identifier. This routine carries +with it a module. This serves as state. This state is initially seeded +by the module where the `use` statement appears at. + +Each identifier in the path gets looked up in this stateful module. A +match means the routine may continue munching. A mismatch means the +routine must yield an unresolved item. + +A munch that reaches the basecase denotes a resolved item. This is +produced much like an unresolved item. This bubbles up to the single +module resolution routine. That one mapped the list of `use` statements. + +This eventually yields a potentially non-empty list of resolution items. +These now need to be merged into the top-level module. This is the +module where the `use` statements that sourced the resolutions lived at. + +This now enters the second stage of a given resolution pass. Merging +iterates through the list of resolution items. Resolved items pack the +item the `use` statement referred to. This deserves further explanation. + +A `use` statement can refer to one of a non-glob or a glob. Non-globs +may be further specified as non-modules or modules. A glob packs in the +resolved item an item container with all items from the parent module. + +A non-glob packs in the resolved item an item container with itself only +as the contents. Notice how a non-module is small. A module carries with +it the subtree rooted at it in the crate's module tree. + +A resolved glob has a very similar topology to a module. It also packs +the subtree rooted at it in the crate's module tree. But it contains no +information on the "subroot" where that subtree was sourced from. + +Now back to the merging routine. Each resolved item also carries with it +the source `use` statement. This is looked up in the module into which +to merge. This is then is removed from the module into which to merge. + +The last step is to extend the module into which to merge. This is the +actual "merging." The packed item container is deconstructed into its +component items. These then extend the lists of items in the target. + +That finishes a single resolution pass. The next thing it does is go +back to the top-level resolution routine. That now determines whether +another pass is in order. + +This is determined by diffing the number of `use` statements. The +operands are the pre-pass module and the post-pass module. Passes over a +single module terminate if this comparison yields no differences. + +This is also what allows unresolvable `use`-statements not to break the +algorithm. These will not change once resolvable `use`-statements are +fully resolved. That is the sign that the pass system catches on. + +This is then repeated depth-first across all modules. The root module is +processed last. This one has at that point the entire subtree resolved. +It may need more than one pass. But it will resolve resovable imports. + +#### Extra notes + +A few details were glossed over the above explanation. These are not +essential to understanding the algorithm. Chief among them is the path +manipulation routine and the rename case of `use`-statement resolution. + +The latter will be explained first. A `use` statement in Rust can have a +trailing renamed identifier. This identifier needs to also be reflected +in `ctest`'s view of the crate's module tree. + +An item's path is often straightforward to rename. Modification of the +last segment of a path is all it takes to rename a non-module. But a +module is also a subtree for other items and modules. + +A module thus needs to recursively rename all items within the tree. +That is the purpose of the path manipulation routine. It modifies a +given module subtree with an externally-sourced closure. + +Use of this routine in the rename case is straightforward. The case of +modules first modifies the root's path. Then each item builds its path +anew. The base path becomes the root's new path. Other segments remain. + +The path manipulation routine is also used in the merge routine. Here +the logic is slightly more intricate. Path manipulation takes the module +in which a tree is rooted at. But a resolution item packs a container. + +Recall item containers differ from modules. Items containers exist +within modules. So path manipulation during merging requires a virtual +module. This itself requires special dummy values. + +A dummy path segment for the virtual module must be non-empty. This is +because the passed closure assumes so. This holds for all descendant +items. But there is another subtlety to this path manipulation call. + +The passed subtree contains not the modules. These need further +consideration. A module to be merged should not have the entire path of +descendant items modified. + +Only segments prior to the root's identifier should be removed. This +collides with the above path manipulation call. So modules are handled +separately. They do follow the same logic as the rename case above. + +One more thing happens in a pass. A merged module's items are checked +for duplicates. This can happen from overlapping `use`-statements. The +type for an item container implements that operation. + +Consider this example. + +```rust +use foo::*; + +use bar::*; + +mod foo { + pub struct Foo; + + mod bar { + pub struct Foo; + } +} +``` + +The crate root imports would bring two identical `Foo`s into scope. +Cases like these are in need of deduplication. + +Performance has not been taken into consideration. Certain data +structures could be improved. One example would be the use of +associative arrays for items. This would avoid deduplication.