From df898818ef2de32fcf3ed4db4d7745f2e6120bc8 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Fri, 28 Jun 2024 10:16:59 -0700 Subject: [PATCH 01/25] WIP Get ImportProvider working with mutability and provided methods. --- Cargo.toml | 9 +++ examples/custom_import_resolution.rs | 47 ++++++++++++ examples/custom_import_schemes.rs | 53 +++++++++++++ src/inner_runtime.rs | 8 +- src/lib.rs | 1 + src/module_loader.rs | 109 +++++++++++++++++++++++---- 6 files changed, 209 insertions(+), 18 deletions(-) create mode 100644 examples/custom_import_resolution.rs create mode 100644 examples/custom_import_schemes.rs diff --git a/Cargo.toml b/Cargo.toml index 9b8cef01..33876325 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ web = ["console", "url", "crypto", "deno_web", "deno_tls", "deno_fetch", "url_im # Features for the module loader fs_import = [] url_import = ["reqwest"] +custom_import = [] # Enables the use of the SnapshotBuilder snapshot_builder = [] @@ -84,6 +85,14 @@ required-features = ["snapshot_builder"] name = "url_import" required-features = ["fs_import", "url_import"] +[[example]] +name = "custom_import_resolution" +required-features = ["custom_import", "url_import"] + +[[example]] +name = "custom_import_schemes" +required-features = ["custom_import", "url_import"] + [[bench]] name = "runtime" harness = false diff --git a/examples/custom_import_resolution.rs b/examples/custom_import_resolution.rs new file mode 100644 index 00000000..700156d5 --- /dev/null +++ b/examples/custom_import_resolution.rs @@ -0,0 +1,47 @@ +use deno_core::{anyhow::Error, ModuleSpecifier, ResolutionKind}; +use rustyscript::{ImportProvider, Runtime}; + +struct MyImportProvider { + replacement_specifier: String, +} +impl MyImportProvider { + fn new(str: &str) -> Self { + Self { + replacement_specifier: String::from(str), + } + } +} +impl ImportProvider for MyImportProvider { + fn resolve( + &mut self, + specifier: &str, + referrer: &str, + _kind: ResolutionKind, + ) -> std::result::Result { + if specifier == "secret_special_specifier" { + // Substitute another URL in certain situations... + Ok(ModuleSpecifier::parse(&self.replacement_specifier)?) + } else { + // Or fall back to deno's import behavior (also used by rustyscript) + Ok(deno_core::resolve_import(specifier, referrer)?) + } + } +} +fn main() { + let options = rustyscript::RuntimeOptions { + import_provider: Some(Box::new(MyImportProvider::new("https://deno.land/std@0.224.0/assert/mod.ts"))), + ..Default::default() + }; + let mut runtime = Runtime::new(options).expect("Could not create runtime"); + + let module = rustyscript::Module::new( + "custom_imports.js", + " + import { assertEquals } from 'secret_special_specifier' + + assertEquals(1, 1) + ", + ); + + runtime.load_module(&module).expect("Could not load module"); +} diff --git a/examples/custom_import_schemes.rs b/examples/custom_import_schemes.rs new file mode 100644 index 00000000..f65f05f2 --- /dev/null +++ b/examples/custom_import_schemes.rs @@ -0,0 +1,53 @@ +use deno_core::{anyhow::Error, ModuleSpecifier}; +use rustyscript::{ImportProvider, Runtime}; + +struct MyImportProvider { + example_code: String, +} +impl MyImportProvider { + fn new(str: &str) -> Self { + Self { + example_code: String::from(str), + } + } +} +impl ImportProvider for MyImportProvider { + fn import( + &mut self, + specifier: ModuleSpecifier, + _referrer: Option, + _is_dyn_import: bool, + _requested_module_type: deno_core::RequestedModuleType, + ) -> Result { + if specifier.scheme() == "examplescheme" { + // Load code from anywhere + Ok(self.example_code.clone()) + } else { + // Make sure to handle unrecognized schemes as well + Err(Error::msg(format!("unrecognized scheme for module import: {specifier}"))) + } + } +} +fn main() { + let options = rustyscript::RuntimeOptions { + import_provider: Some(Box::new(MyImportProvider::new(" + export function return_number(n) { + return n + } + "))), + ..Default::default() + }; + let mut runtime = Runtime::new(options).expect("Could not create runtime"); + + let module = rustyscript::Module::new( + "custom_imports.js", + " + import { assertEquals } from 'https://deno.land/std@0.224.0/assert/mod.ts' + import { return_number } from 'examplescheme://any_specifier' + + assertEquals(return_number(1), 1) + ", + ); + + runtime.load_module(&module).expect("Could not load module"); +} diff --git a/src/inner_runtime.rs b/src/inner_runtime.rs index 22d1d9fb..4f488731 100644 --- a/src/inner_runtime.rs +++ b/src/inner_runtime.rs @@ -1,7 +1,7 @@ use crate::{ cache_provider::ModuleCacheProvider, ext, - module_loader::RustyLoader, + module_loader::{ImportProvider, RustyLoader}, traits::{ToDefinedValue, ToModuleSpecifier, ToV8String}, transpiler::{self, transpile_extension}, v8_value::JsFunction, @@ -55,6 +55,9 @@ pub struct InnerRuntimeOptions { /// Optional cache provider for the module loader pub module_cache: Option>, + + /// Optional import provider for the module loader + pub import_provider: Option>, /// Optional snapshot to load into the runtime /// This will reduce load times, but requires the same extensions to be loaded @@ -70,6 +73,7 @@ impl Default for InnerRuntimeOptions { default_entrypoint: Default::default(), timeout: Duration::MAX, module_cache: None, + import_provider: None, startup_snapshot: None, extension_options: Default::default(), @@ -86,7 +90,7 @@ pub struct InnerRuntime { } impl InnerRuntime { pub fn new(options: InnerRuntimeOptions) -> Result { - let loader = Rc::new(RustyLoader::new(options.module_cache)); + let loader = Rc::new(RustyLoader::new(options.module_cache, options.import_provider)); // If a snapshot is provided, do not reload ops let extensions = if options.startup_snapshot.is_some() { diff --git a/src/lib.rs b/src/lib.rs index 89de45a7..5801f098 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -234,6 +234,7 @@ pub use inner_runtime::{FunctionArguments, RsAsyncFunction, RsFunction}; pub use module::{Module, StaticModule}; pub use module_handle::ModuleHandle; pub use module_wrapper::ModuleWrapper; +pub use module_loader::ImportProvider; pub use runtime::{Runtime, RuntimeOptions, Undefined}; pub use utilities::{evaluate, import, resolve_path, validate}; pub use v8_value::JsFunction; diff --git a/src/module_loader.rs b/src/module_loader.rs index 24024447..440a4a08 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -14,19 +14,51 @@ use std::{ rc::Rc, }; +#[allow(unused_variables)] +#[cfg(feature = "custom_import")] +pub trait ImportProvider { + fn resolve( + &mut self, + specifier: &str, + referrer: &str, + kind: deno_core::ResolutionKind, + ) -> Result { + Ok(deno_core::resolve_import(specifier, referrer)?) + } + fn import( + &mut self, + specifier: ModuleSpecifier, + referrer: Option, + is_dyn_import: bool, + requested_module_type: deno_core::RequestedModuleType, + ) -> Result { + return Err(anyhow!( + "unrecognized schema for module import: {specifier}" + )); + } +} + type SourceMapCache = HashMap)>; #[derive(Clone)] struct InnerRustyLoader { cache_provider: Rc>>, + import_provider: Rc>>>, fs_whlist: Rc>>, source_map_cache: Rc>, } impl InnerRustyLoader { - fn new(cache_provider: Option>) -> Self { + fn new( + cache_provider: Option>, + import_provider: Option>, + ) -> Self { Self { cache_provider: Rc::new(cache_provider), + import_provider: Rc::new(match import_provider { + Some(p) => Some(RefCell::new(p)), + None => None, + }), fs_whlist: Rc::new(RefCell::new(HashSet::new())), source_map_cache: Rc::new(RefCell::new(SourceMapCache::new())), } @@ -97,9 +129,21 @@ impl ModuleLoader for RustyLoader { &self, specifier: &str, referrer: &str, - _kind: deno_core::ResolutionKind, + kind: deno_core::ResolutionKind, ) -> Result { - let url = deno_core::resolve_import(specifier, referrer)?; + let url: ModuleSpecifier = if self.inner.import_provider.is_some() { + let mut import_provider = self + .inner + .import_provider + .as_ref() + .as_ref() + .unwrap() + .borrow_mut(); + import_provider.resolve(specifier, referrer, kind)? + } else { + deno_core::resolve_import(specifier, referrer)? + }; + if referrer == "." { self.whitelist_add(url.as_str()); } @@ -126,24 +170,28 @@ impl ModuleLoader for RustyLoader { } _ => { + #[cfg(feature = "custom_import")] + // Custom imports can handle any URL scheme + return Ok(url); + #[cfg(not(feature = "custom_import"))] return Err(anyhow!( - "unrecognized schema for module import: {specifier}" + "unrecognized schema for module import: {specifier}" )); } } - Ok(url) } fn load( &self, module_specifier: &ModuleSpecifier, - _maybe_referrer: Option<&ModuleSpecifier>, - _is_dyn_import: bool, - _requested_module_type: deno_core::RequestedModuleType, + maybe_referrer: Option<&ModuleSpecifier>, + is_dyn_import: bool, + requested_module_type: deno_core::RequestedModuleType, ) -> deno_core::ModuleLoadResponse { let inner = self.inner.clone(); let module_specifier = module_specifier.clone(); + let maybe_referrer = maybe_referrer.cloned(); // We check permissions first match module_specifier.scheme() { // Remote fetch imports @@ -175,20 +223,49 @@ impl ModuleLoader for RustyLoader { .boxed_local(), ), - _ => ModuleLoadResponse::Sync(Err(anyhow!( - "{} imports are not allowed here: {}", - module_specifier.scheme(), - module_specifier.as_str() - ))), + _ => { + #[cfg(feature = "custom_import")] + if inner.import_provider.is_some() { + return ModuleLoadResponse::Async( + async move { + inner + .load(module_specifier, |specifier| { + let import_provider = inner.import_provider.as_ref().as_ref().unwrap(); + let maybe_referrer = maybe_referrer.clone(); + let requested_module_type = requested_module_type.clone(); + async move { + import_provider.borrow_mut().import( + specifier, + maybe_referrer, + is_dyn_import, + requested_module_type, + ) + } + }) + .await + } + .boxed_local(), + ); + } + + return ModuleLoadResponse::Sync(Err(anyhow!( + "{} imports are not allowed here: {}", + module_specifier.scheme(), + module_specifier.as_str() + ))); + } } } } #[allow(dead_code)] impl RustyLoader { - pub fn new(cache_provider: Option>) -> Self { + pub fn new( + cache_provider: Option>, + import_provider: Option>, + ) -> Self { Self { - inner: Rc::new(InnerRustyLoader::new(cache_provider)), + inner: Rc::new(InnerRustyLoader::new(cache_provider, import_provider)), } } @@ -247,7 +324,7 @@ mod test { .get(&specifier) .expect("Expected to get cached source"); - let loader = RustyLoader::new(Some(Box::new(cache_provider))); + let loader = RustyLoader::new(Some(Box::new(cache_provider)), None); let response = loader.load( &specifier, None, From 531cddebb9379b20b892001d4e34df4425c95047 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Mon, 1 Jul 2024 08:59:03 -0700 Subject: [PATCH 02/25] WIP Gate the `pub use` statement instead of the trait itself. (Fixes error when feature not enabled) --- src/lib.rs | 1 + src/module_loader.rs | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 7e52be0e..b6611375 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -239,6 +239,7 @@ pub use inner_runtime::{FunctionArguments, RsAsyncFunction, RsFunction}; pub use module::{Module, StaticModule}; pub use module_handle::ModuleHandle; pub use module_wrapper::ModuleWrapper; +#[cfg(feature = "custom_import")] pub use module_loader::ImportProvider; pub use runtime::{Runtime, RuntimeOptions, Undefined}; pub use utilities::{evaluate, import, resolve_path, validate}; diff --git a/src/module_loader.rs b/src/module_loader.rs index 440a4a08..9eeb42f0 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -15,7 +15,6 @@ use std::{ }; #[allow(unused_variables)] -#[cfg(feature = "custom_import")] pub trait ImportProvider { fn resolve( &mut self, From 8db9164799ccccd137331e014f017221fa72a15f Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:42:49 -0700 Subject: [PATCH 03/25] WIP Change ImportProvider's resolve method to return an Option. --- examples/custom_import_resolution.rs | 18 ++++++------- src/module_loader.rs | 40 +++++++++++++--------------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/examples/custom_import_resolution.rs b/examples/custom_import_resolution.rs index 700156d5..990f2ed0 100644 --- a/examples/custom_import_resolution.rs +++ b/examples/custom_import_resolution.rs @@ -1,4 +1,4 @@ -use deno_core::{anyhow::Error, ModuleSpecifier, ResolutionKind}; +use deno_core::{ModuleSpecifier, ResolutionKind}; use rustyscript::{ImportProvider, Runtime}; struct MyImportProvider { @@ -14,16 +14,16 @@ impl MyImportProvider { impl ImportProvider for MyImportProvider { fn resolve( &mut self, - specifier: &str, - referrer: &str, + specifier: &ModuleSpecifier, + _referrer: &str, _kind: ResolutionKind, - ) -> std::result::Result { - if specifier == "secret_special_specifier" { + ) -> std::option::Option> { + if specifier.as_str() == "example:secret_special_specifier" { // Substitute another URL in certain situations... - Ok(ModuleSpecifier::parse(&self.replacement_specifier)?) + Some(ModuleSpecifier::parse(&self.replacement_specifier).map_err(|e| e.into())) } else { - // Or fall back to deno's import behavior (also used by rustyscript) - Ok(deno_core::resolve_import(specifier, referrer)?) + // Or fall back to the default resolve behavior + None } } } @@ -37,7 +37,7 @@ fn main() { let module = rustyscript::Module::new( "custom_imports.js", " - import { assertEquals } from 'secret_special_specifier' + import { assertEquals } from 'example:secret_special_specifier' assertEquals(1, 1) ", diff --git a/src/module_loader.rs b/src/module_loader.rs index 9eeb42f0..475f41a2 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -3,7 +3,7 @@ use crate::{ transpiler, }; use deno_core::{ - anyhow::{self, anyhow}, + anyhow::{self, anyhow, Ok}, futures::FutureExt, ModuleLoadResponse, ModuleLoader, ModuleSource, ModuleSourceCode, ModuleSpecifier, ModuleType, SourceMapGetter, @@ -18,11 +18,11 @@ use std::{ pub trait ImportProvider { fn resolve( &mut self, - specifier: &str, + specifier: &ModuleSpecifier, referrer: &str, kind: deno_core::ResolutionKind, - ) -> Result { - Ok(deno_core::resolve_import(specifier, referrer)?) + ) -> Option> { + None } fn import( &mut self, @@ -130,18 +130,17 @@ impl ModuleLoader for RustyLoader { referrer: &str, kind: deno_core::ResolutionKind, ) -> Result { - let url: ModuleSpecifier = if self.inner.import_provider.is_some() { - let mut import_provider = self - .inner - .import_provider - .as_ref() - .as_ref() - .unwrap() - .borrow_mut(); - import_provider.resolve(specifier, referrer, kind)? - } else { - deno_core::resolve_import(specifier, referrer)? - }; + let url: ModuleSpecifier = deno_core::resolve_import(specifier, referrer)?; + + #[cfg(feature = "custom_import")] + if let Some(import_provider) = self.inner.import_provider.as_ref().as_ref() { + let resolve_result = import_provider.borrow_mut().resolve(&url, referrer, kind); + + // ImportProvider's resolve method should return None if default resolution is preferred, and Some(Err) if a url is not allowed + if let Some(result) = resolve_result { + return result; + } + } if referrer == "." { self.whitelist_add(url.as_str()); @@ -169,12 +168,8 @@ impl ModuleLoader for RustyLoader { } _ => { - #[cfg(feature = "custom_import")] - // Custom imports can handle any URL scheme - return Ok(url); - #[cfg(not(feature = "custom_import"))] return Err(anyhow!( - "unrecognized schema for module import: {specifier}" + "unrecognized schema for module import: {specifier}" )); } } @@ -229,7 +224,8 @@ impl ModuleLoader for RustyLoader { async move { inner .load(module_specifier, |specifier| { - let import_provider = inner.import_provider.as_ref().as_ref().unwrap(); + let import_provider = + inner.import_provider.as_ref().as_ref().unwrap(); let maybe_referrer = maybe_referrer.clone(); let requested_module_type = requested_module_type.clone(); async move { From 8b0268cd3f4729eb8895e790015de145d338eb1b Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:30:48 -0700 Subject: [PATCH 04/25] WIP Avoid unneccessary clone. Use Rc instead? --- examples/custom_import_schemes.rs | 4 ++-- src/module_loader.rs | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/custom_import_schemes.rs b/examples/custom_import_schemes.rs index f65f05f2..40170862 100644 --- a/examples/custom_import_schemes.rs +++ b/examples/custom_import_schemes.rs @@ -14,8 +14,8 @@ impl MyImportProvider { impl ImportProvider for MyImportProvider { fn import( &mut self, - specifier: ModuleSpecifier, - _referrer: Option, + specifier: &ModuleSpecifier, + _referrer: &Option, _is_dyn_import: bool, _requested_module_type: deno_core::RequestedModuleType, ) -> Result { diff --git a/src/module_loader.rs b/src/module_loader.rs index 475f41a2..d7c356b7 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -26,8 +26,8 @@ pub trait ImportProvider { } fn import( &mut self, - specifier: ModuleSpecifier, - referrer: Option, + specifier: &ModuleSpecifier, + referrer: &Option, is_dyn_import: bool, requested_module_type: deno_core::RequestedModuleType, ) -> Result { @@ -185,7 +185,6 @@ impl ModuleLoader for RustyLoader { ) -> deno_core::ModuleLoadResponse { let inner = self.inner.clone(); let module_specifier = module_specifier.clone(); - let maybe_referrer = maybe_referrer.cloned(); // We check permissions first match module_specifier.scheme() { // Remote fetch imports @@ -220,17 +219,18 @@ impl ModuleLoader for RustyLoader { _ => { #[cfg(feature = "custom_import")] if inner.import_provider.is_some() { + let maybe_referrer = Rc::new(maybe_referrer.cloned()); return ModuleLoadResponse::Async( async move { inner .load(module_specifier, |specifier| { let import_provider = inner.import_provider.as_ref().as_ref().unwrap(); - let maybe_referrer = maybe_referrer.clone(); + let maybe_referrer = maybe_referrer.as_ref(); let requested_module_type = requested_module_type.clone(); async move { import_provider.borrow_mut().import( - specifier, + &specifier, maybe_referrer, is_dyn_import, requested_module_type, From 4c2f9f7ebe8a471fbe74fc34f6a621234f95996c Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:34:14 -0700 Subject: [PATCH 05/25] WIP Replace match statement with .map() to reduce clutter. --- src/module_loader.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/module_loader.rs b/src/module_loader.rs index d7c356b7..0a2e100e 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -54,10 +54,7 @@ impl InnerRustyLoader { ) -> Self { Self { cache_provider: Rc::new(cache_provider), - import_provider: Rc::new(match import_provider { - Some(p) => Some(RefCell::new(p)), - None => None, - }), + import_provider: Rc::new(import_provider.map(RefCell::new)), fs_whlist: Rc::new(RefCell::new(HashSet::new())), source_map_cache: Rc::new(RefCell::new(SourceMapCache::new())), } From caa643f208e03373554c80193fd6b728e77b26e1 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:38:21 -0700 Subject: [PATCH 06/25] WIP Change ImportProvider's feature flag name. --- Cargo.toml | 6 +++--- src/lib.rs | 2 +- src/module_loader.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0b51ae11..07479d8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ web = ["console", "url", "crypto", "deno_web", "deno_tls", "deno_fetch", "url_im # Features for the module loader fs_import = [] url_import = ["reqwest"] -custom_import = [] +import_provider = [] # Enables the use of the SnapshotBuilder snapshot_builder = [] @@ -87,11 +87,11 @@ required-features = ["fs_import", "url_import"] [[example]] name = "custom_import_resolution" -required-features = ["custom_import", "url_import"] +required-features = ["import_provider", "url_import"] [[example]] name = "custom_import_schemes" -required-features = ["custom_import", "url_import"] +required-features = ["import_provider", "url_import"] [[example]] name = "async_javascript" diff --git a/src/lib.rs b/src/lib.rs index b6611375..7b3902fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -239,7 +239,7 @@ pub use inner_runtime::{FunctionArguments, RsAsyncFunction, RsFunction}; pub use module::{Module, StaticModule}; pub use module_handle::ModuleHandle; pub use module_wrapper::ModuleWrapper; -#[cfg(feature = "custom_import")] +#[cfg(feature = "import_provider")] pub use module_loader::ImportProvider; pub use runtime::{Runtime, RuntimeOptions, Undefined}; pub use utilities::{evaluate, import, resolve_path, validate}; diff --git a/src/module_loader.rs b/src/module_loader.rs index 0a2e100e..5b84693a 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -129,7 +129,7 @@ impl ModuleLoader for RustyLoader { ) -> Result { let url: ModuleSpecifier = deno_core::resolve_import(specifier, referrer)?; - #[cfg(feature = "custom_import")] + #[cfg(feature = "import_provider")] if let Some(import_provider) = self.inner.import_provider.as_ref().as_ref() { let resolve_result = import_provider.borrow_mut().resolve(&url, referrer, kind); @@ -214,7 +214,7 @@ impl ModuleLoader for RustyLoader { ), _ => { - #[cfg(feature = "custom_import")] + #[cfg(feature = "import_provider")] if inner.import_provider.is_some() { let maybe_referrer = Rc::new(maybe_referrer.cloned()); return ModuleLoadResponse::Async( From 305f3f461e4b1385f132923a0768c0318e0d8fa2 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Mon, 1 Jul 2024 21:55:10 -0700 Subject: [PATCH 07/25] WIP Change ImportProvider's import method to return option. --- examples/custom_import_schemes.rs | 38 +++++++++++++++++++++---------- src/module_loader.rs | 19 +++++++++++----- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/examples/custom_import_schemes.rs b/examples/custom_import_schemes.rs index 40170862..d0e14097 100644 --- a/examples/custom_import_schemes.rs +++ b/examples/custom_import_schemes.rs @@ -1,4 +1,4 @@ -use deno_core::{anyhow::Error, ModuleSpecifier}; +use deno_core::ModuleSpecifier; use rustyscript::{ImportProvider, Runtime}; struct MyImportProvider { @@ -12,29 +12,43 @@ impl MyImportProvider { } } impl ImportProvider for MyImportProvider { - fn import( + fn resolve( &mut self, specifier: &ModuleSpecifier, - _referrer: &Option, - _is_dyn_import: bool, - _requested_module_type: deno_core::RequestedModuleType, - ) -> Result { + _referrer: &str, + _kind: deno_core::ResolutionKind, + ) -> Option> { + // Provide a resolution only if our custom URL scheme is used if specifier.scheme() == "examplescheme" { - // Load code from anywhere - Ok(self.example_code.clone()) + Some(Ok(specifier.clone())) } else { - // Make sure to handle unrecognized schemes as well - Err(Error::msg(format!("unrecognized scheme for module import: {specifier}"))) + None + } + } + fn import( + &mut self, + specifier: &ModuleSpecifier, + _referrer: &Option, + _is_dyn_import: bool, + _requested_module_type: deno_core::RequestedModuleType, + ) -> Option> { + // Load module from example_code for this one scheme, but otherwise return None + if specifier.scheme() == "examplescheme" { + Some(Ok(self.example_code.clone())) + } else { + None } } } fn main() { let options = rustyscript::RuntimeOptions { - import_provider: Some(Box::new(MyImportProvider::new(" + import_provider: Some(Box::new(MyImportProvider::new( + " export function return_number(n) { return n } - "))), + ", + ))), ..Default::default() }; let mut runtime = Runtime::new(options).expect("Could not create runtime"); diff --git a/src/module_loader.rs b/src/module_loader.rs index 5b84693a..60358d7d 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -30,10 +30,8 @@ pub trait ImportProvider { referrer: &Option, is_dyn_import: bool, requested_module_type: deno_core::RequestedModuleType, - ) -> Result { - return Err(anyhow!( - "unrecognized schema for module import: {specifier}" - )); + ) -> Option< Result > { + None } } @@ -226,12 +224,21 @@ impl ModuleLoader for RustyLoader { let maybe_referrer = maybe_referrer.as_ref(); let requested_module_type = requested_module_type.clone(); async move { - import_provider.borrow_mut().import( + let import = import_provider.borrow_mut().import( &specifier, maybe_referrer, is_dyn_import, requested_module_type, - ) + ); + if let Some(import) = import { + import + } else { + Err(anyhow!( + "{} imports are not allowed here: {}", + specifier.scheme(), + specifier.as_str() + )) + } } }) .await From 0ed9a1a3eb5cf50192de2f3c4cef2cb0447ae68a Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Mon, 1 Jul 2024 22:38:47 -0700 Subject: [PATCH 08/25] WIP Add test_import_provider to existing module loader tests. --- src/module_loader.rs | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/module_loader.rs b/src/module_loader.rs index 60358d7d..c4589588 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -301,6 +301,8 @@ impl SourceMapGetter for RustyLoader { #[cfg(test)] mod test { + use deno_core::ResolutionKind; + use super::*; use crate::{ cache_provider::{ClonableSource, MemoryModuleCacheProvider}, @@ -349,4 +351,75 @@ mod test { _ => panic!("Unexpected response"), } } + + #[cfg(feature = "import_provider")] + struct TestImportProvider { i: usize } + impl TestImportProvider { + fn new() -> Self { + Self { i: 0 } + } + } + impl ImportProvider for TestImportProvider { + fn resolve( + &mut self, + specifier: &ModuleSpecifier, + _referrer: &str, + _kind: deno_core::ResolutionKind, + ) -> Option> { + match specifier.scheme() { + "test" => { + self.i += 1; + Some(Ok(ModuleSpecifier::parse(&format!("test://{}", self.i)).unwrap())) + }, + _ => None, + } + } + fn import( + &mut self, + specifier: &ModuleSpecifier, + _referrer: &Option, + _is_dyn_import: bool, + _requested_module_type: deno_core::RequestedModuleType, + ) -> Option> { + match specifier.as_str() { + "test://1" => Some(Ok("console.log('Rock')".to_string())), + "test://2" => Some(Ok("console.log('Paper')".to_string())), + "test://3" => Some(Ok("console.log('Scissors')".to_string())), + _ => None, + } + } + } + + #[tokio::test] + #[cfg(feature = "import_provider")] + async fn test_import_provider() { + let loader = RustyLoader::new(None, Some(Box::new(TestImportProvider::new()))); + let expected_responses = vec![ + "console.log('Rock')".to_string(), + "console.log('Paper')".to_string(), + "console.log('Scissors')".to_string(), + ]; + for i in 0..3 { + let specifier = loader.resolve("test://anything", "", ResolutionKind::Import).unwrap(); + let response = loader.load( + &specifier, + None, + false, + deno_core::RequestedModuleType::None, + ); + match response { + ModuleLoadResponse::Async(future) => { + let source = future.await.expect("Expected to get source"); + let source = if let ModuleSourceCode::String(s) = source.code { + s + } else { + panic!("Unexpected source code type"); + }; + assert_eq!(source, expected_responses[i].clone().into()); + } + _ => panic!("Unexpected response"), + } + } + } + } From c96f36625be23c4e0ed3f5db3a0bd5f05c1a2ad3 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Mon, 1 Jul 2024 22:44:42 -0700 Subject: [PATCH 09/25] WIP add tentative doc comments to ImportProvider. --- src/module_loader.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/module_loader.rs b/src/module_loader.rs index c4589588..ff1cc1e7 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -15,7 +15,9 @@ use std::{ }; #[allow(unused_variables)] +/// A trait that can be implemented to provide custom import resolution. Passed to the runtime via `RuntimeOptions::import_provider` pub trait ImportProvider { + /// Resolve an import statement's specifier to a URL to later be imported fn resolve( &mut self, specifier: &ModuleSpecifier, @@ -24,6 +26,7 @@ pub trait ImportProvider { ) -> Option> { None } + /// Retrieve a JavaScript/TypeScript module from a given URL and return it as a string. fn import( &mut self, specifier: &ModuleSpecifier, From df2c58d20a78b3c667b18f269cfa1a3fa4e32ece Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Mon, 1 Jul 2024 22:50:23 -0700 Subject: [PATCH 10/25] WIP Fix feature flags on ImportProvider test. --- src/module_loader.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/module_loader.rs b/src/module_loader.rs index ff1cc1e7..401c7b0f 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -357,11 +357,13 @@ mod test { #[cfg(feature = "import_provider")] struct TestImportProvider { i: usize } + #[cfg(feature = "import_provider")] impl TestImportProvider { fn new() -> Self { Self { i: 0 } } } + #[cfg(feature = "import_provider")] impl ImportProvider for TestImportProvider { fn resolve( &mut self, From 7da17d4c08aef3fa1b89b209851cb104fb54ed8c Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Sun, 14 Jul 2024 15:33:51 -0700 Subject: [PATCH 11/25] WIP Improve comments in custom import example. --- examples/custom_import_schemes.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/examples/custom_import_schemes.rs b/examples/custom_import_schemes.rs index d0e14097..68b6d460 100644 --- a/examples/custom_import_schemes.rs +++ b/examples/custom_import_schemes.rs @@ -13,15 +13,17 @@ impl MyImportProvider { } impl ImportProvider for MyImportProvider { fn resolve( - &mut self, - specifier: &ModuleSpecifier, - _referrer: &str, - _kind: deno_core::ResolutionKind, - ) -> Option> { - // Provide a resolution only if our custom URL scheme is used + &mut self, + specifier: &ModuleSpecifier, + _referrer: &str, + _kind: deno_core::ResolutionKind, + ) -> Option> { if specifier.scheme() == "examplescheme" { + // Return Some(Ok) without modifying the returned specifier in order to allow this kind of import Some(Ok(specifier.clone())) } else { + // Return None in order to allow rustyscript to decide the specifier's validity + // If you wish to disallow a particular resolution, Some(Error) can be returned instead None } } @@ -32,10 +34,14 @@ impl ImportProvider for MyImportProvider { _is_dyn_import: bool, _requested_module_type: deno_core::RequestedModuleType, ) -> Option> { - // Load module from example_code for this one scheme, but otherwise return None + // If the url_import or fs_import features are enabled, this method is only called if + // rustyscript doesn't have a built-in importer that matches the specifier. if specifier.scheme() == "examplescheme" { + // Return a string containing the desired module code Some(Ok(self.example_code.clone())) } else { + // When None is returned from this method, rustyscript will not allow the import, even + // if the resolution was valid. None } } From c01fa892bd4f1cb3fb601a4223fbe8890f93826d Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Sun, 14 Jul 2024 15:34:28 -0700 Subject: [PATCH 12/25] WIP ImportProvider test formatting. --- src/module_loader.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/module_loader.rs b/src/module_loader.rs index 401c7b0f..64b588b4 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -33,7 +33,7 @@ pub trait ImportProvider { referrer: &Option, is_dyn_import: bool, requested_module_type: deno_core::RequestedModuleType, - ) -> Option< Result > { + ) -> Option> { None } } @@ -354,9 +354,11 @@ mod test { _ => panic!("Unexpected response"), } } - + #[cfg(feature = "import_provider")] - struct TestImportProvider { i: usize } + struct TestImportProvider { + i: usize, + } #[cfg(feature = "import_provider")] impl TestImportProvider { fn new() -> Self { @@ -374,8 +376,10 @@ mod test { match specifier.scheme() { "test" => { self.i += 1; - Some(Ok(ModuleSpecifier::parse(&format!("test://{}", self.i)).unwrap())) - }, + Some(Ok( + ModuleSpecifier::parse(&format!("test://{}", self.i)).unwrap() + )) + } _ => None, } } @@ -394,7 +398,7 @@ mod test { } } } - + #[tokio::test] #[cfg(feature = "import_provider")] async fn test_import_provider() { @@ -405,7 +409,9 @@ mod test { "console.log('Scissors')".to_string(), ]; for i in 0..3 { - let specifier = loader.resolve("test://anything", "", ResolutionKind::Import).unwrap(); + let specifier = loader + .resolve("test://anything", "", ResolutionKind::Import) + .unwrap(); let response = loader.load( &specifier, None, @@ -426,5 +432,4 @@ mod test { } } } - } From 1b9c44b2d02a3a309314c067dcdeca198cb47ec7 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Sun, 14 Jul 2024 15:40:35 -0700 Subject: [PATCH 13/25] WIP Add clarifying comment to import resolution example. --- examples/custom_import_resolution.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/custom_import_resolution.rs b/examples/custom_import_resolution.rs index 990f2ed0..2e48629a 100644 --- a/examples/custom_import_resolution.rs +++ b/examples/custom_import_resolution.rs @@ -25,6 +25,7 @@ impl ImportProvider for MyImportProvider { // Or fall back to the default resolve behavior None } + // You can also return Some(Error) here if you wish to disallow a particular resolution } } fn main() { From 7d0651d2645ada9d685133a86f1dff5372d68bd3 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Sun, 14 Jul 2024 16:34:40 -0700 Subject: [PATCH 14/25] WIP Format custom import example. --- examples/custom_import_resolution.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/custom_import_resolution.rs b/examples/custom_import_resolution.rs index 2e48629a..fbacd81c 100644 --- a/examples/custom_import_resolution.rs +++ b/examples/custom_import_resolution.rs @@ -30,7 +30,9 @@ impl ImportProvider for MyImportProvider { } fn main() { let options = rustyscript::RuntimeOptions { - import_provider: Some(Box::new(MyImportProvider::new("https://deno.land/std@0.224.0/assert/mod.ts"))), + import_provider: Some(Box::new(MyImportProvider::new( + "https://deno.land/std@0.224.0/assert/mod.ts", + ))), ..Default::default() }; let mut runtime = Runtime::new(options).expect("Could not create runtime"); From 1609a01c4074a9d42b9b30f96b82dbdf8a7828de Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Sun, 14 Jul 2024 17:14:52 -0700 Subject: [PATCH 15/25] WIP FIx incorrect function call in snapshot builder code. --- src/snapshot_builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snapshot_builder.rs b/src/snapshot_builder.rs index 2675c600..a4f68a38 100644 --- a/src/snapshot_builder.rs +++ b/src/snapshot_builder.rs @@ -55,7 +55,7 @@ pub struct SnapshotBuilder { impl SnapshotBuilder { /// Creates a new snapshot builder with the given options pub fn new(options: InnerRuntimeOptions) -> Result { - let loader = Rc::new(RustyLoader::new(options.module_cache)); + let loader = Rc::new(RustyLoader::new(options.module_cache, options.import_provider)); // If a snapshot is provided, do not reload ops let extensions = if options.startup_snapshot.is_some() { From fdf20d3698d779afd47611b552bc575b1d32a870 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Sun, 14 Jul 2024 19:13:16 -0700 Subject: [PATCH 16/25] WIP Fix formatting. --- src/inner_runtime.rs | 7 +++++-- src/lib.rs | 2 +- src/module_loader.rs | 2 +- src/snapshot_builder.rs | 5 ++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/inner_runtime.rs b/src/inner_runtime.rs index c47f6cad..eaffa44b 100644 --- a/src/inner_runtime.rs +++ b/src/inner_runtime.rs @@ -54,7 +54,7 @@ pub struct InnerRuntimeOptions { /// Optional cache provider for the module loader pub module_cache: Option>, - + /// Optional import provider for the module loader pub import_provider: Option>, @@ -104,7 +104,10 @@ pub struct InnerRuntime { } impl InnerRuntime { pub fn new(options: InnerRuntimeOptions) -> Result { - let loader = Rc::new(RustyLoader::new(options.module_cache, options.import_provider)); + let loader = Rc::new(RustyLoader::new( + options.module_cache, + options.import_provider, + )); // If a snapshot is provided, do not reload ops let extensions = if options.startup_snapshot.is_some() { diff --git a/src/lib.rs b/src/lib.rs index 5227e070..87363231 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -291,9 +291,9 @@ pub use error::Error; pub use inner_runtime::{FunctionArguments, RsAsyncFunction, RsFunction}; pub use module::{Module, StaticModule}; pub use module_handle::ModuleHandle; -pub use module_wrapper::ModuleWrapper; #[cfg(feature = "import_provider")] pub use module_loader::ImportProvider; +pub use module_wrapper::ModuleWrapper; pub use runtime::{Runtime, RuntimeOptions, Undefined}; pub use utilities::{evaluate, import, init_platform, resolve_path, validate}; diff --git a/src/module_loader.rs b/src/module_loader.rs index 7b18727a..49008e8f 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -52,7 +52,7 @@ struct InnerRustyLoader { impl InnerRustyLoader { /// Creates a new instance of InnerRustyLoader - /// An optional cache provider can be provided to manage module code caching, as well as an import provider to manage module resolution. + /// An optional cache provider can be provided to manage module code caching, as well as an import provider to manage module resolution. fn new( cache_provider: Option>, import_provider: Option>, diff --git a/src/snapshot_builder.rs b/src/snapshot_builder.rs index a4f68a38..c38852e3 100644 --- a/src/snapshot_builder.rs +++ b/src/snapshot_builder.rs @@ -55,7 +55,10 @@ pub struct SnapshotBuilder { impl SnapshotBuilder { /// Creates a new snapshot builder with the given options pub fn new(options: InnerRuntimeOptions) -> Result { - let loader = Rc::new(RustyLoader::new(options.module_cache, options.import_provider)); + let loader = Rc::new(RustyLoader::new( + options.module_cache, + options.import_provider, + )); // If a snapshot is provided, do not reload ops let extensions = if options.startup_snapshot.is_some() { From 9adf4a2eeb50bd68946f0b098d17278f22ce68fd Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Thu, 18 Jul 2024 18:55:39 -0700 Subject: [PATCH 17/25] WIP Feature-gate ImportProvider struct member & use statement in test block. --- src/module_loader.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/module_loader.rs b/src/module_loader.rs index 49008e8f..35c01143 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -45,6 +45,7 @@ type SourceMapCache = HashMap>)>; #[derive(Clone)] struct InnerRustyLoader { cache_provider: Rc>>, + #[cfg(feature = "import_provider")] import_provider: Rc>>>, fs_whlist: Rc>>, source_map_cache: Rc>, @@ -59,6 +60,7 @@ impl InnerRustyLoader { ) -> Self { Self { cache_provider: Rc::new(cache_provider), + #[cfg(feature = "import_provider")] import_provider: Rc::new(import_provider.map(RefCell::new)), fs_whlist: Rc::new(RefCell::new(HashSet::new())), source_map_cache: Rc::new(RefCell::new(SourceMapCache::new())), @@ -354,8 +356,6 @@ impl SourceMapGetter for RustyLoader { #[cfg(test)] mod test { - use deno_core::ResolutionKind; - use super::*; use crate::{ cache_provider::{ClonableSource, MemoryModuleCacheProvider}, @@ -405,6 +405,9 @@ mod test { } } + #[cfg(feature = "import_provider")] + use deno_core::ResolutionKind; + #[cfg(feature = "import_provider")] struct TestImportProvider { i: usize, From dcb79634c9f04bbb70dabaf26365be1bbec361e7 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Thu, 18 Jul 2024 19:16:05 -0700 Subject: [PATCH 18/25] WIP Finish adding feature gates to internal references to ImportProvider. --- src/inner_runtime.rs | 7 +++++-- src/module_loader.rs | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/inner_runtime.rs b/src/inner_runtime.rs index eaffa44b..22858b57 100644 --- a/src/inner_runtime.rs +++ b/src/inner_runtime.rs @@ -1,7 +1,7 @@ use crate::{ cache_provider::ModuleCacheProvider, ext, - module_loader::{ImportProvider, RustyLoader}, + module_loader::RustyLoader, traits::{ToDefinedValue, ToModuleSpecifier, ToV8String}, transpiler::{self, transpile_extension}, Error, Module, ModuleHandle, @@ -56,7 +56,8 @@ pub struct InnerRuntimeOptions { pub module_cache: Option>, /// Optional import provider for the module loader - pub import_provider: Option>, + #[cfg(feature = "import_provider")] + pub import_provider: Option>, /// Optional snapshot to load into the runtime /// This will reduce load times, but requires the same extensions to be loaded @@ -81,6 +82,7 @@ impl Default for InnerRuntimeOptions { default_entrypoint: Default::default(), timeout: Duration::MAX, module_cache: None, + #[cfg(feature = "import_provider")] import_provider: None, startup_snapshot: None, isolate_params: None, @@ -106,6 +108,7 @@ impl InnerRuntime { pub fn new(options: InnerRuntimeOptions) -> Result { let loader = Rc::new(RustyLoader::new( options.module_cache, + #[cfg(feature = "import_provider")] options.import_provider, )); diff --git a/src/module_loader.rs b/src/module_loader.rs index 35c01143..01f7055d 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -15,6 +15,7 @@ use std::{ }; #[allow(unused_variables)] +#[cfg(feature = "import_provider")] /// A trait that can be implemented to provide custom import resolution. Passed to the runtime via `RuntimeOptions::import_provider` pub trait ImportProvider { /// Resolve an import statement's specifier to a URL to later be imported @@ -56,7 +57,7 @@ impl InnerRustyLoader { /// An optional cache provider can be provided to manage module code caching, as well as an import provider to manage module resolution. fn new( cache_provider: Option>, - import_provider: Option>, + #[cfg(feature = "import_provider")] import_provider: Option>, ) -> Self { Self { cache_provider: Rc::new(cache_provider), @@ -299,10 +300,14 @@ impl RustyLoader { /// An optional cache provider can be provided to manage module code caching, as well as an import provider to manage module resolution. pub fn new( cache_provider: Option>, - import_provider: Option>, + #[cfg(feature = "import_provider")] import_provider: Option>, ) -> Self { Self { - inner: Rc::new(InnerRustyLoader::new(cache_provider, import_provider)), + inner: Rc::new(InnerRustyLoader::new( + cache_provider, + #[cfg(feature = "import_provider")] + import_provider, + )), } } @@ -378,7 +383,11 @@ mod test { .get(&specifier) .expect("Expected to get cached source"); - let loader = RustyLoader::new(Some(Box::new(cache_provider)), None); + let loader = RustyLoader::new( + Some(Box::new(cache_provider)), + #[cfg(feature = "import_provider")] + None, + ); let response = loader.load( &specifier, None, From 2fc1907a87b8653116a2d10c19f292dd039efa07 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Thu, 18 Jul 2024 19:29:42 -0700 Subject: [PATCH 19/25] WIP Missed one. --- src/snapshot_builder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/snapshot_builder.rs b/src/snapshot_builder.rs index c38852e3..8dbb03da 100644 --- a/src/snapshot_builder.rs +++ b/src/snapshot_builder.rs @@ -57,6 +57,7 @@ impl SnapshotBuilder { pub fn new(options: InnerRuntimeOptions) -> Result { let loader = Rc::new(RustyLoader::new( options.module_cache, + #[cfg(feature = "import_provider")] options.import_provider, )); From 8b32539de7f4fc7cf9538fd72412ea93b50728c1 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Thu, 18 Jul 2024 21:23:11 -0700 Subject: [PATCH 20/25] WIP Fix duplicated lines from merge. --- src/module_loader.rs | 81 -------------------------------------------- 1 file changed, 81 deletions(-) diff --git a/src/module_loader.rs b/src/module_loader.rs index 4e55ba30..a2122d4a 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -495,85 +495,4 @@ mod test { } } } - - #[cfg(feature = "import_provider")] - use deno_core::ResolutionKind; - - #[cfg(feature = "import_provider")] - struct TestImportProvider { - i: usize, - } - #[cfg(feature = "import_provider")] - impl TestImportProvider { - fn new() -> Self { - Self { i: 0 } - } - } - #[cfg(feature = "import_provider")] - impl ImportProvider for TestImportProvider { - fn resolve( - &mut self, - specifier: &ModuleSpecifier, - _referrer: &str, - _kind: deno_core::ResolutionKind, - ) -> Option> { - match specifier.scheme() { - "test" => { - self.i += 1; - Some(Ok( - ModuleSpecifier::parse(&format!("test://{}", self.i)).unwrap() - )) - } - _ => None, - } - } - fn import( - &mut self, - specifier: &ModuleSpecifier, - _referrer: &Option, - _is_dyn_import: bool, - _requested_module_type: deno_core::RequestedModuleType, - ) -> Option> { - match specifier.as_str() { - "test://1" => Some(Ok("console.log('Rock')".to_string())), - "test://2" => Some(Ok("console.log('Paper')".to_string())), - "test://3" => Some(Ok("console.log('Scissors')".to_string())), - _ => None, - } - } - } - - #[tokio::test] - #[cfg(feature = "import_provider")] - async fn test_import_provider() { - let loader = RustyLoader::new(None, Some(Box::new(TestImportProvider::new()))); - let expected_responses = vec![ - "console.log('Rock')".to_string(), - "console.log('Paper')".to_string(), - "console.log('Scissors')".to_string(), - ]; - for i in 0..3 { - let specifier = loader - .resolve("test://anything", "", ResolutionKind::Import) - .unwrap(); - let response = loader.load( - &specifier, - None, - false, - deno_core::RequestedModuleType::None, - ); - match response { - ModuleLoadResponse::Async(future) => { - let source = future.await.expect("Expected to get source"); - let source = if let ModuleSourceCode::String(s) = source.code { - s - } else { - panic!("Unexpected source code type"); - }; - assert_eq!(source, expected_responses[i].clone().into()); - } - _ => panic!("Unexpected response"), - } - } - } } From 7e239e550de2c59f58e5c15ecf21587b7f3783c1 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Fri, 19 Jul 2024 19:42:33 -0700 Subject: [PATCH 21/25] WIP Remove unnecessary anyhow::Ok --- src/module_loader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module_loader.rs b/src/module_loader.rs index a2122d4a..4e301f07 100644 --- a/src/module_loader.rs +++ b/src/module_loader.rs @@ -3,7 +3,7 @@ use crate::{ transpiler, }; use deno_core::{ - anyhow::{self, anyhow, Ok}, + anyhow::{self, anyhow}, futures::FutureExt, ModuleLoadResponse, ModuleLoader, ModuleSource, ModuleSourceCode, ModuleSpecifier, ModuleType, SourceMapGetter, From 29da8683d5100bfb86c6a8f226ea28343c1ef66c Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Sat, 20 Jul 2024 08:24:03 -0700 Subject: [PATCH 22/25] WIP Correct comment regarding optional module loader features in Cargo.toml --- Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e318d51e..01ef4d77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,11 +64,14 @@ web_stub = ["webidl"] websocket = ["deno_websocket", "web"] # Features for the module loader -# Both will break sandboxing # - fs_import allows arbitrary file imports # - url_import allows importing from the web +# - import_provider allows custom handling of imports +# +# Both fs_import and url_import will break sandboxing fs_import = [] url_import = ["reqwest"] +# import_provider may break sandboxing depending on your implementation import_provider = [] # Enables the use of the SnapshotBuilder runtime From 0a801a206d574bce5dd2a06ff9cb3f299d5f3968 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Sun, 21 Jul 2024 07:55:58 -0700 Subject: [PATCH 23/25] Add import_provider feature to readme.md --- readme.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/readme.md b/readme.md index 6a25db5a..27cd6765 100644 --- a/readme.md +++ b/readme.md @@ -233,11 +233,14 @@ Please note that the `web` feature will also enable `fs_import` and `url_import` | | | | | |`fs_import` |Enables importing arbitrary code from the filesystem through JS |**NO** |None | |`url_import` |Enables importing arbitrary code from network locations through JS |**NO** |`reqwest` | +|`import_provider` |Enables importing arbitrary code from anywhere by implementing a trait |**NO**† |None | | | | | | |`worker` |Enables access to the threaded worker API [`worker`] |yes |None | |`snapshot_builder`|Enables access to [`SnapshotBuilder`], a runtime for creating snapshots that can improve start-times |yes |None | |`web_stub` |Enables a subset of `web` features that do not break sandboxing |yes |`deno_webidl` | +† Your implementation may still preserve sandboxing if you only allow importing JS/TS code that you trust. Exercise caution. + ---- Please also check out [@Bromeon/js_sandbox](https://github.com/Bromeon/js-sandbox), another great crate in this niche From fdb9d8f3b7bd2d608a8307a107846a555ca39105 Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Sun, 21 Jul 2024 08:07:57 -0700 Subject: [PATCH 24/25] Add import_provider feature to crate docs --- src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 83f294ba..0ffde8a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -240,11 +240,14 @@ //! | | | | | //! |`fs_import` |Enables importing arbitrary code from the filesystem through JS |**NO** |None | //! |`url_import` |Enables importing arbitrary code from network locations through JS |**NO** |`reqwest` | +//! |`import_provider` |Enables importing arbitrary code from anywhere by implementing a trait |**NO**† |None | //! | | | | | //! |`worker` |Enables access to the threaded worker API [`worker`] |yes |None | //! |`snapshot_builder`|Enables access to [`SnapshotBuilder`], a runtime for creating snapshots that can improve start-times |yes |None | //! |`web_stub` |Enables a subset of `web` features that do not break sandboxing |yes |`deno_webidl` | //! +//! † Your implementation may still preserve sandboxing if you only allow importing JS/TS code that you trust. Exercise caution. +//! //! ---- //! //! Please also check out [@Bromeon/js_sandbox](https://github.com/Bromeon/js-sandbox), another great crate in this niche From ce97a85664946d56fe5846b221ddc90a681883cb Mon Sep 17 00:00:00 2001 From: Amber Connelly <113668892+ac-z@users.noreply.github.com> Date: Sun, 21 Jul 2024 13:47:44 -0700 Subject: [PATCH 25/25] Make daggers superscript in doc comment for readability --- readme.md | 4 ++-- src/lib.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index 27cd6765..74bb94e0 100644 --- a/readme.md +++ b/readme.md @@ -233,13 +233,13 @@ Please note that the `web` feature will also enable `fs_import` and `url_import` | | | | | |`fs_import` |Enables importing arbitrary code from the filesystem through JS |**NO** |None | |`url_import` |Enables importing arbitrary code from network locations through JS |**NO** |`reqwest` | -|`import_provider` |Enables importing arbitrary code from anywhere by implementing a trait |**NO**† |None | +|`import_provider` |Enables importing arbitrary code from anywhere by implementing a trait |**NO**|None | | | | | | |`worker` |Enables access to the threaded worker API [`worker`] |yes |None | |`snapshot_builder`|Enables access to [`SnapshotBuilder`], a runtime for creating snapshots that can improve start-times |yes |None | |`web_stub` |Enables a subset of `web` features that do not break sandboxing |yes |`deno_webidl` | -† Your implementation may still preserve sandboxing if you only allow importing JS/TS code that you trust. Exercise caution. + Your implementation may still preserve sandboxing if you only allow importing JS/TS code that you trust. Exercise caution. ---- diff --git a/src/lib.rs b/src/lib.rs index 0ffde8a3..14b38617 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -240,13 +240,13 @@ //! | | | | | //! |`fs_import` |Enables importing arbitrary code from the filesystem through JS |**NO** |None | //! |`url_import` |Enables importing arbitrary code from network locations through JS |**NO** |`reqwest` | -//! |`import_provider` |Enables importing arbitrary code from anywhere by implementing a trait |**NO**† |None | +//! |`import_provider` |Enables importing arbitrary code from anywhere by implementing a trait |**NO**|None | //! | | | | | //! |`worker` |Enables access to the threaded worker API [`worker`] |yes |None | //! |`snapshot_builder`|Enables access to [`SnapshotBuilder`], a runtime for creating snapshots that can improve start-times |yes |None | //! |`web_stub` |Enables a subset of `web` features that do not break sandboxing |yes |`deno_webidl` | //! -//! † Your implementation may still preserve sandboxing if you only allow importing JS/TS code that you trust. Exercise caution. +//! Your implementation may still preserve sandboxing if you only allow importing JS/TS code that you trust. Exercise caution. //! //! ---- //!