From 86b3ac5108e2893091475a0318fcba6c14e19140 Mon Sep 17 00:00:00 2001 From: Ry Dahl Date: Sun, 3 Nov 2019 10:39:27 -0500 Subject: [PATCH] feat: lockfiles (#3231) Use --lock-write=lock.json or --lock-check=lock.json on the command line. --- cli/checksum.rs | 20 ++++++++++ cli/compilers/ts.rs | 18 +-------- cli/flags.rs | 46 +++++++++++++++++++++- cli/lib.rs | 14 +++++++ cli/lockfile.rs | 70 ++++++++++++++++++++++++++++++++++ cli/state.rs | 41 ++++++++++++++++---- cli/tests/integration_tests.rs | 28 ++++++++++++++ cli/tests/lock_check_err.json | 4 ++ cli/tests/lock_check_err.out | 2 + cli/tests/lock_check_err2.json | 9 +++++ cli/tests/lock_check_err2.out | 2 + cli/tests/lock_check_ok.json | 4 ++ cli/tests/lock_check_ok2.json | 10 +++++ std/manual.md | 14 ++++++- 14 files changed, 255 insertions(+), 27 deletions(-) create mode 100644 cli/checksum.rs create mode 100644 cli/lockfile.rs create mode 100644 cli/tests/lock_check_err.json create mode 100644 cli/tests/lock_check_err.out create mode 100644 cli/tests/lock_check_err2.json create mode 100644 cli/tests/lock_check_err2.out create mode 100644 cli/tests/lock_check_ok.json create mode 100644 cli/tests/lock_check_ok2.json diff --git a/cli/checksum.rs b/cli/checksum.rs new file mode 100644 index 0000000000..f92fd62cff --- /dev/null +++ b/cli/checksum.rs @@ -0,0 +1,20 @@ +use ring; +use std::fmt::Write; + +pub fn gen(v: Vec<&[u8]>) -> String { + let mut ctx = ring::digest::Context::new(&ring::digest::SHA256); + for src in v.iter() { + ctx.update(src); + } + let digest = ctx.finish(); + let mut out = String::new(); + // TODO There must be a better way to do this... + for byte in digest.as_ref() { + write!(&mut out, "{:02x}", byte).unwrap(); + } + out +} + +pub fn gen2(s: &str) -> String { + gen(vec![s.as_bytes()]) +} diff --git a/cli/compilers/ts.rs b/cli/compilers/ts.rs index e38920820d..c2a4ccc4a7 100644 --- a/cli/compilers/ts.rs +++ b/cli/compilers/ts.rs @@ -18,9 +18,7 @@ use deno::ModuleSpecifier; use futures::Future; use futures::Stream; use regex::Regex; -use ring; use std::collections::HashSet; -use std::fmt::Write; use std::fs; use std::io; use std::path::PathBuf; @@ -178,20 +176,6 @@ fn req( j.to_string().into_boxed_str().into_boxed_bytes() } -fn gen_hash(v: Vec<&[u8]>) -> String { - let mut ctx = ring::digest::Context::new(&ring::digest::SHA256); - for src in v.iter() { - ctx.update(src); - } - let digest = ctx.finish(); - let mut out = String::new(); - // TODO There must be a better way to do this... - for byte in digest.as_ref() { - write!(&mut out, "{:02x}", byte).unwrap(); - } - out -} - /// Emit a SHA256 hash based on source code, deno version and TS config. /// Used to check if a recompilation for source code is needed. pub fn source_code_version_hash( @@ -199,7 +183,7 @@ pub fn source_code_version_hash( version: &str, config_hash: &[u8], ) -> String { - gen_hash(vec![source_code, version.as_bytes(), config_hash]) + crate::checksum::gen(vec![source_code, version.as_bytes(), config_hash]) } pub struct TsCompiler { diff --git a/cli/flags.rs b/cli/flags.rs index ca98873519..c6bb0a20f7 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -60,6 +60,9 @@ pub struct DenoFlags { pub v8_flags: Option>, // Use tokio::runtime::current_thread pub current_thread: bool, + + pub lock: Option, + pub lock_write: bool, } static ENV_VARIABLES_HELP: &str = "ENVIRONMENT VARIABLES: @@ -131,7 +134,7 @@ pub fn create_cli_app<'a, 'b>() -> App<'a, 'b> { .global_settings(&[AppSettings::ColorNever, AppSettings::UnifiedHelpMessage, AppSettings::DisableVersion]) .settings(&[AppSettings::AllowExternalSubcommands]) .after_help(ENV_VARIABLES_HELP) - .long_about("A secure runtime for JavaScript and TypeScript built with V8, Rust, and Tokio. + .long_about("A secure JavaScript and TypeScript runtime Docs: https://deno.land/manual.html Modules: https://deno.land/x/ @@ -143,7 +146,7 @@ To run the REPL: To execute a sandboxed script: - deno https://deno.land/welcome.ts + deno https://deno.land/std/examples/welcome.ts To evaluate code from the command line: @@ -223,6 +226,18 @@ Examples: https://github.com/WICG/import-maps#the-import-map", } }) .global(true), + ).arg( + Arg::with_name("lock") + .long("lock") + .value_name("FILE") + .help("Check the specified lock file") + .takes_value(true) + .global(true), + ).arg( + Arg::with_name("lock-write") + .long("lock-write") + .help("Write lock file. Use with --lock.") + .global(true), ).arg( Arg::with_name("v8-options") .long("v8-options") @@ -634,6 +649,13 @@ pub fn parse_flags( } } } + if matches.is_present("lock") { + let lockfile = matches.value_of("lock").unwrap(); + flags.lock = Some(lockfile.to_string()); + } + if matches.is_present("lock-write") { + flags.lock_write = true; + } flags = parse_run_args(flags, matches); // flags specific to "run" subcommand @@ -1890,4 +1912,24 @@ mod tests { assert_eq!(subcommand, DenoSubcommand::Run); assert_eq!(argv, svec!["deno", "script.ts"]) } + + #[test] + fn test_flags_from_vec_38() { + let (flags, subcommand, argv) = flags_from_vec(svec![ + "deno", + "--lock-write", + "--lock=lock.json", + "script.ts" + ]); + assert_eq!( + flags, + DenoFlags { + lock_write: true, + lock: Some("lock.json".to_string()), + ..DenoFlags::default() + } + ); + assert_eq!(subcommand, DenoSubcommand::Run); + assert_eq!(argv, svec!["deno", "script.ts"]) + } } diff --git a/cli/lib.rs b/cli/lib.rs index 5e416c6ac7..4f53195081 100644 --- a/cli/lib.rs +++ b/cli/lib.rs @@ -17,6 +17,7 @@ extern crate serde; extern crate serde_derive; extern crate url; +mod checksum; pub mod colors; pub mod compilers; pub mod deno_dir; @@ -32,6 +33,7 @@ mod http_body; mod http_util; mod import_map; mod js; +mod lockfile; pub mod msg; pub mod ops; pub mod permissions; @@ -362,6 +364,18 @@ fn run_script(flags: DenoFlags, argv: Vec) { worker .execute_mod_async(&main_module, None, false) + .and_then(move |()| { + if state.flags.lock_write { + if let Some(ref lockfile) = state.lockfile { + let g = lockfile.lock().unwrap(); + g.write()?; + } else { + eprintln!("--lock flag must be specified when using --lock-write"); + std::process::exit(11); + } + } + Ok(()) + }) .and_then(move |()| { js_check(worker.execute("window.dispatchEvent(new Event('load'))")); worker.then(move |result| { diff --git a/cli/lockfile.rs b/cli/lockfile.rs new file mode 100644 index 0000000000..f8700dac14 --- /dev/null +++ b/cli/lockfile.rs @@ -0,0 +1,70 @@ +use crate::compilers::CompiledModule; +use serde_json::json; +pub use serde_json::Value; +use std::collections::HashMap; +use std::io::Result; + +pub struct Lockfile { + need_read: bool, + map: HashMap, + pub filename: String, +} + +impl Lockfile { + pub fn new(filename: String) -> Lockfile { + Lockfile { + map: HashMap::new(), + filename, + need_read: true, + } + } + + pub fn write(&self) -> Result<()> { + let j = json!(self.map); + let s = serde_json::to_string_pretty(&j).unwrap(); + let mut f = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&self.filename)?; + use std::io::Write; + f.write_all(s.as_bytes())?; + debug!("lockfile write {}", self.filename); + Ok(()) + } + + pub fn read(&mut self) -> Result<()> { + debug!("lockfile read {}", self.filename); + let s = std::fs::read_to_string(&self.filename)?; + self.map = serde_json::from_str(&s)?; + self.need_read = false; + Ok(()) + } + + /// Lazily reads the filename, checks the given module is included. + /// Returns Ok(true) if check passed + pub fn check(&mut self, m: &CompiledModule) -> Result { + if m.name.starts_with("file:") { + return Ok(true); + } + if self.need_read { + self.read()?; + } + assert!(!self.need_read); + Ok(if let Some(lockfile_checksum) = self.map.get(&m.name) { + let compiled_checksum = crate::checksum::gen2(&m.code); + lockfile_checksum == &compiled_checksum + } else { + false + }) + } + + // Returns true if module was not already inserted. + pub fn insert(&mut self, m: &CompiledModule) -> bool { + if m.name.starts_with("file:") { + return false; + } + let checksum = crate::checksum::gen2(&m.code); + self.map.insert(m.name.clone(), checksum).is_none() + } +} diff --git a/cli/state.rs b/cli/state.rs index ca64e6d6c3..1f44f254f5 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -9,6 +9,7 @@ use crate::file_fetcher::SourceFileFetcher; use crate::flags; use crate::global_timer::GlobalTimer; use crate::import_map::ImportMap; +use crate::lockfile::Lockfile; use crate::msg; use crate::ops::JsonOp; use crate::permissions::DenoPermissions; @@ -88,6 +89,8 @@ pub struct State { pub ts_compiler: TsCompiler, pub include_deno_namespace: bool, + + pub lockfile: Option>, } impl Clone for ThreadSafeState { @@ -255,6 +258,13 @@ impl ThreadSafeState { let modules = Arc::new(Mutex::new(deno::Modules::new())); + // Note: reads lazily from disk on first call to lockfile.check() + let lockfile = if let Some(filename) = &flags.lock { + Some(Mutex::new(Lockfile::new(filename.to_string()))) + } else { + None + }; + let state = State { main_module, modules, @@ -276,6 +286,7 @@ impl ThreadSafeState { js_compiler: JsCompiler {}, json_compiler: JsonCompiler {}, include_deno_namespace, + lockfile, }; Ok(ThreadSafeState(Arc::new(state))) @@ -285,31 +296,47 @@ impl ThreadSafeState { self: &Self, module_specifier: &ModuleSpecifier, ) -> impl Future { - let state_ = self.clone(); + let state1 = self.clone(); + let state2 = self.clone(); self .file_fetcher .fetch_source_file_async(&module_specifier) .and_then(move |out| match out.media_type { msg::MediaType::Unknown => { - state_.js_compiler.compile_async(state_.clone(), &out) + state1.js_compiler.compile_async(state1.clone(), &out) } msg::MediaType::Json => { - state_.json_compiler.compile_async(state_.clone(), &out) + state1.json_compiler.compile_async(state1.clone(), &out) } msg::MediaType::TypeScript | msg::MediaType::TSX | msg::MediaType::JSX => { - state_.ts_compiler.compile_async(state_.clone(), &out) + state1.ts_compiler.compile_async(state1.clone(), &out) } msg::MediaType::JavaScript => { - if state_.ts_compiler.compile_js { - state_.ts_compiler.compile_async(state_.clone(), &out) + if state1.ts_compiler.compile_js { + state1.ts_compiler.compile_async(state1.clone(), &out) } else { - state_.js_compiler.compile_async(state_.clone(), &out) + state1.js_compiler.compile_async(state1.clone(), &out) } } }) + .and_then(move |compiled_module| { + if let Some(ref lockfile) = state2.lockfile { + let mut g = lockfile.lock().unwrap(); + if state2.flags.lock_write { + g.insert(&compiled_module); + } else if !g.check(&compiled_module)? { + eprintln!( + "Subresource integrety check failed --lock={}\n{}", + g.filename, compiled_module.name + ); + std::process::exit(10); + } + } + Ok(compiled_module) + }) } /// Read main module from argv diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 5032877167..dd9388014a 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -352,6 +352,34 @@ itest!(_050_more_jsons { output: "050_more_jsons.ts.out", }); +itest!(lock_check_ok { + args: "run --lock=lock_check_ok.json http://127.0.0.1:4545/cli/tests/003_relative_import.ts", + output: "003_relative_import.ts.out", + http_server: true, +}); + +itest!(lock_check_ok2 { + args: "run 019_media_types.ts --lock=lock_check_ok2.json", + output: "019_media_types.ts.out", + http_server: true, +}); + +itest!(lock_check_err { + args: "run --lock=lock_check_err.json http://127.0.0.1:4545/cli/tests/003_relative_import.ts", + output: "lock_check_err.out", + check_stderr: true, + exit_code: 10, + http_server: true, +}); + +itest!(lock_check_err2 { + args: "run 019_media_types.ts --lock=lock_check_err2.json", + output: "lock_check_err2.out", + check_stderr: true, + exit_code: 10, + http_server: true, +}); + itest!(async_error { exit_code: 1, args: "run --reload async_error.ts", diff --git a/cli/tests/lock_check_err.json b/cli/tests/lock_check_err.json new file mode 100644 index 0000000000..43fc532277 --- /dev/null +++ b/cli/tests/lock_check_err.json @@ -0,0 +1,4 @@ +{ + "http://127.0.0.1:4545/cli/tests/subdir/print_hello.ts": "5c93c66125878389f47f4abcac003f4be1276c5223612c26302460d71841e287", + "http://127.0.0.1:4545/cli/tests/003_relative_import.ts": "bad" +} diff --git a/cli/tests/lock_check_err.out b/cli/tests/lock_check_err.out new file mode 100644 index 0000000000..823df3183b --- /dev/null +++ b/cli/tests/lock_check_err.out @@ -0,0 +1,2 @@ +[WILDCARD]Subresource integrety check failed --lock=lock_check_err.json +http://127.0.0.1:4545/cli/tests/003_relative_import.ts diff --git a/cli/tests/lock_check_err2.json b/cli/tests/lock_check_err2.json new file mode 100644 index 0000000000..6958a72f4e --- /dev/null +++ b/cli/tests/lock_check_err2.json @@ -0,0 +1,9 @@ +{ + "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts": "c320ab0a259760e5c78b9ea840af3cc29697109594a3a5b5cea47128102b3e9d", + "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts": "42f66736fea7365ff17d5aa9b9655e8551eb81f360dcfb6b77acdd5c9f699e82", + "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts": "54cc82ff3c3b0387df57c7bb8eda4dcd36cbbf499ea483b04ff22c5365d34744", + "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts": "ee0b46f757b5f78681a4eead44820c2349daef7a5903fe3c624f29dbc98772e1" +} diff --git a/cli/tests/lock_check_err2.out b/cli/tests/lock_check_err2.out new file mode 100644 index 0000000000..fd8635a7f8 --- /dev/null +++ b/cli/tests/lock_check_err2.out @@ -0,0 +1,2 @@ +[WILDCARD]Subresource integrety check failed --lock=lock_check_err2.json +http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js diff --git a/cli/tests/lock_check_ok.json b/cli/tests/lock_check_ok.json new file mode 100644 index 0000000000..85670a5d64 --- /dev/null +++ b/cli/tests/lock_check_ok.json @@ -0,0 +1,4 @@ +{ + "http://127.0.0.1:4545/cli/tests/subdir/print_hello.ts": "5c93c66125878389f47f4abcac003f4be1276c5223612c26302460d71841e287", + "http://127.0.0.1:4545/cli/tests/003_relative_import.ts": "da3b7f60f5ff635dbc27f3e5e05420f0f2c34676f080ef935ea547116424adeb" +} diff --git a/cli/tests/lock_check_ok2.json b/cli/tests/lock_check_ok2.json new file mode 100644 index 0000000000..78fc75a9af --- /dev/null +++ b/cli/tests/lock_check_ok2.json @@ -0,0 +1,10 @@ +{ + "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts": "54cc82ff3c3b0387df57c7bb8eda4dcd36cbbf499ea483b04ff22c5365d34744", + "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts": "42f66736fea7365ff17d5aa9b9655e8551eb81f360dcfb6b77acdd5c9f699e82", + "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts": "ee0b46f757b5f78681a4eead44820c2349daef7a5903fe3c624f29dbc98772e1", + "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18", + "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts": "c320ab0a259760e5c78b9ea840af3cc29697109594a3a5b5cea47128102b3e9d" +} diff --git a/std/manual.md b/std/manual.md index 6679bf6b4c..0eb4fd64b4 100644 --- a/std/manual.md +++ b/std/manual.md @@ -574,6 +574,10 @@ always bundle its dependencies. In Deno this is done by checking the `$DENO_DIR` into your source control system, and specifying that path as the `$DENO_DIR` environmental variable at runtime. +**How can I trust a URL that may change** By using a lock file (using the +`--lock` command line flag) you can ensure you're running the code you expect to +be. + **How do you import to a specific version?** Simply specify the version in the URL. For example, this URL fully specifies the code being run: `https://unpkg.com/liltest@0.0.5/dist/liltest.js`. Combined with the @@ -667,7 +671,7 @@ Use `deno help` to see the help text. ``` deno -A secure runtime for JavaScript and TypeScript built with V8, Rust, and Tokio. +A secure JavaScript and TypeScript runtime Docs: https://deno.land/manual.html Modules: https://deno.land/x/ @@ -704,6 +708,8 @@ OPTIONS: --current-thread Use tokio::runtime::current_thread -h, --help Prints help information --importmap Load import map file + --lock Check the specified lock file + --lock-write Write lock file. Use with --lock. -L, --log-level Set log level [possible values: debug, info] --no-fetch Do not download remote modules -r, --reload= Reload source code cache (recompile TypeScript) @@ -905,6 +911,12 @@ Proxy configuration is read from environmental variables: `HTTP_PROXY` and In case of Windows if environmental variables are not found Deno falls back to reading proxies from registry. +## Lock file + +Deno can store and check module subresource integrity for modules using a small +JSON file. Use the `--lock=lock.json` to enable and specify lock file checking. +To update or create a lock use `--lock=lock.json --lock-write`. + ## Import maps Deno supports [import maps](https://github.com/WICG/import-maps).