feat: lockfiles (#3231)

Use --lock-write=lock.json or --lock-check=lock.json on the command
line.

cli/checksum.rs

@@ -0,0 +1,20 @@
+use ring;
+use std::fmt::Write;
+
+pub fn gen(v: Vec<&[u8]>) -> String {
+  let mut ctx = ring::digest::Context::new(&ring::digest::SHA256);
+  for src in v.iter() {
+    ctx.update(src);
+  }
+  let digest = ctx.finish();
+  let mut out = String::new();
+  // TODO There must be a better way to do this...
+  for byte in digest.as_ref() {
+    write!(&mut out, "{:02x}", byte).unwrap();
+  }
+  out
+}
+
+pub fn gen2(s: &str) -> String {
+  gen(vec![s.as_bytes()])
+}

cli/compilers/ts.rs

@@ -18,9 +18,7 @@ use deno::ModuleSpecifier;
 use futures::Future;
 use futures::Stream;
 use regex::Regex;
-use ring;
 use std::collections::HashSet;
-use std::fmt::Write;
 use std::fs;
 use std::io;
 use std::path::PathBuf;
@@ -178,20 +176,6 @@ fn req(
   j.to_string().into_boxed_str().into_boxed_bytes()
 }
 
-fn gen_hash(v: Vec<&[u8]>) -> String {
-  let mut ctx = ring::digest::Context::new(&ring::digest::SHA256);
-  for src in v.iter() {
-    ctx.update(src);
-  }
-  let digest = ctx.finish();
-  let mut out = String::new();
-  // TODO There must be a better way to do this...
-  for byte in digest.as_ref() {
-    write!(&mut out, "{:02x}", byte).unwrap();
-  }
-  out
-}
-
 /// Emit a SHA256 hash based on source code, deno version and TS config.
 /// Used to check if a recompilation for source code is needed.
 pub fn source_code_version_hash(
@@ -199,7 +183,7 @@ pub fn source_code_version_hash(
   version: &str,
   config_hash: &[u8],
 ) -> String {
-  gen_hash(vec![source_code, version.as_bytes(), config_hash])
+  crate::checksum::gen(vec![source_code, version.as_bytes(), config_hash])
 }
 
 pub struct TsCompiler {

cli/flags.rs

@@ -60,6 +60,9 @@ pub struct DenoFlags {
   pub v8_flags: Option<Vec<String>>,
   // Use tokio::runtime::current_thread
   pub current_thread: bool,
+
+  pub lock: Option<String>,
+  pub lock_write: bool,
 }
 
 static ENV_VARIABLES_HELP: &str = "ENVIRONMENT VARIABLES:
@@ -131,7 +134,7 @@ pub fn create_cli_app<'a, 'b>() -> App<'a, 'b> {
     .global_settings(&[AppSettings::ColorNever, AppSettings::UnifiedHelpMessage, AppSettings::DisableVersion])
     .settings(&[AppSettings::AllowExternalSubcommands])
     .after_help(ENV_VARIABLES_HELP)
-    .long_about("A secure runtime for JavaScript and TypeScript built with V8, Rust, and Tokio.
+    .long_about("A secure JavaScript and TypeScript runtime
 
 Docs: https://deno.land/manual.html
 Modules: https://deno.land/x/
@@ -143,7 +146,7 @@ To run the REPL:
 
 To execute a sandboxed script:
 
-  deno https://deno.land/welcome.ts
+  deno https://deno.land/std/examples/welcome.ts
 
 To evaluate code from the command line:
 
@@ -223,6 +226,18 @@ Examples: https://github.com/WICG/import-maps#the-import-map",
           }
         })
         .global(true),
+    ).arg(
+      Arg::with_name("lock")
+        .long("lock")
+        .value_name("FILE")
+        .help("Check the specified lock file")
+        .takes_value(true)
+        .global(true),
+    ).arg(
+      Arg::with_name("lock-write")
+        .long("lock-write")
+        .help("Write lock file. Use with --lock.")
+        .global(true),
     ).arg(
       Arg::with_name("v8-options")
         .long("v8-options")
@@ -634,6 +649,13 @@ pub fn parse_flags(
       }
     }
   }
+  if matches.is_present("lock") {
+    let lockfile = matches.value_of("lock").unwrap();
+    flags.lock = Some(lockfile.to_string());
+  }
+  if matches.is_present("lock-write") {
+    flags.lock_write = true;
+  }
 
   flags = parse_run_args(flags, matches);
   // flags specific to "run" subcommand
@@ -1890,4 +1912,24 @@ mod tests {
     assert_eq!(subcommand, DenoSubcommand::Run);
     assert_eq!(argv, svec!["deno", "script.ts"])
   }
+
+  #[test]
+  fn test_flags_from_vec_38() {
+    let (flags, subcommand, argv) = flags_from_vec(svec![
+      "deno",
+      "--lock-write",
+      "--lock=lock.json",
+      "script.ts"
+    ]);
+    assert_eq!(
+      flags,
+      DenoFlags {
+        lock_write: true,
+        lock: Some("lock.json".to_string()),
+        ..DenoFlags::default()
+      }
+    );
+    assert_eq!(subcommand, DenoSubcommand::Run);
+    assert_eq!(argv, svec!["deno", "script.ts"])
+  }
 }

cli/lib.rs

@@ -17,6 +17,7 @@ extern crate serde;
 extern crate serde_derive;
 extern crate url;
 
+mod checksum;
 pub mod colors;
 pub mod compilers;
 pub mod deno_dir;
@@ -32,6 +33,7 @@ mod http_body;
 mod http_util;
 mod import_map;
 mod js;
+mod lockfile;
 pub mod msg;
 pub mod ops;
 pub mod permissions;
@@ -362,6 +364,18 @@ fn run_script(flags: DenoFlags, argv: Vec<String>) {
 
     worker
       .execute_mod_async(&main_module, None, false)
+      .and_then(move |()| {
+        if state.flags.lock_write {
+          if let Some(ref lockfile) = state.lockfile {
+            let g = lockfile.lock().unwrap();
+            g.write()?;
+          } else {
+            eprintln!("--lock flag must be specified when using --lock-write");
+            std::process::exit(11);
+          }
+        }
+        Ok(())
+      })
       .and_then(move |()| {
         js_check(worker.execute("window.dispatchEvent(new Event('load'))"));
         worker.then(move |result| {

cli/lockfile.rs

@@ -0,0 +1,70 @@
+use crate::compilers::CompiledModule;
+use serde_json::json;
+pub use serde_json::Value;
+use std::collections::HashMap;
+use std::io::Result;
+
+pub struct Lockfile {
+  need_read: bool,
+  map: HashMap<String, String>,
+  pub filename: String,
+}
+
+impl Lockfile {
+  pub fn new(filename: String) -> Lockfile {
+    Lockfile {
+      map: HashMap::new(),
+      filename,
+      need_read: true,
+    }
+  }
+
+  pub fn write(&self) -> Result<()> {
+    let j = json!(self.map);
+    let s = serde_json::to_string_pretty(&j).unwrap();
+    let mut f = std::fs::OpenOptions::new()
+      .write(true)
+      .create(true)
+      .truncate(true)
+      .open(&self.filename)?;
+    use std::io::Write;
+    f.write_all(s.as_bytes())?;
+    debug!("lockfile write {}", self.filename);
+    Ok(())
+  }
+
+  pub fn read(&mut self) -> Result<()> {
+    debug!("lockfile read {}", self.filename);
+    let s = std::fs::read_to_string(&self.filename)?;
+    self.map = serde_json::from_str(&s)?;
+    self.need_read = false;
+    Ok(())
+  }
+
+  /// Lazily reads the filename, checks the given module is included.
+  /// Returns Ok(true) if check passed
+  pub fn check(&mut self, m: &CompiledModule) -> Result<bool> {
+    if m.name.starts_with("file:") {
+      return Ok(true);
+    }
+    if self.need_read {
+      self.read()?;
+    }
+    assert!(!self.need_read);
+    Ok(if let Some(lockfile_checksum) = self.map.get(&m.name) {
+      let compiled_checksum = crate::checksum::gen2(&m.code);
+      lockfile_checksum == &compiled_checksum
+    } else {
+      false
+    })
+  }
+
+  // Returns true if module was not already inserted.
+  pub fn insert(&mut self, m: &CompiledModule) -> bool {
+    if m.name.starts_with("file:") {
+      return false;
+    }
+    let checksum = crate::checksum::gen2(&m.code);
+    self.map.insert(m.name.clone(), checksum).is_none()
+  }
+}

cli/state.rs

@@ -9,6 +9,7 @@ use crate::file_fetcher::SourceFileFetcher;
 use crate::flags;
 use crate::global_timer::GlobalTimer;
 use crate::import_map::ImportMap;
+use crate::lockfile::Lockfile;
 use crate::msg;
 use crate::ops::JsonOp;
 use crate::permissions::DenoPermissions;
@@ -88,6 +89,8 @@ pub struct State {
   pub ts_compiler: TsCompiler,
 
   pub include_deno_namespace: bool,
+
+  pub lockfile: Option<Mutex<Lockfile>>,
 }
 
 impl Clone for ThreadSafeState {
@@ -255,6 +258,13 @@ impl ThreadSafeState {
 
     let modules = Arc::new(Mutex::new(deno::Modules::new()));
 
+    // Note: reads lazily from disk on first call to lockfile.check()
+    let lockfile = if let Some(filename) = &flags.lock {
+      Some(Mutex::new(Lockfile::new(filename.to_string())))
+    } else {
+      None
+    };
+
     let state = State {
       main_module,
       modules,
@@ -276,6 +286,7 @@ impl ThreadSafeState {
       js_compiler: JsCompiler {},
       json_compiler: JsonCompiler {},
       include_deno_namespace,
+      lockfile,
     };
 
     Ok(ThreadSafeState(Arc::new(state)))
@@ -285,31 +296,47 @@ impl ThreadSafeState {
     self: &Self,
     module_specifier: &ModuleSpecifier,
   ) -> impl Future<Item = CompiledModule, Error = ErrBox> {
-    let state_ = self.clone();
+    let state1 = self.clone();
+    let state2 = self.clone();
 
     self
       .file_fetcher
       .fetch_source_file_async(&module_specifier)
       .and_then(move |out| match out.media_type {
         msg::MediaType::Unknown => {
-          state_.js_compiler.compile_async(state_.clone(), &out)
+          state1.js_compiler.compile_async(state1.clone(), &out)
         }
         msg::MediaType::Json => {
-          state_.json_compiler.compile_async(state_.clone(), &out)
+          state1.json_compiler.compile_async(state1.clone(), &out)
         }
         msg::MediaType::TypeScript
         | msg::MediaType::TSX
         | msg::MediaType::JSX => {
-          state_.ts_compiler.compile_async(state_.clone(), &out)
+          state1.ts_compiler.compile_async(state1.clone(), &out)
         }
         msg::MediaType::JavaScript => {
-          if state_.ts_compiler.compile_js {
-            state_.ts_compiler.compile_async(state_.clone(), &out)
+          if state1.ts_compiler.compile_js {
+            state1.ts_compiler.compile_async(state1.clone(), &out)
           } else {
-            state_.js_compiler.compile_async(state_.clone(), &out)
+            state1.js_compiler.compile_async(state1.clone(), &out)
           }
         }
       })
+      .and_then(move |compiled_module| {
+        if let Some(ref lockfile) = state2.lockfile {
+          let mut g = lockfile.lock().unwrap();
+          if state2.flags.lock_write {
+            g.insert(&compiled_module);
+          } else if !g.check(&compiled_module)? {
+            eprintln!(
+              "Subresource integrety check failed --lock={}\n{}",
+              g.filename, compiled_module.name
+            );
+            std::process::exit(10);
+          }
+        }
+        Ok(compiled_module)
+      })
   }
 
   /// Read main module from argv

cli/tests/integration_tests.rs

@@ -352,6 +352,34 @@ itest!(_050_more_jsons {
   output: "050_more_jsons.ts.out",
 });
 
+itest!(lock_check_ok {
+  args: "run --lock=lock_check_ok.json http://127.0.0.1:4545/cli/tests/003_relative_import.ts",
+  output: "003_relative_import.ts.out",
+  http_server: true,
+});
+
+itest!(lock_check_ok2 {
+  args: "run 019_media_types.ts --lock=lock_check_ok2.json",
+  output: "019_media_types.ts.out",
+  http_server: true,
+});
+
+itest!(lock_check_err {
+  args: "run --lock=lock_check_err.json http://127.0.0.1:4545/cli/tests/003_relative_import.ts",
+  output: "lock_check_err.out",
+  check_stderr: true,
+  exit_code: 10,
+  http_server: true,
+});
+
+itest!(lock_check_err2 {
+  args: "run 019_media_types.ts --lock=lock_check_err2.json",
+  output: "lock_check_err2.out",
+  check_stderr: true,
+  exit_code: 10,
+  http_server: true,
+});
+
 itest!(async_error {
   exit_code: 1,
   args: "run --reload async_error.ts",

cli/tests/lock_check_err.json

@@ -0,0 +1,4 @@
+{
+  "http://127.0.0.1:4545/cli/tests/subdir/print_hello.ts": "5c93c66125878389f47f4abcac003f4be1276c5223612c26302460d71841e287",
+  "http://127.0.0.1:4545/cli/tests/003_relative_import.ts": "bad"
+}

cli/tests/lock_check_err.out

@@ -0,0 +1,2 @@
+[WILDCARD]Subresource integrety check failed --lock=lock_check_err.json
+http://127.0.0.1:4545/cli/tests/003_relative_import.ts

cli/tests/lock_check_err2.json

@@ -0,0 +1,9 @@
+{
+  "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18",
+  "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts": "c320ab0a259760e5c78b9ea840af3cc29697109594a3a5b5cea47128102b3e9d",
+  "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts": "42f66736fea7365ff17d5aa9b9655e8551eb81f360dcfb6b77acdd5c9f699e82",
+  "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts": "54cc82ff3c3b0387df57c7bb8eda4dcd36cbbf499ea483b04ff22c5365d34744",
+  "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18",
+  "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18",
+  "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts": "ee0b46f757b5f78681a4eead44820c2349daef7a5903fe3c624f29dbc98772e1"
+}

cli/tests/lock_check_err2.out

@@ -0,0 +1,2 @@
+[WILDCARD]Subresource integrety check failed --lock=lock_check_err2.json
+http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js

cli/tests/lock_check_ok.json

@@ -0,0 +1,4 @@
+{
+  "http://127.0.0.1:4545/cli/tests/subdir/print_hello.ts": "5c93c66125878389f47f4abcac003f4be1276c5223612c26302460d71841e287",
+  "http://127.0.0.1:4545/cli/tests/003_relative_import.ts": "da3b7f60f5ff635dbc27f3e5e05420f0f2c34676f080ef935ea547116424adeb"
+}

cli/tests/lock_check_ok2.json

@@ -0,0 +1,10 @@
+{
+  "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18",
+  "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18",
+  "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts": "54cc82ff3c3b0387df57c7bb8eda4dcd36cbbf499ea483b04ff22c5365d34744",
+  "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18",
+  "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts": "42f66736fea7365ff17d5aa9b9655e8551eb81f360dcfb6b77acdd5c9f699e82",
+  "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts": "ee0b46f757b5f78681a4eead44820c2349daef7a5903fe3c624f29dbc98772e1",
+  "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js": "3a3e002e2f92dc8f045bd4a7c66b4791453ad0417b038dd2b2d9d0f277c44f18",
+  "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts": "c320ab0a259760e5c78b9ea840af3cc29697109594a3a5b5cea47128102b3e9d"
+}

std/manual.md

@@ -574,6 +574,10 @@ always bundle its dependencies. In Deno this is done by checking the `$DENO_DIR`
 into your source control system, and specifying that path as the `$DENO_DIR`
 environmental variable at runtime.
 
+**How can I trust a URL that may change** By using a lock file (using the
+`--lock` command line flag) you can ensure you're running the code you expect to
+be.
+
 **How do you import to a specific version?** Simply specify the version in the
 URL. For example, this URL fully specifies the code being run:
 `https://unpkg.com/liltest@0.0.5/dist/liltest.js`. Combined with the
@@ -667,7 +671,7 @@ Use `deno help` to see the help text.
 
 ```
 deno
-A secure runtime for JavaScript and TypeScript built with V8, Rust, and Tokio.
+A secure JavaScript and TypeScript runtime
 
 Docs: https://deno.land/manual.html
 Modules: https://deno.land/x/
@@ -704,6 +708,8 @@ OPTIONS:
         --current-thread               Use tokio::runtime::current_thread
     -h, --help                         Prints help information
         --importmap <FILE>             Load import map file
+        --lock <FILE>                  Check the specified lock file
+        --lock-write                   Write lock file. Use with --lock.
     -L, --log-level <log-level>        Set log level [possible values: debug, info]
         --no-fetch                     Do not download remote modules
     -r, --reload=<CACHE_BLACKLIST>     Reload source code cache (recompile TypeScript)
@@ -905,6 +911,12 @@ Proxy configuration is read from environmental variables: `HTTP_PROXY` and
 In case of Windows if environmental variables are not found Deno falls back to
 reading proxies from registry.
 
+## Lock file
+
+Deno can store and check module subresource integrity for modules using a small
+JSON file. Use the `--lock=lock.json` to enable and specify lock file checking.
+To update or create a lock use `--lock=lock.json --lock-write`.
+
 ## Import maps
 
 Deno supports [import maps](https://github.com/WICG/import-maps).