Simplify src fetch logic and auto append suffix in cache search (#1322)

src/deno_dir.rs

@@ -1,6 +1,7 @@
 // Copyright 2018 the Deno authors. All rights reserved. MIT license.
 use dirs;
 use errors;
+use errors::DenoError;
 use errors::DenoResult;
 use errors::ErrorKind;
 use fs as deno_fs;
@@ -136,38 +137,79 @@ impl DenoDir {
   }
 
   // Prototype https://github.com/denoland/deno/blob/golang/deno_dir.go#L37-L73
+  /// Fetch remote source code.
   fn fetch_remote_source(
     self: &Self,
     module_name: &str,
     filename: &str,
-  ) -> DenoResult<(String, msg::MediaType)> {
+  ) -> DenoResult<Option<CodeFetchOutput>> {
-    let p = Path::new(filename);
+    let extensions = ["", ".ts", ".js"];
-    // We write a special ".mime" file into the `.deno/deps` directory along side the
+    for ext in extensions.iter() {
-    // cached file, containing just the media type.
+      let filename = [filename, ext].concat();
-    let mut media_type_filename = filename.to_string();
+      let module_name = [module_name, ext].concat();
-    media_type_filename.push_str(".mime");
+      let p = Path::new(&filename);
-    let mt = Path::new(&media_type_filename);
+      // We write a special ".mime" file into the `.deno/deps` directory along side the
+      // cached file, containing just the media type.
+      let media_type_filename = [&filename, ".mime"].concat();
+      let mt = Path::new(&media_type_filename);
+      eprint!("Downloading {}...", &module_name); // no newline
+      let maybe_source = http_util::fetch_sync_string(&module_name);
+      if let Ok((source, content_type)) = maybe_source {
+        eprintln!(""); // next line
+        match p.parent() {
+          Some(ref parent) => fs::create_dir_all(parent),
+          None => Ok(()),
+        }?;
+        deno_fs::write_file(&p, source.as_bytes(), 0o666)?;
+        deno_fs::write_file(&mt, content_type.as_bytes(), 0o666)?;
+        return Ok(Some(CodeFetchOutput {
+          module_name,
+          filename: filename.clone(), // TODO: no clone after NLL rfc
+          media_type: map_content_type(&p, Some(&content_type)),
+          source_code: source,
+          maybe_output_code: None,
+          maybe_source_map: None,
+        }));
+      } else {
+        eprintln!(" NOT FOUND");
+      }
+    }
+    Ok(None)
+  }
 
-    let src = if self.reload || !p.exists() {
+  /// Fetch local or cached source code.
-      eprintln!("Downloading {}", module_name);
+  fn fetch_local_source(
-      let (source, content_type) = http_util::fetch_sync_string(module_name)?;
+    self: &Self,
-      match p.parent() {
+    module_name: &str,
-        Some(ref parent) => fs::create_dir_all(parent),
+    filename: &str,
-        None => Ok(()),
+  ) -> DenoResult<Option<CodeFetchOutput>> {
-      }?;
+    let extensions = ["", ".ts", ".js"];
-      deno_fs::write_file(&p, source.as_bytes(), 0o666)?;
+    for ext in extensions.iter() {
-      deno_fs::write_file(&mt, content_type.as_bytes(), 0o666)?;
+      let filename = [filename, ext].concat();
-      (source, map_content_type(&p, Some(&content_type)))
+      let module_name = [module_name, ext].concat();
-    } else {
+      let p = Path::new(&filename);
-      let source = fs::read_to_string(&p)?;
+      if !p.exists() {
-      // .mime file might not exists with bundled deps
+        continue;
+      }
+      let media_type_filename = [&filename, ".mime"].concat();
+      let mt = Path::new(&media_type_filename);
+      let source_code = fs::read_to_string(&p)?;
+      // .mime file might not exists
+      // this is okay for local source: maybe_content_type_str will be None
       let maybe_content_type_string = fs::read_to_string(&mt).ok();
       // Option<String> -> Option<&str>
       let maybe_content_type_str =
         maybe_content_type_string.as_ref().map(String::as_str);
-      (source, map_content_type(&p, maybe_content_type_str))
+      return Ok(Some(CodeFetchOutput {
-    };
+        module_name,
-    Ok(src)
+        filename: filename.clone(), // TODO: no clone after NLL rfc
+        media_type: map_content_type(&p, maybe_content_type_str),
+        source_code,
+        maybe_output_code: None,
+        maybe_source_map: None,
+      }));
+    }
+    Ok(None) // cannot find locally
   }
 
   // Prototype: https://github.com/denoland/deno/blob/golang/os.go#L122-L138
@@ -177,39 +219,33 @@ impl DenoDir {
     filename: &str,
   ) -> DenoResult<CodeFetchOutput> {
     let is_module_remote = is_remote(module_name);
-    let use_extension = |ext| {
+    // We try fetch local. Two cases:
-      let module_name = format!("{}{}", module_name, ext);
+    // 1. This is a remote module, but no reload provided
-      let filename = format!("{}{}", filename, ext);
+    // 2. This is a local module
-      let (source_code, media_type) = if is_module_remote {
+    if !is_module_remote || !self.reload {
-        self.fetch_remote_source(&module_name, &filename)?
+      let maybe_local_source =
-      } else {
+        self.fetch_local_source(&module_name, &filename)?;
-        assert_eq!(
+      if let Some(output) = maybe_local_source {
-          module_name, filename,
+        return Ok(output);
-          "if a module isn't remote, it should have the same filename"
+      }
-        );
-        let path = Path::new(&filename);
-        (fs::read_to_string(path)?, map_content_type(path, None))
-      };
-      Ok(CodeFetchOutput {
-        module_name: module_name.to_string(),
-        filename: filename.to_string(),
-        media_type,
-        source_code,
-        maybe_output_code: None,
-        maybe_source_map: None,
-      })
-    };
-    let default_attempt = use_extension("");
-    if default_attempt.is_ok() {
-      return default_attempt;
     }
-    debug!("Trying {}.ts...", module_name);
+    // If not remote file, stop here!
-    let ts_attempt = use_extension(".ts");
+    if !is_module_remote {
-    if ts_attempt.is_ok() {
+      return Err(DenoError::from(std::io::Error::new(
-      return ts_attempt;
+        std::io::ErrorKind::NotFound,
+        format!("cannot find local file '{}'", filename),
+      )));
     }
-    debug!("Trying {}.js...", module_name);
+    // not cached/local, try remote
-    use_extension(".js")
+    let maybe_remote_source =
+      self.fetch_remote_source(&module_name, &filename)?;
+    if let Some(output) = maybe_remote_source {
+      return Ok(output);
+    }
+    return Err(DenoError::from(std::io::Error::new(
+      std::io::ErrorKind::NotFound,
+      format!("cannot find remote file '{}'", filename),
+    )));
   }
 
   pub fn code_fetch(
@@ -565,7 +601,72 @@ mod tests {
   }
 
   #[test]
-  fn test_fetch_remote_source_1() {
+  fn test_get_source_code() {
+    use tokio_util;
+    // http_util::fetch_sync_string requires tokio
+    tokio_util::init(|| {
+      let (temp_dir, deno_dir) = test_setup();
+      let module_name = "http://localhost:4545/tests/subdir/mod2.ts";
+      let filename = deno_fs::normalize_path(
+        deno_dir
+          .deps_http
+          .join("localhost_PORT4545/tests/subdir/mod2.ts")
+          .as_ref(),
+      );
+      let mime_file_name = format!("{}.mime", &filename);
+
+      let result = deno_dir.get_source_code(module_name, &filename);
+      assert!(result.is_ok());
+      let r = result.unwrap();
+      assert_eq!(
+        &(r.source_code),
+        "export { printHello } from \"./print_hello.ts\";\n"
+      );
+      assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
+      assert_eq!(
+        fs::read_to_string(&mime_file_name).unwrap(),
+        "application/typescript"
+      );
+
+      // Modify .mime
+      let _ = fs::write(&mime_file_name, "text/javascript");
+      let result2 = deno_dir.get_source_code(module_name, &filename);
+      assert!(result2.is_ok());
+      let r2 = result2.unwrap();
+      assert_eq!(
+        &(r2.source_code),
+        "export { printHello } from \"./print_hello.ts\";\n"
+      );
+      // If get_source_code does not call remote, this should be JavaScript
+      // as we modified before! (we do not overwrite .mime due to no http fetch)
+      assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
+      assert_eq!(
+        fs::read_to_string(&mime_file_name).unwrap(),
+        "text/javascript"
+      );
+
+      // Force self.reload
+      let deno_dir = DenoDir::new(true, Some(temp_dir.path().to_path_buf()))
+        .expect("setup fail");
+      let result3 = deno_dir.get_source_code(module_name, &filename);
+      assert!(result3.is_ok());
+      let r3 = result3.unwrap();
+      assert_eq!(
+        &(r3.source_code),
+        "export { printHello } from \"./print_hello.ts\";\n"
+      );
+      // Now the .mime file should be overwritten back to TypeScript!
+      // (due to http fetch)
+      assert_eq!(&(r3.media_type), &msg::MediaType::TypeScript);
+      assert_eq!(
+        fs::read_to_string(&mime_file_name).unwrap(),
+        "application/typescript"
+      );
+    });
+  }
+
+  #[test]
+  fn test_fetch_source_1() {
     use tokio_util;
     // http_util::fetch_sync_string requires tokio
     tokio_util::init(|| {
@@ -582,24 +683,24 @@ mod tests {
 
       let result = deno_dir.fetch_remote_source(module_name, &filename);
       assert!(result.is_ok());
-      let r = result.unwrap();
+      let r = result.unwrap().unwrap();
-      assert_eq!(&(r.0), "export const loaded = true;\n");
+      assert_eq!(&(r.source_code), "export const loaded = true;\n");
-      assert_eq!(&(r.1), &msg::MediaType::TypeScript);
+      assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
       assert_eq!(fs::read_to_string(&mime_file_name).unwrap(), "video/mp2t");
 
-      // Modify .mime, make sure still read from local
+      // Modify .mime, make sure read from local
       let _ = fs::write(&mime_file_name, "text/javascript");
-      let result2 = deno_dir.fetch_remote_source(module_name, &filename);
+      let result2 = deno_dir.fetch_local_source(module_name, &filename);
       assert!(result2.is_ok());
-      let r2 = result2.unwrap();
+      let r2 = result2.unwrap().unwrap();
-      assert_eq!(&(r2.0), "export const loaded = true;\n");
+      assert_eq!(&(r2.source_code), "export const loaded = true;\n");
       // Not MediaType::TypeScript due to .mime modification
-      assert_eq!(&(r2.1), &msg::MediaType::JavaScript);
+      assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
     });
   }
 
   #[test]
-  fn test_fetch_remote_source_2() {
+  fn test_fetch_source_2() {
     // only local, no http_util::fetch_sync_string called
     let (_temp_dir, deno_dir) = test_setup();
     let cwd = std::env::current_dir().unwrap();
@@ -608,11 +709,11 @@ mod tests {
     let filename =
       format!("{}/tests/subdir/mt_text_typescript.t1.ts", &cwd_string);
 
-    let result = deno_dir.fetch_remote_source(module_name, &filename);
+    let result = deno_dir.fetch_local_source(module_name, &filename);
     assert!(result.is_ok());
-    let r = result.unwrap();
+    let r = result.unwrap().unwrap();
-    assert_eq!(&(r.0), "export const loaded = true;\n");
+    assert_eq!(&(r.source_code), "export const loaded = true;\n");
-    assert_eq!(&(r.1), &msg::MediaType::TypeScript);
+    assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
   }
 
   #[test]