From 8ef7da261149ed03f25bdb5ea2611f8ce84a4d78 Mon Sep 17 00:00:00 2001 From: Kitson Kelly Date: Mon, 22 Oct 2018 13:14:27 +1100 Subject: [PATCH] Enforce media types --- js/compiler.ts | 120 +++++++----- js/compiler_test.ts | 113 ++++++++++- js/os.ts | 22 ++- js/types.ts | 8 - src/deno_dir.rs | 178 +++++++++++++++++- src/http_util.rs | 24 ++- src/msg.fbs | 8 + src/ops.rs | 1 + tests/019_media_types.ts | 23 +++ tests/019_media_types.ts.out | 8 + tests/error_004_missing_module.ts.out | 2 +- tests/error_005_missing_dynamic_import.ts.out | 2 +- tests/error_006_import_ext_failure.ts.out | 2 +- tests/subdir/mt_application_ecmascript.j2.js | 3 + .../subdir/mt_application_x_javascript.j4.js | 3 + tests/subdir/mt_javascript.js | 3 + tests/subdir/mt_text_ecmascript.j3.js | 3 + tests/subdir/mt_text_javascript.j1.js | 3 + tests/subdir/mt_text_typescript.t1.ts | 1 + tests/subdir/mt_video_mp2t.t3.ts | 1 + tests/subdir/mt_video_vdn.t2.ts | 1 + tools/http_server.py | 26 ++- 22 files changed, 472 insertions(+), 83 deletions(-) create mode 100644 tests/019_media_types.ts create mode 100644 tests/019_media_types.ts.out create mode 100644 tests/subdir/mt_application_ecmascript.j2.js create mode 100644 tests/subdir/mt_application_x_javascript.j4.js create mode 100644 tests/subdir/mt_javascript.js create mode 100644 tests/subdir/mt_text_ecmascript.j3.js create mode 100644 tests/subdir/mt_text_javascript.j1.js create mode 100644 tests/subdir/mt_text_typescript.t1.ts create mode 100644 tests/subdir/mt_video_mp2t.t3.ts create mode 100644 tests/subdir/mt_video_vdn.t2.ts diff --git a/js/compiler.ts b/js/compiler.ts index e79df83ea1..c872afa147 100644 --- a/js/compiler.ts +++ b/js/compiler.ts @@ -1,5 +1,6 @@ // Copyright 2018 the Deno authors. All rights reserved. MIT license. import * as ts from "typescript"; +import { MediaType } from "gen/msg_generated"; import { assetSourceCode } from "./assets"; // tslint:disable-next-line:no-circular-imports import * as deno from "./deno"; @@ -85,6 +86,7 @@ export class ModuleMetaData implements ts.IScriptSnapshot { constructor( public readonly moduleId: ModuleId, public readonly fileName: ModuleFileName, + public readonly mediaType: MediaType, public readonly sourceCode: SourceCode = "", public outputCode: OutputCode = "" ) { @@ -107,6 +109,23 @@ export class ModuleMetaData implements ts.IScriptSnapshot { } } +function getExtension( + fileName: ModuleFileName, + mediaType: MediaType +): ts.Extension | undefined { + switch (mediaType) { + case MediaType.JavaScript: + return ts.Extension.Js; + case MediaType.TypeScript: + return fileName.endsWith(".d.ts") ? ts.Extension.Dts : ts.Extension.Ts; + case MediaType.Json: + return ts.Extension.Json; + case MediaType.Unknown: + default: + return undefined; + } +} + /** A singleton class that combines the TypeScript Language Service host API * with Deno specific APIs to provide an interface for compiling and running * TypeScript and JavaScript modules. @@ -319,17 +338,6 @@ export class DenoCompiler return undefined; } - /** Resolve the `fileName` for a given `moduleSpecifier` and - * `containingFile` - */ - private _resolveModuleName( - moduleSpecifier: ModuleSpecifier, - containingFile: ContainingFile - ): ModuleFileName | undefined { - const moduleMetaData = this.resolveModule(moduleSpecifier, containingFile); - return moduleMetaData ? moduleMetaData.fileName : undefined; - } - /** Caches the resolved `fileName` in relationship to the `moduleSpecifier` * and `containingFile` in order to reduce calls to the privileged side * to retrieve the contents of a module. @@ -479,9 +487,10 @@ export class DenoCompiler if (fileName && this._moduleMetaDataMap.has(fileName)) { return this._moduleMetaDataMap.get(fileName)!; } - let moduleId: ModuleId; - let sourceCode: SourceCode; - let outputCode: OutputCode | null; + let moduleId: ModuleId | undefined; + let mediaType = MediaType.Unknown; + let sourceCode: SourceCode | undefined; + let outputCode: OutputCode | undefined; if ( moduleSpecifier.startsWith(ASSETS) || containingFile.startsWith(ASSETS) @@ -492,6 +501,7 @@ export class DenoCompiler moduleId = moduleSpecifier.split("/").pop()!; const assetName = moduleId.includes(".") ? moduleId : `${moduleId}.d.ts`; assert(assetName in assetSourceCode, `No such asset "${assetName}"`); + mediaType = MediaType.TypeScript; sourceCode = assetSourceCode[assetName]; fileName = `${ASSETS}/${assetName}`; outputCode = ""; @@ -499,25 +509,38 @@ export class DenoCompiler // We query Rust with a CodeFetch message. It will load the sourceCode, // and if there is any outputCode cached, will return that as well. const fetchResponse = this._os.codeFetch(moduleSpecifier, containingFile); - moduleId = fetchResponse.moduleName!; - fileName = fetchResponse.filename!; - sourceCode = fetchResponse.sourceCode!; - outputCode = fetchResponse.outputCode!; + moduleId = fetchResponse.moduleName; + fileName = fetchResponse.filename; + mediaType = fetchResponse.mediaType; + sourceCode = fetchResponse.sourceCode; + outputCode = fetchResponse.outputCode; } - assert(sourceCode!.length > 0); - this._log("resolveModule sourceCode length:", sourceCode.length); - this._log("resolveModule has outputCode:", outputCode! != null); - this._setFileName(moduleSpecifier, containingFile, fileName); + assert(moduleId != null, "No module ID."); + assert(fileName != null, "No file name."); + assert(sourceCode ? sourceCode.length > 0 : false, "No source code."); + assert( + mediaType !== MediaType.Unknown, + `Unknown media type for: "${moduleSpecifier}" from "${containingFile}".` + ); + this._log( + "resolveModule sourceCode length:", + sourceCode && sourceCode.length + ); + this._log("resolveModule has outputCode:", outputCode != null); + this._log("resolveModule has media type:", MediaType[mediaType]); + // fileName is asserted above, but TypeScript does not track so not null + this._setFileName(moduleSpecifier, containingFile, fileName!); if (fileName && this._moduleMetaDataMap.has(fileName)) { return this._moduleMetaDataMap.get(fileName)!; } const moduleMetaData = new ModuleMetaData( - moduleId, - fileName, + moduleId!, + fileName!, + mediaType, sourceCode, outputCode ); - this._moduleMetaDataMap.set(fileName, moduleMetaData); + this._moduleMetaDataMap.set(fileName!, moduleMetaData); return moduleMetaData; } @@ -563,16 +586,20 @@ export class DenoCompiler getScriptKind(fileName: ModuleFileName): ts.ScriptKind { this._log("getScriptKind()", fileName); - const suffix = fileName.substr(fileName.lastIndexOf(".") + 1); - switch (suffix) { - case "ts": - return ts.ScriptKind.TS; - case "js": - return ts.ScriptKind.JS; - case "json": - return ts.ScriptKind.JSON; - default: - return this._options.allowJs ? ts.ScriptKind.JS : ts.ScriptKind.TS; + const moduleMetaData = this._getModuleMetaData(fileName); + if (moduleMetaData) { + switch (moduleMetaData.mediaType) { + case MediaType.TypeScript: + return ts.ScriptKind.TS; + case MediaType.JavaScript: + return ts.ScriptKind.JS; + case MediaType.Json: + return ts.ScriptKind.JSON; + default: + return this._options.allowJs ? ts.ScriptKind.JS : ts.ScriptKind.TS; + } + } else { + return this._options.allowJs ? ts.ScriptKind.JS : ts.ScriptKind.TS; } } @@ -619,17 +646,17 @@ export class DenoCompiler resolveModuleNames( moduleNames: ModuleSpecifier[], containingFile: ContainingFile - ): ts.ResolvedModule[] { + ): Array { this._log("resolveModuleNames()", { moduleNames, containingFile }); return moduleNames.map(name => { - let resolvedFileName; + let moduleMetaData: ModuleMetaData; if (name === "deno") { // builtin modules are part of the runtime lib - resolvedFileName = this._resolveModuleName(LIB_RUNTIME, ASSETS); + moduleMetaData = this.resolveModule(LIB_RUNTIME, ASSETS); } else if (name === "typescript") { - resolvedFileName = this._resolveModuleName("typescript.d.ts", ASSETS); + moduleMetaData = this.resolveModule("typescript.d.ts", ASSETS); } else { - resolvedFileName = this._resolveModuleName(name, containingFile); + moduleMetaData = this.resolveModule(name, containingFile); } // According to the interface we shouldn't return `undefined` but if we // fail to return the same length of modules to those we cannot resolve @@ -638,12 +665,15 @@ export class DenoCompiler // TODO: all this does is push the problem downstream, and TypeScript // will complain it can't identify the type of the file and throw // a runtime exception, so we need to handle missing modules better - resolvedFileName = resolvedFileName || ""; + const resolvedFileName = moduleMetaData.fileName || ""; // This flags to the compiler to not go looking to transpile functional // code, anything that is in `/$asset$/` is just library code const isExternalLibraryImport = resolvedFileName.startsWith(ASSETS); - // TODO: we should be returning a ts.ResolveModuleFull - return { resolvedFileName, isExternalLibraryImport }; + return { + resolvedFileName, + isExternalLibraryImport, + extension: getExtension(resolvedFileName, moduleMetaData.mediaType) + }; }); } @@ -662,9 +692,7 @@ export class DenoCompiler private static _instance: DenoCompiler | undefined; - /** - * Returns the instance of `DenoCompiler` or creates a new instance. - */ + /** Returns the instance of `DenoCompiler` or creates a new instance. */ static instance(): DenoCompiler { return ( DenoCompiler._instance || (DenoCompiler._instance = new DenoCompiler()) diff --git a/js/compiler_test.ts b/js/compiler_test.ts index f05a96e52b..7d1bdc150c 100644 --- a/js/compiler_test.ts +++ b/js/compiler_test.ts @@ -11,6 +11,7 @@ const { DenoCompiler } = (deno as any)._compiler; interface ModuleInfo { moduleName: string | null; filename: string | null; + mediaType: MediaType | null; sourceCode: string | null; outputCode: string | null; } @@ -27,15 +28,24 @@ const originals = { _window: (compilerInstance as any)._window }; +enum MediaType { + JavaScript = 0, + TypeScript = 1, + Json = 2, + Unknown = 3 +} + function mockModuleInfo( moduleName: string | null, filename: string | null, + mediaType: MediaType | null, sourceCode: string | null, outputCode: string | null ): ModuleInfo { return { moduleName, filename, + mediaType, sourceCode, outputCode }; @@ -61,6 +71,7 @@ export class A { const modAModuleInfo = mockModuleInfo( "modA", "/root/project/modA.ts", + MediaType.TypeScript, modASource, undefined ); @@ -75,6 +86,7 @@ export class B { const modBModuleInfo = mockModuleInfo( "modB", "/root/project/modB.ts", + MediaType.TypeScript, modBSource, undefined ); @@ -107,21 +119,31 @@ const moduleMap: { "foo/bar.ts": mockModuleInfo( "/root/project/foo/bar.ts", "/root/project/foo/bar.ts", + MediaType.TypeScript, fooBarTsSource, null ), "foo/baz.ts": mockModuleInfo( "/root/project/foo/baz.ts", "/root/project/foo/baz.ts", + MediaType.TypeScript, fooBazTsSource, fooBazTsOutput ), - "modA.ts": modAModuleInfo + "modA.ts": modAModuleInfo, + "some.txt": mockModuleInfo( + "/root/project/some.txt", + "/root/project/some.text", + MediaType.Unknown, + "console.log();", + null + ) }, "/root/project/foo/baz.ts": { "./bar.ts": mockModuleInfo( "/root/project/foo/bar.ts", "/root/project/foo/bar.ts", + MediaType.TypeScript, fooBarTsSource, fooBarTsOutput ) @@ -131,6 +153,43 @@ const moduleMap: { }, "/root/project/modB.ts": { "./modA.ts": modAModuleInfo + }, + "/moduleKinds": { + "foo.ts": mockModuleInfo( + "foo", + "/moduleKinds/foo.ts", + MediaType.TypeScript, + "console.log('foo');", + undefined + ), + "foo.d.ts": mockModuleInfo( + "foo", + "/moduleKinds/foo.d.ts", + MediaType.TypeScript, + "console.log('foo');", + undefined + ), + "foo.js": mockModuleInfo( + "foo", + "/moduleKinds/foo.js", + MediaType.JavaScript, + "console.log('foo');", + undefined + ), + "foo.json": mockModuleInfo( + "foo", + "/moduleKinds/foo.json", + MediaType.Json, + "console.log('foo');", + undefined + ), + "foo.txt": mockModuleInfo( + "foo", + "/moduleKinds/foo.txt", + MediaType.JavaScript, + "console.log('foo');", + undefined + ) } }; @@ -180,6 +239,7 @@ const osMock = { moduleCache[fileName] = mockModuleInfo( fileName, fileName, + MediaType.TypeScript, sourceCode, outputCode ); @@ -192,7 +252,7 @@ const osMock = { return moduleMap[containingFile][moduleSpecifier]; } } - return mockModuleInfo(null, null, null, null); + return mockModuleInfo(null, null, null, null, null); }, exit(code: number): never { throw new Error(`os.exit(${code})`); @@ -405,6 +465,23 @@ test(function compilerResolveModule() { teardown(); }); +test(function compilerResolveModuleUnknownMediaType() { + setup(); + let didThrow = false; + try { + compilerInstance.resolveModule("some.txt", "/root/project"); + } catch (e) { + assert(e instanceof Error); + assertEqual( + e.message, + `Unknown media type for: "some.txt" from "/root/project".` + ); + didThrow = true; + } + assert(didThrow); + teardown(); +}); + test(function compilerGetModuleDependencies() { setup(); const bazDeps = ["require", "exports", "./bar.ts"]; @@ -484,11 +561,33 @@ test(function compilerRecompileFlag() { }); test(function compilerGetScriptKind() { - assertEqual(compilerInstance.getScriptKind("foo.ts"), ts.ScriptKind.TS); - assertEqual(compilerInstance.getScriptKind("foo.d.ts"), ts.ScriptKind.TS); - assertEqual(compilerInstance.getScriptKind("foo.js"), ts.ScriptKind.JS); - assertEqual(compilerInstance.getScriptKind("foo.json"), ts.ScriptKind.JSON); - assertEqual(compilerInstance.getScriptKind("foo.txt"), ts.ScriptKind.JS); + setup(); + compilerInstance.resolveModule("foo.ts", "/moduleKinds"); + compilerInstance.resolveModule("foo.d.ts", "/moduleKinds"); + compilerInstance.resolveModule("foo.js", "/moduleKinds"); + compilerInstance.resolveModule("foo.json", "/moduleKinds"); + compilerInstance.resolveModule("foo.txt", "/moduleKinds"); + assertEqual( + compilerInstance.getScriptKind("/moduleKinds/foo.ts"), + ts.ScriptKind.TS + ); + assertEqual( + compilerInstance.getScriptKind("/moduleKinds/foo.d.ts"), + ts.ScriptKind.TS + ); + assertEqual( + compilerInstance.getScriptKind("/moduleKinds/foo.js"), + ts.ScriptKind.JS + ); + assertEqual( + compilerInstance.getScriptKind("/moduleKinds/foo.json"), + ts.ScriptKind.JSON + ); + assertEqual( + compilerInstance.getScriptKind("/moduleKinds/foo.txt"), + ts.ScriptKind.JS + ); + teardown(); }); test(function compilerGetScriptVersion() { diff --git a/js/os.ts b/js/os.ts index a4a7d3cc70..0d54de998c 100644 --- a/js/os.ts +++ b/js/os.ts @@ -1,11 +1,18 @@ // Copyright 2018 the Deno authors. All rights reserved. MIT license. -import { ModuleInfo } from "./types"; import * as msg from "gen/msg_generated"; import { assert } from "./util"; import * as util from "./util"; import * as flatbuffers from "./flatbuffers"; import { sendSync } from "./dispatch"; +interface CodeInfo { + moduleName: string | undefined; + filename: string | undefined; + mediaType: msg.MediaType; + sourceCode: string | undefined; + outputCode: string | undefined; +} + /** Exit the Deno process with optional exit code. */ export function exit(exitCode = 0): never { const builder = flatbuffers.createBuilder(); @@ -20,7 +27,7 @@ export function exit(exitCode = 0): never { export function codeFetch( moduleSpecifier: string, containingFile: string -): ModuleInfo { +): CodeInfo { util.log("os.ts codeFetch", moduleSpecifier, containingFile); // Send CodeFetch message const builder = flatbuffers.createBuilder(); @@ -38,11 +45,14 @@ export function codeFetch( ); const codeFetchRes = new msg.CodeFetchRes(); assert(baseRes!.inner(codeFetchRes) != null); + // flatbuffers returns `null` for an empty value, this does not fit well with + // idiomatic TypeScript under strict null checks, so converting to `undefined` return { - moduleName: codeFetchRes.moduleName(), - filename: codeFetchRes.filename(), - sourceCode: codeFetchRes.sourceCode(), - outputCode: codeFetchRes.outputCode() + moduleName: codeFetchRes.moduleName() || undefined, + filename: codeFetchRes.filename() || undefined, + mediaType: codeFetchRes.mediaType(), + sourceCode: codeFetchRes.sourceCode() || undefined, + outputCode: codeFetchRes.outputCode() || undefined }; } diff --git a/js/types.ts b/js/types.ts index 4e35d1227d..954816811a 100644 --- a/js/types.ts +++ b/js/types.ts @@ -1,14 +1,6 @@ // Copyright 2018 the Deno authors. All rights reserved. MIT license. export type TypedArray = Uint8Array | Float32Array | Int32Array; -// @internal -export interface ModuleInfo { - moduleName: string | null; - filename: string | null; - sourceCode: string | null; - outputCode: string | null; -} - // tslint:disable:max-line-length // Following definitions adapted from: // https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/node/index.d.ts diff --git a/src/deno_dir.rs b/src/deno_dir.rs index e8cc86b186..186dce1b8c 100644 --- a/src/deno_dir.rs +++ b/src/deno_dir.rs @@ -5,6 +5,7 @@ use errors::DenoResult; use errors::ErrorKind; use fs as deno_fs; use http_util; +use msg; use ring; use std; use std::fmt::Write; @@ -109,21 +110,28 @@ impl DenoDir { self: &DenoDir, module_name: &str, filename: &str, - ) -> DenoResult { + ) -> DenoResult<(String, msg::MediaType)> { let p = Path::new(filename); + // We write a special ".mime" file into the `.deno/deps` directory along side the + // cached file, containing just the media type. + let mut media_type_filename = filename.to_string(); + media_type_filename.push_str(".mime"); + let mt = Path::new(&media_type_filename); let src = if self.reload || !p.exists() { println!("Downloading {}", module_name); - let source = http_util::fetch_sync_string(module_name)?; + let (source, content_type) = http_util::fetch_sync_string(module_name)?; match p.parent() { Some(ref parent) => fs::create_dir_all(parent), None => Ok(()), }?; deno_fs::write_file(&p, source.as_bytes(), 0o666)?; - source + deno_fs::write_file(&mt, content_type.as_bytes(), 0o666)?; + (source, map_content_type(&p, Some(&content_type))) } else { let source = fs::read_to_string(&p)?; - source + let content_type = fs::read_to_string(&mt)?; + (source, map_content_type(&p, Some(&content_type))) }; Ok(src) } @@ -141,18 +149,20 @@ impl DenoDir { let use_extension = |ext| { let module_name = format!("{}{}", module_name, ext); let filename = format!("{}{}", filename, ext); - let source_code = if is_module_remote { + let (source_code, media_type) = if is_module_remote { self.fetch_remote_source(&module_name, &filename)? } else { assert_eq!( module_name, filename, "if a module isn't remote, it should have the same filename" ); - fs::read_to_string(Path::new(&filename))? + let path = Path::new(&filename); + (fs::read_to_string(path)?, map_content_type(path, None)) }; return Ok(CodeFetchOutput { module_name: module_name.to_string(), filename: filename.to_string(), + media_type, source_code, maybe_output_code: None, }); @@ -215,6 +225,7 @@ impl DenoDir { Ok(output_code) => Ok(CodeFetchOutput { module_name: out.module_name, filename: out.filename, + media_type: out.media_type, source_code: out.source_code, maybe_output_code: Some(output_code), }), @@ -324,6 +335,7 @@ fn test_get_cache_filename() { pub struct CodeFetchOutput { pub module_name: String, pub filename: String, + pub media_type: msg::MediaType, pub source_code: String, pub maybe_output_code: Option, } @@ -646,3 +658,157 @@ fn parse_local_or_remote(p: &str) -> Result { Url::from_file_path(p).map_err(|_err| url::ParseError::IdnaError) } } + +fn map_file_extension(path: &Path) -> msg::MediaType { + match path.extension() { + None => msg::MediaType::Unknown, + Some(os_str) => match os_str.to_str() { + Some("ts") => msg::MediaType::TypeScript, + Some("js") => msg::MediaType::JavaScript, + Some("json") => msg::MediaType::Json, + _ => msg::MediaType::Unknown, + }, + } +} + +#[test] +fn test_map_file_extension() { + assert_eq!( + map_file_extension(Path::new("foo/bar.ts")), + msg::MediaType::TypeScript + ); + assert_eq!( + map_file_extension(Path::new("foo/bar.d.ts")), + msg::MediaType::TypeScript + ); + assert_eq!( + map_file_extension(Path::new("foo/bar.js")), + msg::MediaType::JavaScript + ); + assert_eq!( + map_file_extension(Path::new("foo/bar.json")), + msg::MediaType::Json + ); + assert_eq!( + map_file_extension(Path::new("foo/bar.txt")), + msg::MediaType::Unknown + ); + assert_eq!( + map_file_extension(Path::new("foo/bar")), + msg::MediaType::Unknown + ); +} + +// convert a ContentType string into a enumerated MediaType +fn map_content_type(path: &Path, content_type: Option<&str>) -> msg::MediaType { + match content_type { + Some(content_type) => { + // sometimes there is additional data after the media type in + // Content-Type so we have to do a bit of manipulation so we are only + // dealing with the actual media type + let ct_vector: Vec<&str> = content_type.split(";").collect(); + let ct: &str = ct_vector.first().unwrap(); + match ct.to_lowercase().as_ref() { + "application/typescript" + | "text/typescript" + | "video/vnd.dlna.mpeg-tts" + | "video/mp2t" => msg::MediaType::TypeScript, + "application/javascript" + | "text/javascript" + | "application/ecmascript" + | "text/ecmascript" + | "application/x-javascript" => msg::MediaType::JavaScript, + "application/json" | "text/json" => msg::MediaType::Json, + "text/plain" => map_file_extension(path), + _ => { + debug!("unknown content type: {}", content_type); + msg::MediaType::Unknown + } + } + } + None => map_file_extension(path), + } +} + +#[test] +fn test_map_content_type() { + // Extension only + assert_eq!( + map_content_type(Path::new("foo/bar.ts"), None), + msg::MediaType::TypeScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar.d.ts"), None), + msg::MediaType::TypeScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar.js"), None), + msg::MediaType::JavaScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar.json"), None), + msg::MediaType::Json + ); + assert_eq!( + map_content_type(Path::new("foo/bar.txt"), None), + msg::MediaType::Unknown + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), None), + msg::MediaType::Unknown + ); + + // Media Type + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("application/typescript")), + msg::MediaType::TypeScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("text/typescript")), + msg::MediaType::TypeScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("video/vnd.dlna.mpeg-tts")), + msg::MediaType::TypeScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("video/mp2t")), + msg::MediaType::TypeScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("application/javascript")), + msg::MediaType::JavaScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("text/javascript")), + msg::MediaType::JavaScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("application/ecmascript")), + msg::MediaType::JavaScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("text/ecmascript")), + msg::MediaType::JavaScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("application/x-javascript")), + msg::MediaType::JavaScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("application/json")), + msg::MediaType::Json + ); + assert_eq!( + map_content_type(Path::new("foo/bar"), Some("text/json")), + msg::MediaType::Json + ); + assert_eq!( + map_content_type(Path::new("foo/bar.ts"), Some("text/plain")), + msg::MediaType::TypeScript + ); + assert_eq!( + map_content_type(Path::new("foo/bar.ts"), Some("foo/bar")), + msg::MediaType::Unknown + ); +} diff --git a/src/http_util.rs b/src/http_util.rs index 1c6079c08c..9c5ff95292 100644 --- a/src/http_util.rs +++ b/src/http_util.rs @@ -4,9 +4,10 @@ use errors::{DenoError, DenoResult}; use tokio_util; use futures::future::{loop_fn, Loop}; -use futures::{Future, Stream}; +use futures::{future, Future, Stream}; use hyper; use hyper::client::{Client, HttpConnector}; +use hyper::header::CONTENT_TYPE; use hyper::Uri; use hyper_rustls; @@ -27,7 +28,7 @@ pub fn get_client() -> Client { // The CodeFetch message is used to load HTTP javascript resources and expects a // synchronous response, this utility method supports that. -pub fn fetch_sync_string(module_name: &str) -> DenoResult { +pub fn fetch_sync_string(module_name: &str) -> DenoResult<(String, String)> { let url = module_name.parse::().unwrap(); let client = get_client(); // TODO(kevinkassimo): consider set a max redirection counter @@ -63,11 +64,18 @@ pub fn fetch_sync_string(module_name: &str) -> DenoResult { Ok(Loop::Break(response)) }) }).and_then(|response| { - response + let content_type = response + .headers() + .get(CONTENT_TYPE) + .map(|content_type| content_type.to_str().unwrap().to_string()); + let body = response .into_body() .concat2() .map(|body| String::from_utf8(body.to_vec()).unwrap()) - .map_err(|err| DenoError::from(err)) + .map_err(|err| DenoError::from(err)); + body.join(future::ok(content_type)) + }).and_then(|(body_string, maybe_content_type)| { + future::ok((body_string, maybe_content_type.unwrap())) }); tokio_util::block_on(fetch_future) @@ -77,9 +85,11 @@ pub fn fetch_sync_string(module_name: &str) -> DenoResult { fn test_fetch_sync_string() { // Relies on external http server. See tools/http_server.py tokio_util::init(|| { - let p = fetch_sync_string("http://127.0.0.1:4545/package.json").unwrap(); + let (p, m) = + fetch_sync_string("http://127.0.0.1:4545/package.json").unwrap(); println!("package.json len {}", p.len()); assert!(p.len() > 1); + assert!(m == "application/json") }); } @@ -87,8 +97,10 @@ fn test_fetch_sync_string() { fn test_fetch_sync_string_with_redirect() { // Relies on external http server. See tools/http_server.py tokio_util::init(|| { - let p = fetch_sync_string("http://127.0.0.1:4546/package.json").unwrap(); + let (p, m) = + fetch_sync_string("http://127.0.0.1:4546/package.json").unwrap(); println!("package.json len {}", p.len()); assert!(p.len() > 1); + assert!(m == "application/json") }); } diff --git a/src/msg.fbs b/src/msg.fbs index 9aa9b069c7..870da1c404 100644 --- a/src/msg.fbs +++ b/src/msg.fbs @@ -102,6 +102,13 @@ table CwdRes { cwd: string; } +enum MediaType: byte { + JavaScript = 0, + TypeScript, + Json, + Unknown +} + table Base { cmd_id: uint32; sync: bool = true; // TODO(ry) Change default to false. @@ -137,6 +144,7 @@ table CodeFetchRes { // is the location of the locally downloaded source code. module_name: string; filename: string; + media_type: MediaType; source_code: string; output_code: string; // Non-empty only if cached. } diff --git a/src/ops.rs b/src/ops.rs index 4adea46f65..83a719a29e 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -253,6 +253,7 @@ fn op_code_fetch( let mut msg_args = msg::CodeFetchResArgs { module_name: Some(builder.create_string(&out.module_name)), filename: Some(builder.create_string(&out.filename)), + media_type: out.media_type, source_code: Some(builder.create_string(&out.source_code)), ..Default::default() }; diff --git a/tests/019_media_types.ts b/tests/019_media_types.ts new file mode 100644 index 0000000000..dbd951b3e9 --- /dev/null +++ b/tests/019_media_types.ts @@ -0,0 +1,23 @@ +// When run against the test HTTP server, it will serve different media types +// based on the URL containing `.t#.` strings, which exercises the different +// mapping of media types end to end. + +// tslint:disable:max-line-length +import { loaded as loadedTs1 } from "http://localhost:4545/tests/subdir/mt_text_typescript.t1.ts"; +import { loaded as loadedTs2 } from "http://localhost:4545/tests/subdir/mt_video_vdn.t2.ts"; +import { loaded as loadedTs3 } from "http://localhost:4545/tests/subdir/mt_video_mp2t.t3.ts"; +import { loaded as loadedJs1 } from "http://localhost:4545/tests/subdir/mt_text_javascript.j1.js"; +import { loaded as loadedJs2 } from "http://localhost:4545/tests/subdir/mt_application_ecmascript.j2.js"; +import { loaded as loadedJs3 } from "http://localhost:4545/tests/subdir/mt_text_ecmascript.j3.js"; +import { loaded as loadedJs4 } from "http://localhost:4545/tests/subdir/mt_application_x_javascript.j4.js"; + +console.log( + "success", + loadedTs1, + loadedTs2, + loadedTs3, + loadedJs1, + loadedJs2, + loadedJs3, + loadedJs4 +); diff --git a/tests/019_media_types.ts.out b/tests/019_media_types.ts.out new file mode 100644 index 0000000000..b127519b5b --- /dev/null +++ b/tests/019_media_types.ts.out @@ -0,0 +1,8 @@ +Downloading http://localhost:4545/tests/subdir/mt_text_typescript.t1.ts +Downloading http://localhost:4545/tests/subdir/mt_video_vdn.t2.ts +Downloading http://localhost:4545/tests/subdir/mt_video_mp2t.t3.ts +Downloading http://localhost:4545/tests/subdir/mt_text_javascript.j1.js +Downloading http://localhost:4545/tests/subdir/mt_application_ecmascript.j2.js +Downloading http://localhost:4545/tests/subdir/mt_text_ecmascript.j3.js +Downloading http://localhost:4545/tests/subdir/mt_application_x_javascript.j4.js +success true true true true true true true diff --git a/tests/error_004_missing_module.ts.out b/tests/error_004_missing_module.ts.out index 03ebec0ac9..64932ddd01 100644 --- a/tests/error_004_missing_module.ts.out +++ b/tests/error_004_missing_module.ts.out @@ -4,8 +4,8 @@ NotFound: Cannot resolve module "bad-module.ts" from "[WILDCARD]/tests/error_004 at sendSync ([WILDCARD]/js/dispatch.ts:[WILDCARD]) at Object.codeFetch ([WILDCARD]/js/os.ts:[WILDCARD]) at DenoCompiler.resolveModule ([WILDCARD]/js/compiler.ts:[WILDCARD]) - at DenoCompiler._resolveModuleName ([WILDCARD]/js/compiler.ts:[WILDCARD]) at moduleNames.map.name ([WILDCARD]/js/compiler.ts:[WILDCARD]) at Array.map () at DenoCompiler.resolveModuleNames ([WILDCARD]/js/compiler.ts:[WILDCARD]) at Object.compilerHost.resolveModuleNames ([WILDCARD]/third_party/node_modules/typescript/lib/typescript.js:[WILDCARD]) + at resolveModuleNamesWorker ([WILDCARD]/third_party/node_modules/typescript/lib/typescript.js:[WILDCARD]) diff --git a/tests/error_005_missing_dynamic_import.ts.out b/tests/error_005_missing_dynamic_import.ts.out index b050c7801a..eb991de9db 100644 --- a/tests/error_005_missing_dynamic_import.ts.out +++ b/tests/error_005_missing_dynamic_import.ts.out @@ -4,8 +4,8 @@ NotFound: Cannot resolve module "bad-module.ts" from "[WILDCARD]/tests/error_005 at sendSync ([WILDCARD]/js/dispatch.ts:[WILDCARD]) at Object.codeFetch ([WILDCARD]/js/os.ts:[WILDCARD]) at DenoCompiler.resolveModule ([WILDCARD]/js/compiler.ts:[WILDCARD]) - at DenoCompiler._resolveModuleName ([WILDCARD]/js/compiler.ts:[WILDCARD]) at moduleNames.map.name ([WILDCARD]/js/compiler.ts:[WILDCARD]) at Array.map () at DenoCompiler.resolveModuleNames ([WILDCARD]/js/compiler.ts:[WILDCARD]) at Object.compilerHost.resolveModuleNames ([WILDCARD]/third_party/node_modules/typescript/lib/typescript.js:[WILDCARD]) + at resolveModuleNamesWorker ([WILDCARD]/third_party/node_modules/typescript/lib/typescript.js:[WILDCARD]) diff --git a/tests/error_006_import_ext_failure.ts.out b/tests/error_006_import_ext_failure.ts.out index 3abc6fa6df..f2b6aa5991 100644 --- a/tests/error_006_import_ext_failure.ts.out +++ b/tests/error_006_import_ext_failure.ts.out @@ -4,8 +4,8 @@ NotFound: Cannot resolve module "./non-existent" from "[WILDCARD]/tests/error_00 at sendSync ([WILDCARD]/js/dispatch.ts:[WILDCARD]) at Object.codeFetch ([WILDCARD]/js/os.ts:[WILDCARD]) at DenoCompiler.resolveModule ([WILDCARD]/js/compiler.ts:[WILDCARD]) - at DenoCompiler._resolveModuleName ([WILDCARD]/js/compiler.ts:[WILDCARD]) at moduleNames.map.name ([WILDCARD]/js/compiler.ts:[WILDCARD]) at Array.map () at DenoCompiler.resolveModuleNames ([WILDCARD]/js/compiler.ts:[WILDCARD]) at Object.compilerHost.resolveModuleNames ([WILDCARD]/third_party/node_modules/typescript/lib/typescript.js:[WILDCARD]) + at resolveModuleNamesWorker ([WILDCARD]/third_party/node_modules/typescript/lib/typescript.js:[WILDCARD]) diff --git a/tests/subdir/mt_application_ecmascript.j2.js b/tests/subdir/mt_application_ecmascript.j2.js new file mode 100644 index 0000000000..ec30e0595c --- /dev/null +++ b/tests/subdir/mt_application_ecmascript.j2.js @@ -0,0 +1,3 @@ +define(["exports"], function(exports) { + exports.loaded = true; +}); diff --git a/tests/subdir/mt_application_x_javascript.j4.js b/tests/subdir/mt_application_x_javascript.j4.js new file mode 100644 index 0000000000..ec30e0595c --- /dev/null +++ b/tests/subdir/mt_application_x_javascript.j4.js @@ -0,0 +1,3 @@ +define(["exports"], function(exports) { + exports.loaded = true; +}); diff --git a/tests/subdir/mt_javascript.js b/tests/subdir/mt_javascript.js new file mode 100644 index 0000000000..ec30e0595c --- /dev/null +++ b/tests/subdir/mt_javascript.js @@ -0,0 +1,3 @@ +define(["exports"], function(exports) { + exports.loaded = true; +}); diff --git a/tests/subdir/mt_text_ecmascript.j3.js b/tests/subdir/mt_text_ecmascript.j3.js new file mode 100644 index 0000000000..ec30e0595c --- /dev/null +++ b/tests/subdir/mt_text_ecmascript.j3.js @@ -0,0 +1,3 @@ +define(["exports"], function(exports) { + exports.loaded = true; +}); diff --git a/tests/subdir/mt_text_javascript.j1.js b/tests/subdir/mt_text_javascript.j1.js new file mode 100644 index 0000000000..ec30e0595c --- /dev/null +++ b/tests/subdir/mt_text_javascript.j1.js @@ -0,0 +1,3 @@ +define(["exports"], function(exports) { + exports.loaded = true; +}); diff --git a/tests/subdir/mt_text_typescript.t1.ts b/tests/subdir/mt_text_typescript.t1.ts new file mode 100644 index 0000000000..e67d2a0172 --- /dev/null +++ b/tests/subdir/mt_text_typescript.t1.ts @@ -0,0 +1 @@ +export const loaded = true; diff --git a/tests/subdir/mt_video_mp2t.t3.ts b/tests/subdir/mt_video_mp2t.t3.ts new file mode 100644 index 0000000000..e67d2a0172 --- /dev/null +++ b/tests/subdir/mt_video_mp2t.t3.ts @@ -0,0 +1 @@ +export const loaded = true; diff --git a/tests/subdir/mt_video_vdn.t2.ts b/tests/subdir/mt_video_vdn.t2.ts new file mode 100644 index 0000000000..e67d2a0172 --- /dev/null +++ b/tests/subdir/mt_video_vdn.t2.ts @@ -0,0 +1 @@ +export const loaded = true; diff --git a/tools/http_server.py b/tools/http_server.py index c96b070e6d..011f3c31a1 100755 --- a/tools/http_server.py +++ b/tools/http_server.py @@ -14,9 +14,33 @@ PORT = 4545 REDIRECT_PORT = 4546 +class ContentTypeHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + def guess_type(self, path): + if ".t1." in path: + return "text/typescript" + if ".t2." in path: + return "video/vnd.dlna.mpeg-tts" + if ".t3." in path: + return "video/mp2t" + if ".j1." in path: + return "text/javascript" + if ".j2." in path: + return "application/ecmascript" + if ".j3." in path: + return "text/ecmascript" + if ".j4." in path: + return "application/x-javascript" + return SimpleHTTPServer.SimpleHTTPRequestHandler.guess_type(self, path) + + def server(): os.chdir(root_path) # Hopefully the main thread doesn't also chdir. - Handler = SimpleHTTPServer.SimpleHTTPRequestHandler + Handler = ContentTypeHandler + Handler.extensions_map.update({ + ".ts": "application/typescript", + ".js": "application/javascript", + ".json": "application/json", + }) SocketServer.TCPServer.allow_reuse_address = True s = SocketServer.TCPServer(("", PORT), Handler) print "Deno test server http://localhost:%d/" % PORT