0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-03-03 09:31:22 -05:00

chore: port http_server.py to rust (#6364)

This commit is contained in:
Ryan Dahl 2020-07-04 13:05:01 -04:00 committed by GitHub
parent fca492907c
commit 5f9e600c5b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 457 additions and 498 deletions

51
Cargo.lock generated
View file

@ -121,6 +121,15 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
[[package]]
name = "base64"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
dependencies = [
"byteorder",
]
[[package]]
name = "base64"
version = "0.11.0"
@ -408,7 +417,7 @@ dependencies = [
"termcolor",
"test_util",
"tokio",
"tokio-rustls",
"tokio-rustls 0.13.1",
"tokio-tungstenite",
"url",
"utime",
@ -941,9 +950,9 @@ dependencies = [
"futures-util",
"hyper",
"log 0.4.8",
"rustls",
"rustls 0.17.0",
"tokio",
"tokio-rustls",
"tokio-rustls 0.13.1",
"webpki",
]
@ -1815,11 +1824,11 @@ dependencies = [
"mime_guess 2.0.3",
"percent-encoding",
"pin-project-lite",
"rustls",
"rustls 0.17.0",
"serde",
"serde_urlencoded",
"tokio",
"tokio-rustls",
"tokio-rustls 0.13.1",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
@ -1852,6 +1861,19 @@ dependencies = [
"semver",
]
[[package]]
name = "rustls"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b25a18b1bf7387f0145e7f8324e700805aade3842dd3db2e74e4cdeb4677c09e"
dependencies = [
"base64 0.10.1",
"log 0.4.8",
"ring",
"sct",
"webpki",
]
[[package]]
name = "rustls"
version = "0.17.0"
@ -2349,10 +2371,14 @@ dependencies = [
name = "test_util"
version = "0.1.0"
dependencies = [
"bytes 0.5.5",
"futures 0.3.5",
"lazy_static",
"os_pipe",
"regex",
"tempfile",
"tokio",
"warp",
]
[[package]]
@ -2455,6 +2481,18 @@ dependencies = [
"syn 1.0.33",
]
[[package]]
name = "tokio-rustls"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3068d891551949b37681724d6b73666787cc63fa8e255c812a41d2513aff9775"
dependencies = [
"futures-core",
"rustls 0.16.0",
"tokio",
"webpki",
]
[[package]]
name = "tokio-rustls"
version = "0.13.1"
@ -2462,7 +2500,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4"
dependencies = [
"futures-core",
"rustls",
"rustls 0.17.0",
"tokio",
"webpki",
]
@ -2721,6 +2759,7 @@ dependencies = [
"serde_json",
"serde_urlencoded",
"tokio",
"tokio-rustls 0.12.3",
"tokio-tungstenite",
"tower-service",
"urlencoding",

View file

@ -249,7 +249,7 @@ mod tests {
#[tokio::test]
async fn test_fetch_string() {
let http_server_guard = test_util::http_server();
// Relies on external http server. See tools/http_server.py
// Relies on external http server. See target/debug/test_server
let url =
Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap();
let client = create_http_client(None).unwrap();
@ -268,7 +268,7 @@ mod tests {
#[tokio::test]
async fn test_fetch_gzip() {
let http_server_guard = test_util::http_server();
// Relies on external http server. See tools/http_server.py
// Relies on external http server. See target/debug/test_server
let url = Url::parse(
"http://127.0.0.1:4545/cli/tests/053_import_compression/gziped",
)
@ -317,7 +317,7 @@ mod tests {
#[tokio::test]
async fn test_fetch_brotli() {
let http_server_guard = test_util::http_server();
// Relies on external http server. See tools/http_server.py
// Relies on external http server. See target/debug/test_server
let url = Url::parse(
"http://127.0.0.1:4545/cli/tests/053_import_compression/brotli",
)
@ -342,7 +342,7 @@ mod tests {
#[tokio::test]
async fn test_fetch_once_with_redirect() {
let http_server_guard = test_util::http_server();
// Relies on external http server. See tools/http_server.py
// Relies on external http server. See target/debug/test_server
let url =
Url::parse("http://127.0.0.1:4546/cli/tests/fixture.json").unwrap();
// Dns resolver substitutes `127.0.0.1` with `localhost`
@ -399,7 +399,7 @@ mod tests {
#[tokio::test]
async fn test_fetch_with_cafile_string() {
let http_server_guard = test_util::http_server();
// Relies on external http server. See tools/http_server.py
// Relies on external http server. See target/debug/test_server
let url =
Url::parse("https://localhost:5545/cli/tests/fixture.json").unwrap();
@ -425,7 +425,7 @@ mod tests {
#[tokio::test]
async fn test_fetch_with_cafile_gzip() {
let http_server_guard = test_util::http_server();
// Relies on external http server. See tools/http_server.py
// Relies on external http server. See target/debug/test_server
let url = Url::parse(
"https://localhost:5545/cli/tests/053_import_compression/gziped",
)
@ -487,7 +487,7 @@ mod tests {
#[tokio::test]
async fn test_fetch_with_cafile_brotli() {
let http_server_guard = test_util::http_server();
// Relies on external http server. See tools/http_server.py
// Relies on external http server. See target/debug/test_server
let url = Url::parse(
"https://localhost:5545/cli/tests/053_import_compression/brotli",
)

View file

@ -1,3 +0,0 @@
Content-Encoding: br
Content-Type: application/javascript
Content-Length: 26

View file

@ -1,3 +0,0 @@
Content-Encoding: gzip
Content-Type: application/javascript
Content-Length: 39

View file

@ -2600,7 +2600,8 @@ fn test_permissions_net_listen_allow_localhost_4555_fail() {
#[test]
fn test_permissions_net_listen_allow_localhost() {
// Port 4600 is chosen to not colide with those used by tools/http_server.py
// Port 4600 is chosen to not colide with those used by
// target/debug/test_server
let (_, err) = util::run_and_collect_output(
true,
"run --allow-net=localhost complex_permissions_test.ts netListen localhost:4600",

View file

@ -75,6 +75,6 @@ RUST_BACKTRACE=1 cargo run -- run --unstable --allow-read --allow-write cli/test
### Http server
`tools/http_server.py` is required to run when one's running unit tests. During
CI it's spawned automatically, but if you want to run tests manually make sure
that server is spawned otherwise there'll be cascade of test failures.
`target/debug/test_server` is required to run when one's running unit tests.
During CI it's spawned automatically, but if you want to run tests manually make
sure that server is spawned otherwise there'll be cascade of test failures.

View file

@ -36,7 +36,7 @@ unitTest(
{ perms: { net: true } },
async function bodyMultipartFormData(): Promise<void> {
const response = await fetch(
"http://localhost:4545/cli/tests/subdir/multipart_form_data.txt"
"http://localhost:4545/multipart_form_data.txt"
);
const text = await response.text();

View file

@ -67,7 +67,6 @@ unitTest({ perms: { net: true } }, async function fetchHeaders(): Promise<
const response = await fetch("http://localhost:4545/cli/tests/fixture.json");
const headers = response.headers;
assertEquals(headers.get("Content-Type"), "application/json");
assert(headers.get("Server")!.startsWith("SimpleHTTP"));
const _json = await response.json();
});
@ -162,13 +161,10 @@ unitTest(
{ perms: { net: true } },
async function fetchBodyReaderBigBody(): Promise<void> {
const data = "a".repeat(10 << 10); // 10mb
const response = await fetch(
"http://localhost:4545/cli/tests/echo_server",
{
method: "POST",
body: data,
}
);
const response = await fetch("http://localhost:4545/echo_server", {
method: "POST",
body: data,
});
assert(response.body !== null);
const reader = await response.body.getReader();
let total = 0;
@ -210,7 +206,7 @@ unitTest(
{ perms: { net: true } },
async function fetchMultipartFormDataSuccess(): Promise<void> {
const response = await fetch(
"http://localhost:4545/cli/tests/subdir/multipart_form_data.txt"
"http://localhost:4545/multipart_form_data.txt"
);
const formData = await response.formData();
assert(formData.has("field_1"));
@ -315,12 +311,12 @@ unitTest(
perms: { net: true },
},
async function fetchWithRedirection(): Promise<void> {
const response = await fetch("http://localhost:4546/"); // will redirect to http://localhost:4545/
const response = await fetch("http://localhost:4546/README.md");
assertEquals(response.status, 200);
assertEquals(response.statusText, "OK");
assertEquals(response.url, "http://localhost:4545/");
assertEquals(response.url, "http://localhost:4545/README.md");
const body = await response.text();
assert(body.includes("<title>Directory listing for /</title>"));
assert(body.includes("Deno"));
}
);
@ -329,11 +325,13 @@ unitTest(
perms: { net: true },
},
async function fetchWithRelativeRedirection(): Promise<void> {
const response = await fetch("http://localhost:4545/cli/tests"); // will redirect to /cli/tests/
const response = await fetch(
"http://localhost:4545/cli/tests/001_hello.js"
);
assertEquals(response.status, 200);
assertEquals(response.statusText, "OK");
const body = await response.text();
assert(body.includes("<title>Directory listing for /cli/tests/</title>"));
assert(body.includes("Hello"));
}
);
@ -769,13 +767,10 @@ unitTest(
{ perms: { net: true } },
async function fetchBodyReaderWithCancelAndNewReader(): Promise<void> {
const data = "a".repeat(1 << 10);
const response = await fetch(
"http://localhost:4545/cli/tests/echo_server",
{
method: "POST",
body: data,
}
);
const response = await fetch("http://localhost:4545/echo_server", {
method: "POST",
body: data,
});
assert(response.body !== null);
const firstReader = await response.body.getReader();
@ -801,13 +796,10 @@ unitTest(
async function fetchBodyReaderWithReadCancelAndNewReader(): Promise<void> {
const data = "a".repeat(1 << 10);
const response = await fetch(
"http://localhost:4545/cli/tests/echo_server",
{
method: "POST",
body: data,
}
);
const response = await fetch("http://localhost:4545/echo_server", {
method: "POST",
body: data,
});
assert(response.body !== null);
const firstReader = await response.body.getReader();
@ -848,7 +840,7 @@ unitTest(
for (const status of nullBodyStatus) {
const headers = new Headers([["x-status", String(status)]]);
const res = await fetch("http://localhost:4545/cli/tests/echo_server", {
const res = await fetch("http://localhost:4545/echo_server", {
body: "deno",
method: "POST",
headers,

View file

@ -1,2 +0,0 @@
Content-Type: application/javascript
X-Deno-Warning: foobar

View file

@ -5,10 +5,16 @@ authors = ["the Deno authors"]
edition = "2018"
publish = false
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[[bin]]
name = "test_server"
path = "src/test_server.rs"
[dependencies]
tokio = { version = "0.2.21", features = ["rt-core", "tcp", "udp", "uds", "process", "fs", "blocking", "sync", "io-std", "macros", "time"] }
futures = { version = "0.3.5", features = ["compat", "io-compat"] }
bytes = "0.5.5"
lazy_static = "1.4.0"
os_pipe = "0.9.2"
regex = "1.3.9"
tempfile = "3.1.0"
warp = { version = "0.2.3", features = ["tls"] }

View file

@ -3,10 +3,12 @@
#[macro_use]
extern crate lazy_static;
use futures::future::{self, FutureExt};
use os_pipe::pipe;
use regex::Regex;
use std::io::Read;
use std::io::Write;
use std::mem::replace;
use std::path::PathBuf;
use std::process::Child;
use std::process::Command;
@ -15,6 +17,20 @@ use std::process::Stdio;
use std::sync::Mutex;
use std::sync::MutexGuard;
use tempfile::TempDir;
use warp::http::Uri;
use warp::http::{HeaderValue, Response, StatusCode};
use warp::hyper::Body;
use warp::reply::with_header;
use warp::reply::Reply;
use warp::Filter;
const PORT: u16 = 4545;
const REDIRECT_PORT: u16 = 4546;
const ANOTHER_REDIRECT_PORT: u16 = 4547;
const DOUBLE_REDIRECTS_PORT: u16 = 4548;
const INF_REDIRECTS_PORT: u16 = 4549;
const REDIRECT_ABSOLUTE_PORT: u16 = 4550;
const HTTPS_PORT: u16 = 5545;
pub const PERMISSION_VARIANTS: [&str; 5] =
["read", "write", "env", "net", "run"];
@ -54,27 +70,358 @@ pub fn deno_exe_path() -> PathBuf {
p
}
pub fn test_server_path() -> PathBuf {
let mut p = target_dir().join("test_server");
if cfg!(windows) {
p.set_extension("exe");
}
p
}
#[tokio::main]
pub async fn run_all_servers() {
let routes = warp::path::full().map(|path: warp::path::FullPath| {
let p = path.as_str();
assert_eq!(&p[0..1], "/");
let url = format!("http://localhost:{}{}", PORT, p);
let u = url.parse::<Uri>().unwrap();
warp::redirect(u)
});
let redirect_server_fut =
warp::serve(routes).bind(([127, 0, 0, 1], REDIRECT_PORT));
let routes = warp::path::full().map(|path: warp::path::FullPath| {
let p = path.as_str();
assert_eq!(&p[0..1], "/");
let url = format!("http://localhost:{}/cli/tests/subdir{}", PORT, p);
let u = url.parse::<Uri>().unwrap();
warp::redirect(u)
});
let another_redirect_server_fut =
warp::serve(routes).bind(([127, 0, 0, 1], ANOTHER_REDIRECT_PORT));
let routes = warp::path::full().map(|path: warp::path::FullPath| {
let p = path.as_str();
assert_eq!(&p[0..1], "/");
let url = format!("http://localhost:{}{}", REDIRECT_PORT, p);
let u = url.parse::<Uri>().unwrap();
warp::redirect(u)
});
let double_redirect_server_fut =
warp::serve(routes).bind(([127, 0, 0, 1], DOUBLE_REDIRECTS_PORT));
let routes = warp::path::full().map(|path: warp::path::FullPath| {
let p = path.as_str();
assert_eq!(&p[0..1], "/");
let url = format!("http://localhost:{}{}", INF_REDIRECTS_PORT, p);
let u = url.parse::<Uri>().unwrap();
warp::redirect(u)
});
let inf_redirect_server_fut =
warp::serve(routes).bind(([127, 0, 0, 1], INF_REDIRECTS_PORT));
// redirect server that redirect to absolute paths under same host
// redirects /REDIRECT/file_name to /file_name
let routes = warp::path("REDIRECT")
.and(warp::path::peek())
.map(|path: warp::path::Peek| {
let p = path.as_str();
let url = format!("/{}", p);
let u = url.parse::<Uri>().unwrap();
warp::redirect(u)
})
.or(
warp::any()
.and(warp::path::peek())
.and(warp::fs::dir(root_path()))
.map(custom_headers),
);
let absolute_redirect_server_fut =
warp::serve(routes).bind(([127, 0, 0, 1], REDIRECT_ABSOLUTE_PORT));
let echo_server = warp::path("echo_server")
.and(warp::post())
.and(warp::body::bytes())
.and(warp::header::optional::<String>("x-status"))
.and(warp::header::optional::<String>("content-type"))
.and(warp::header::optional::<String>("user-agent"))
.map(
|bytes: bytes::Bytes,
status: Option<String>,
content_type: Option<String>,
user_agent: Option<String>|
-> Box<dyn Reply> {
let mut res = Response::new(Body::from(bytes));
if let Some(v) = status {
*res.status_mut() = StatusCode::from_bytes(v.as_bytes()).unwrap();
}
let h = res.headers_mut();
if let Some(v) = content_type {
h.insert("content-type", HeaderValue::from_str(&v).unwrap());
}
if let Some(v) = user_agent {
h.insert("user-agent", HeaderValue::from_str(&v).unwrap());
}
Box::new(res)
},
);
let echo_multipart_file = warp::path("echo_multipart_file")
.and(warp::post())
.and(warp::body::bytes())
.map(|bytes: bytes::Bytes| -> Box<dyn Reply> {
let start = b"--boundary\t \r\n\
Content-Disposition: form-data; name=\"field_1\"\r\n\
\r\n\
value_1 \r\n\
\r\n--boundary\r\n\
Content-Disposition: form-data; name=\"file\"; \
filename=\"file.bin\"\r\n\
Content-Type: application/octet-stream\r\n\
\r\n";
let end = b"\r\n--boundary--\r\n";
let b = [start as &[u8], &bytes, end].concat();
let mut res = Response::new(Body::from(b));
let h = res.headers_mut();
h.insert(
"content-type",
HeaderValue::from_static("multipart/form-data;boundary=boundary"),
);
Box::new(res)
});
let multipart_form_data =
warp::path("multipart_form_data.txt").map(|| -> Box<dyn Reply> {
let b = "Preamble\r\n\
--boundary\t \r\n\
Content-Disposition: form-data; name=\"field_1\"\r\n\
\r\n\
value_1 \r\n\
\r\n--boundary\r\n\
Content-Disposition: form-data; name=\"field_2\";\
filename=\"file.js\"\r\n\
Content-Type: text/javascript\r\n\
\r\n\
console.log(\"Hi\")\
\r\n--boundary--\r\n\
Epilogue";
let mut res = Response::new(Body::from(b));
res.headers_mut().insert(
"content-type",
HeaderValue::from_static("multipart/form-data;boundary=boundary"),
);
Box::new(res)
});
let etag_script = warp::path!("etag_script.ts")
.and(warp::header::optional::<String>("if-none-match"))
.map(|if_none_match| -> Box<dyn Reply> {
if if_none_match == Some("33a64df551425fcc55e".to_string()) {
let r =
warp::reply::with_status(warp::reply(), StatusCode::NOT_MODIFIED);
let r = with_header(r, "Content-type", "application/typescript");
let r = with_header(r, "ETag", "33a64df551425fcc55e");
Box::new(r)
} else {
let mut res = Response::new(Body::from("console.log('etag')"));
let h = res.headers_mut();
h.insert(
"Content-type",
HeaderValue::from_static("application/typescript"),
);
h.insert("ETag", HeaderValue::from_static("33a64df551425fcc55e"));
Box::new(res)
}
});
let xtypescripttypes = warp::path!("xTypeScriptTypes.js")
.map(|| {
let mut res = Response::new(Body::from("export const foo = 'foo';"));
let h = res.headers_mut();
h.insert(
"Content-type",
HeaderValue::from_static("application/javascript"),
);
h.insert(
"X-TypeScript-Types",
HeaderValue::from_static("./xTypeScriptTypes.d.ts"),
);
res
})
.or(warp::path!("xTypeScriptTypes.d.ts").map(|| {
let mut res = Response::new(Body::from("export const foo: 'foo';"));
res.headers_mut().insert(
"Content-type",
HeaderValue::from_static("application/typescript"),
);
res
}))
.or(warp::path!("type_directives_redirect.js").map(|| {
let mut res = Response::new(Body::from("export const foo = 'foo';"));
let h = res.headers_mut();
h.insert(
"Content-type",
HeaderValue::from_static("application/javascript"),
);
h.insert(
"X-TypeScript-Types",
HeaderValue::from_static(
"http://localhost:4547/xTypeScriptTypesRedirect.d.ts",
),
);
res
}))
.or(warp::path!("cli"/"tests"/"subdir"/"xTypeScriptTypesRedirect.d.ts").map(|| {
let mut res = Response::new(Body::from(
"import './xTypeScriptTypesRedirected.d.ts';",
));
let h = res.headers_mut();
h.insert(
"Content-type",
HeaderValue::from_static("application/typescript"),
);
res
}))
.or(warp::path!("cli"/"tests"/"subdir"/"xTypeScriptTypesRedirected.d.ts").map(|| {
let mut res = Response::new(Body::from("export const foo: 'foo';"));
let h = res.headers_mut();
h.insert(
"Content-type",
HeaderValue::from_static("application/typescript"),
);
res
}))
.or(warp::path!("referenceTypes.js").map(|| {
let mut res = Response::new(Body::from("/// <reference types=\"./xTypeScriptTypes.d.ts\" />\r\nexport const foo = \"foo\";\r\n"));
let h = res.headers_mut();
h.insert(
"Content-type",
HeaderValue::from_static("application/javascript"),
);
res
}));
let content_type_handler = warp::any()
.and(warp::path::peek())
.and(warp::fs::dir(root_path()))
.map(custom_headers)
.or(etag_script)
.or(xtypescripttypes)
.or(echo_server)
.or(echo_multipart_file)
.or(multipart_form_data);
let http_fut =
warp::serve(content_type_handler.clone()).bind(([127, 0, 0, 1], PORT));
let https_fut = warp::serve(content_type_handler.clone())
.tls()
.cert_path("std/http/testdata/tls/localhost.crt")
.key_path("std/http/testdata/tls/localhost.key")
.bind(([127, 0, 0, 1], HTTPS_PORT));
let mut server_fut = async {
futures::join!(
http_fut,
https_fut,
redirect_server_fut,
another_redirect_server_fut,
inf_redirect_server_fut,
double_redirect_server_fut,
absolute_redirect_server_fut,
)
}
.boxed();
let mut did_print_ready = false;
future::poll_fn(move |cx| {
let poll_result = server_fut.poll_unpin(cx);
if !replace(&mut did_print_ready, true) {
println!("ready");
}
poll_result
})
.await;
}
fn custom_headers(path: warp::path::Peek, f: warp::fs::File) -> Box<dyn Reply> {
let p = path.as_str();
if p.ends_with("cli/tests/x_deno_warning.js") {
let f = with_header(f, "Content-Type", "application/javascript");
let f = with_header(f, "X-Deno-Warning", "foobar");
return Box::new(f);
}
if p.ends_with("cli/tests/053_import_compression/brotli") {
let f = with_header(f, "Content-Encoding", "br");
let f = with_header(f, "Content-Type", "application/javascript");
let f = with_header(f, "Content-Length", "26");
return Box::new(f);
}
if p.ends_with("cli/tests/053_import_compression/gziped") {
let f = with_header(f, "Content-Encoding", "gzip");
let f = with_header(f, "Content-Type", "application/javascript");
let f = with_header(f, "Content-Length", "39");
return Box::new(f);
}
let content_type = if p.contains(".t1.") {
Some("text/typescript")
} else if p.contains(".t2.") {
Some("video/vnd.dlna.mpeg-tts")
} else if p.contains(".t3.") {
Some("video/mp2t")
} else if p.contains(".t4.") {
Some("application/x-typescript")
} else if p.contains(".j1.") {
Some("text/javascript")
} else if p.contains(".j2.") {
Some("application/ecmascript")
} else if p.contains(".j3.") {
Some("text/ecmascript")
} else if p.contains(".j4.") {
Some("application/x-javascript")
} else if p.contains("form_urlencoded") {
Some("application/x-www-form-urlencoded")
} else if p.contains("unknown_ext") || p.contains("no_ext") {
Some("text/typescript")
} else if p.contains("mismatch_ext") {
Some("text/javascript")
} else if p.ends_with(".ts") || p.ends_with(".tsx") {
Some("application/typescript")
} else if p.ends_with(".js") || p.ends_with(".jsx") {
Some("application/javascript")
} else if p.ends_with(".json") {
Some("application/json")
} else {
None
};
if let Some(t) = content_type {
Box::new(with_header(f, "Content-Type", t))
} else {
Box::new(f)
}
}
pub struct HttpServerGuard<'a> {
#[allow(dead_code)]
g: MutexGuard<'a, ()>,
child: Child,
test_server: Child,
}
impl<'a> Drop for HttpServerGuard<'a> {
fn drop(&mut self) {
match self.child.try_wait() {
match self.test_server.try_wait() {
Ok(None) => {
self.child.kill().expect("failed to kill http_server.py");
self.test_server.kill().expect("failed to kill test_server");
let _ = self.test_server.wait();
}
Ok(Some(status)) => {
panic!("http_server.py exited unexpectedly {}", status)
}
Err(e) => panic!("http_server.py err {}", e),
Ok(Some(status)) => panic!("test_server exited unexpectedly {}", status),
Err(e) => panic!("test_server error: {}", e),
}
}
}
/// Starts tools/http_server.py when the returned guard is dropped, the server
/// Starts target/debug/test_server when the returned guard is dropped, the server
/// will be killed.
pub fn http_server<'a>() -> HttpServerGuard<'a> {
// TODO(bartlomieju) Allow tests to use the http server in parallel.
@ -86,18 +433,16 @@ pub fn http_server<'a>() -> HttpServerGuard<'a> {
r.unwrap()
};
println!("tools/http_server.py starting...");
let mut child = Command::new("python")
println!("test_server starting...");
let mut test_server = Command::new(test_server_path())
.current_dir(root_path())
.args(&["-u", "tools/http_server.py"])
.stdout(Stdio::piped())
.spawn()
.expect("failed to execute child");
.expect("failed to execute test_server");
let stdout = child.stdout.as_mut().unwrap();
let stdout = test_server.stdout.as_mut().unwrap();
use std::io::{BufRead, BufReader};
let lines = BufReader::new(stdout).lines();
// Wait for "ready" on stdout. See tools/http_server.py
for maybe_line in lines {
if let Ok(line) = maybe_line {
if line.starts_with("ready") {
@ -108,7 +453,7 @@ pub fn http_server<'a>() -> HttpServerGuard<'a> {
}
}
HttpServerGuard { child, g }
HttpServerGuard { test_server, g }
}
/// Helper function to strip ansi codes.

View file

@ -0,0 +1,3 @@
fn main() {
test_util::run_all_servers();
}

View file

@ -3,7 +3,7 @@
# Performs benchmark and append data to //website/data.json.
# If //website/data.json doesn't exist, this script tries to import it from
# gh-pages branch.
# To view the results locally run ./tools/http_server.py and visit
# To view the results locally run target/debug/test_server and visit
# http://localhost:4545/website
import os
@ -12,11 +12,11 @@ import json
import time
import tempfile
import subprocess
from util import build_path, executable_suffix, root_path, run, run_output
from util import (build_path, executable_suffix, root_path, run, run_output,
build_mode)
import third_party
from http_benchmark import http_benchmark
import throughput_benchmark
import http_server
# The list of the tuples of the benchmark name, arguments and return code
exec_time_benchmarks = [
@ -239,7 +239,6 @@ def main():
build_dir = build_path()
sha1 = run_output(["git", "rev-parse", "HEAD"],
exit_on_fail=True).out.strip()
http_server.spawn()
deno_exe = os.path.join(build_dir, "deno")
@ -253,7 +252,11 @@ def main():
# TODO(ry) The "benchmark" benchmark should actually be called "exec_time".
# When this is changed, the historical data in gh-pages branch needs to be
# changed too.
server_cmd = os.path.join("target", build_mode(), "test_server")
p = subprocess.Popen([server_cmd])
new_data["benchmark"] = run_exec_time(deno_exe, build_dir)
p.kill()
p.wait()
new_data["binary_size"] = get_binary_sizes(build_dir)
new_data["bundle_size"] = bundle_benchmark(deno_exe)

View file

@ -65,6 +65,4 @@ class TestBenchmark(DenoTestCase):
if __name__ == '__main__':
# FIME this doesn't appear to be the case.
# This test assumes tools/http_server.py is running in the background.
run_tests()

View file

@ -1,420 +0,0 @@
#!/usr/bin/env python
# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
# Many tests expect there to be an http server on port 4545 servering the deno
# root directory.
from collections import namedtuple
from contextlib import contextmanager
import os
import SimpleHTTPServer
import SocketServer
import socket
import sys
from time import sleep
from threading import Thread
from util import root_path
import ssl
import getopt
import argparse
PORT = 4545
REDIRECT_PORT = 4546
ANOTHER_REDIRECT_PORT = 4547
DOUBLE_REDIRECTS_PORT = 4548
INF_REDIRECTS_PORT = 4549
REDIRECT_ABSOLUTE_PORT = 4550
HTTPS_PORT = 5545
def create_http_arg_parser():
parser = argparse.ArgumentParser()
parser.add_argument('--verbose', '-v', action='store_true')
return parser
HttpArgParser = create_http_arg_parser()
args, unknown = HttpArgParser.parse_known_args(sys.argv[1:])
CERT_FILE = os.path.join(root_path, "std/http/testdata/tls/localhost.crt")
KEY_FILE = os.path.join(root_path, "std/http/testdata/tls/localhost.key")
QUIET = not args.verbose
class SSLTCPServer(SocketServer.TCPServer):
def __init__(self,
server_address,
request_handler,
certfile,
keyfile,
ssl_version=ssl.PROTOCOL_TLSv1_2,
bind_and_activate=True):
SocketServer.TCPServer.__init__(self, server_address, request_handler,
bind_and_activate)
self.certfile = certfile
self.keyfile = keyfile
self.ssl_version = ssl_version
def get_request(self):
newsocket, fromaddr = self.socket.accept()
connstream = ssl.wrap_socket(
newsocket,
server_side=True,
certfile=self.certfile,
keyfile=self.keyfile,
ssl_version=self.ssl_version)
return connstream, fromaddr
class SSLThreadingTCPServer(SocketServer.ThreadingMixIn, SSLTCPServer):
pass
class QuietSimpleHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def log_request(self, code='-', size='-'):
if not QUIET:
SimpleHTTPServer.SimpleHTTPRequestHandler.log_request(
self, code, size)
class ContentTypeHandler(QuietSimpleHTTPRequestHandler):
def do_GET(self):
# Check if there is a custom header configuration ending
# with ".header" before sending the file
maybe_header_file_path = "./" + self.path + ".header"
if os.path.exists(maybe_header_file_path):
self.protocol_version = 'HTTP/1.1'
self.send_response(200, 'OK')
f = open(maybe_header_file_path)
for line in f:
kv = line.split(": ")
self.send_header(kv[0].strip(), kv[1].strip())
f.close()
self.end_headers()
body = open("./" + self.path)
self.wfile.write(body.read())
body.close()
return
if "etag_script.ts" in self.path:
self.protocol_version = 'HTTP/1.1'
if_not_match = self.headers.getheader('if-none-match')
if if_not_match == "33a64df551425fcc55e":
self.send_response(304, 'Not Modified')
self.send_header('Content-type', 'application/typescript')
self.send_header('ETag', '33a64df551425fcc55e')
self.end_headers()
else:
self.send_response(200, 'OK')
self.send_header('Content-type', 'application/typescript')
self.send_header('ETag', '33a64df551425fcc55e')
self.end_headers()
self.wfile.write(bytes("console.log('etag')"))
return
if "xTypeScriptTypes.js" in self.path:
self.protocol_version = "HTTP/1.1"
self.send_response(200, 'OK')
self.send_header('Content-type', 'application/javascript')
self.send_header('X-TypeScript-Types', './xTypeScriptTypes.d.ts')
self.end_headers()
self.wfile.write(bytes("export const foo = 'foo';"))
return
if "type_directives_redirect.js" in self.path:
self.protocol_version = "HTTP/1.1"
self.send_response(200, 'OK')
self.send_header('Content-type', 'application/javascript')
self.send_header(
'X-TypeScript-Types',
'http://localhost:4547/xTypeScriptTypesRedirect.d.ts')
self.end_headers()
self.wfile.write(bytes("export const foo = 'foo';"))
return
if "xTypeScriptTypesRedirect.d.ts" in self.path:
self.protocol_version = "HTTP/1.1"
self.send_response(200, 'OK')
self.send_header('Content-type', 'application/typescript')
self.end_headers()
self.wfile.write(
bytes("import './xTypeScriptTypesRedirected.d.ts';"))
return
if "xTypeScriptTypesRedirected.d.ts" in self.path:
self.protocol_version = "HTTP/1.1"
self.send_response(200, 'OK')
self.send_header('Content-type', 'application/typescript')
self.end_headers()
self.wfile.write(bytes("export const foo: 'foo';"))
return
if "xTypeScriptTypes.d.ts" in self.path:
self.protocol_version = "HTTP/1.1"
self.send_response(200, 'OK')
self.send_header('Content-type', 'application/typescript')
self.end_headers()
self.wfile.write(bytes("export const foo: 'foo';"))
return
if "referenceTypes.js" in self.path:
self.protocol_version = "HTTP/1.1"
self.send_response(200, 'OK')
self.send_header('Content-type', 'application/javascript')
self.end_headers()
self.wfile.write(
bytes('/// <reference types="./xTypeScriptTypes.d.ts" />\r\n'
'export const foo = "foo";\r\n'))
return
if "multipart_form_data.txt" in self.path:
self.protocol_version = 'HTTP/1.1'
self.send_response(200, 'OK')
self.send_header('Content-type',
'multipart/form-data;boundary=boundary')
self.end_headers()
self.wfile.write(
bytes('Preamble\r\n'
'--boundary\t \r\n'
'Content-Disposition: form-data; name="field_1"\r\n'
'\r\n'
'value_1 \r\n'
'\r\n--boundary\r\n'
'Content-Disposition: form-data; name="field_2"; '
'filename="file.js"\r\n'
'Content-Type: text/javascript\r\n'
'\r\n'
'console.log("Hi")'
'\r\n--boundary--\r\n'
'Epilogue'))
return
return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
def do_POST(self):
# Simple echo server for request reflection
if "echo_server" in self.path:
status = int(self.headers.getheader('x-status', "200"))
self.protocol_version = 'HTTP/1.1'
self.send_response(status, 'OK')
if self.headers.has_key('content-type'):
self.send_header('content-type',
self.headers.getheader('content-type'))
if self.headers.has_key('user-agent'):
self.send_header('user-agent',
self.headers.getheader('user-agent'))
self.end_headers()
data_string = self.rfile.read(int(self.headers['Content-Length']))
self.wfile.write(bytes(data_string))
return
if "echo_multipart_file" in self.path:
self.protocol_version = 'HTTP/1.1'
self.send_response(200, 'OK')
self.send_header('Content-type',
'multipart/form-data;boundary=boundary')
self.end_headers()
file_content = self.rfile.read(int(self.headers['Content-Length']))
self.wfile.write(
bytes('--boundary\t \r\n'
'Content-Disposition: form-data; name="field_1"\r\n'
'\r\n'
'value_1 \r\n'
'\r\n--boundary\r\n'
'Content-Disposition: form-data; name="file"; '
'filename="file.bin"\r\n'
'Content-Type: application/octet-stream\r\n'
'\r\n') + bytes(file_content) +
bytes('\r\n--boundary--\r\n'))
return
self.protocol_version = 'HTTP/1.1'
self.send_response(501)
self.send_header('content-type', 'text/plain')
self.end_headers()
self.wfile.write(bytes('Server does not support this operation'))
def guess_type(self, path):
if ".t1." in path:
return "text/typescript"
if ".t2." in path:
return "video/vnd.dlna.mpeg-tts"
if ".t3." in path:
return "video/mp2t"
if ".t4." in path:
return "application/x-typescript"
if ".j1." in path:
return "text/javascript"
if ".j2." in path:
return "application/ecmascript"
if ".j3." in path:
return "text/ecmascript"
if ".j4." in path:
return "application/x-javascript"
if "form_urlencoded" in path:
return "application/x-www-form-urlencoded"
if "no_ext" in path:
return "text/typescript"
if "unknown_ext" in path:
return "text/typescript"
if "mismatch_ext" in path:
return "text/javascript"
return SimpleHTTPServer.SimpleHTTPRequestHandler.guess_type(self, path)
RunningServer = namedtuple("RunningServer", ["server", "thread"])
def get_socket(port, handler, use_https):
SocketServer.TCPServer.allow_reuse_address = True
if os.name != "nt":
# We use AF_INET6 to avoid flaky test issue, particularly with
# the test 019_media_types. It's not well understood why this fixes the
# flaky tests, but it does appear to...
# See https://github.com/denoland/deno/issues/3332
SocketServer.TCPServer.address_family = socket.AF_INET6
if use_https:
return SSLThreadingTCPServer(("", port), handler, CERT_FILE, KEY_FILE)
return SocketServer.TCPServer(("", port), handler)
def server():
os.chdir(root_path) # Hopefully the main thread doesn't also chdir.
Handler = ContentTypeHandler
Handler.extensions_map.update({
".ts": "application/typescript",
".js": "application/javascript",
".tsx": "application/typescript",
".jsx": "application/javascript",
".json": "application/json",
})
s = get_socket(PORT, Handler, False)
if not QUIET:
print "Deno test server http://localhost:%d/" % PORT
return RunningServer(s, start(s))
def base_redirect_server(host_port, target_port, extra_path_segment=""):
os.chdir(root_path)
target_host = "http://localhost:%d" % target_port
class RedirectHandler(QuietSimpleHTTPRequestHandler):
def do_GET(self):
self.send_response(301)
self.send_header('Location',
target_host + extra_path_segment + self.path)
self.end_headers()
s = get_socket(host_port, RedirectHandler, False)
if not QUIET:
print "redirect server http://localhost:%d/ -> http://localhost:%d/" % (
host_port, target_port)
return RunningServer(s, start(s))
# redirect server
def redirect_server():
return base_redirect_server(REDIRECT_PORT, PORT)
# another redirect server pointing to the same port as the one above
# BUT with an extra subdir path
def another_redirect_server():
return base_redirect_server(
ANOTHER_REDIRECT_PORT, PORT, extra_path_segment="/cli/tests/subdir")
# redirect server that points to another redirect server
def double_redirects_server():
return base_redirect_server(DOUBLE_REDIRECTS_PORT, REDIRECT_PORT)
# redirect server that points to itself
def inf_redirects_server():
return base_redirect_server(INF_REDIRECTS_PORT, INF_REDIRECTS_PORT)
# redirect server that redirect to absolute paths under same host
# redirects /REDIRECT/file_name to /file_name
def absolute_redirect_server():
os.chdir(root_path)
class AbsoluteRedirectHandler(ContentTypeHandler):
def do_GET(self):
print(self.path)
if (self.path.startswith("/REDIRECT/")):
self.send_response(302)
self.send_header('Location',
self.path.split('/REDIRECT', 1)[1])
self.end_headers()
else:
ContentTypeHandler.do_GET(self)
s = get_socket(REDIRECT_ABSOLUTE_PORT, AbsoluteRedirectHandler, False)
if not QUIET:
print("absolute redirect server http://localhost:%d/" %
REDIRECT_ABSOLUTE_PORT)
return RunningServer(s, start(s))
def https_server():
os.chdir(root_path) # Hopefully the main thread doesn't also chdir.
Handler = ContentTypeHandler
Handler.extensions_map.update({
".ts": "application/typescript",
".js": "application/javascript",
".tsx": "application/typescript",
".jsx": "application/javascript",
".json": "application/json",
})
s = get_socket(HTTPS_PORT, Handler, True)
if not QUIET:
print "Deno https test server https://localhost:%d/" % HTTPS_PORT
return RunningServer(s, start(s))
def start(s):
thread = Thread(target=s.serve_forever, kwargs={"poll_interval": 0.05})
thread.daemon = True
thread.start()
return thread
@contextmanager
def spawn():
servers = (server(), redirect_server(), another_redirect_server(),
double_redirects_server(), https_server(),
absolute_redirect_server(), inf_redirects_server())
# In order to wait for each of the servers to be ready, we try connecting to
# them with a tcp socket.
for running_server in servers:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
port = running_server.server.server_address[1]
client.connect(("127.0.0.1", port))
print "connected", port
client.close()
assert running_server.thread.is_alive()
# The following output "ready" is specificly looked for in cli/test_util.rs
# to prevent race conditions.
print "ready"
try:
yield servers
finally:
for s in servers:
# Make sure all servers still running,
# if not assume there was an error
assert s.thread.is_alive()
s.server.shutdown()
def main():
with spawn() as servers:
try:
while all(s.thread.is_alive() for s in servers):
sleep(1)
except KeyboardInterrupt:
pass
sys.exit(1)
if __name__ == '__main__':
main()

View file

@ -3,7 +3,7 @@
# Performs benchmark and append data to //website/data.json.
# If //website/data.json doesn't exist, this script tries to import it from
# gh-pages branch.
# To view the results locally run ./tools/http_server.py and visit
# To view the results locally run target/debug/test_server and visit
# http://localhost:4545/website
import os