diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 01c8803e65..b82b2fe7c3 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -24,12 +24,14 @@ const prCacheKeyPrefix = `${cacheVersion}-cargo-target-\${{ matrix.os }}-\${{ matrix.profile }}-\${{ matrix.job }}-`; const installPkgsCommand = - "sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15"; + "sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15 clang-tools-15 clang-format-15 clang-tidy-15"; const sysRootStep = { name: "Set up incremental LTO and sysroot build", run: `# Avoid running man-db triggers, which sometimes takes several minutes # to complete. sudo apt-get remove --purge -y man-db +# Remove older clang before we install +sudo apt-get remove 'clang-12*' 'clang-13*' 'clang-14*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'lld-12*' 'lld-13*' 'lld-14*' # Install clang-15, lld-15, and debootstrap. echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" | @@ -40,6 +42,8 @@ sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg sudo apt-get update # this was unreliable sometimes, so try again if it fails ${installPkgsCommand} || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && ${installPkgsCommand} +# Fix alternatives +(yes '' | sudo update-alternatives --force --all) || true # Create ubuntu-16.04 sysroot environment, which is used to avoid # depending on a very recent version of glibc. @@ -461,6 +465,7 @@ const ci = { "python --version", "rustc --version", "cargo --version", + "which dpkg && dpkg -l", // Deno is installed when linting. 'if [ "${{ matrix.job }}" == "lint" ]', "then", @@ -575,7 +580,12 @@ const ci = { "(github.ref == 'refs/heads/main' ||", "startsWith(github.ref, 'refs/tags/'))))", ].join("\n"), - run: "cargo build --release --locked --all-targets", + run: [ + // output fs space before and after building + "df -h", + "cargo build --release --locked --all-targets", + "df -h", + ].join("\n"), }, { name: "Upload PR artifact (linux)", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efa424e0bf..92833ac629 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,6 +209,8 @@ jobs: # Avoid running man-db triggers, which sometimes takes several minutes # to complete. sudo apt-get remove --purge -y man-db + # Remove older clang before we install + sudo apt-get remove 'clang-12*' 'clang-13*' 'clang-14*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'lld-12*' 'lld-13*' 'lld-14*' # Install clang-15, lld-15, and debootstrap. echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" | @@ -218,7 +220,9 @@ jobs: sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg sudo apt-get update # this was unreliable sometimes, so try again if it fails - sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15 + sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15 clang-tools-15 clang-format-15 clang-tidy-15 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends debootstrap clang-15 lld-15 clang-tools-15 clang-format-15 clang-tidy-15 + # Fix alternatives + (yes '' | sudo update-alternatives --force --all) || true # Create ubuntu-16.04 sysroot environment, which is used to avoid # depending on a very recent version of glibc. @@ -277,6 +281,7 @@ jobs: python --version rustc --version cargo --version + which dpkg && dpkg -l if [ "${{ matrix.job }}" == "lint" ] then deno --version @@ -349,7 +354,10 @@ jobs: (github.repository == 'denoland/deno' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))))) - run: cargo build --release --locked --all-targets + run: |- + df -h + cargo build --release --locked --all-targets + df -h - name: Upload PR artifact (linux) if: |- !(github.event_name == 'pull_request' && matrix.skip_pr) && (matrix.job == 'test' && diff --git a/Cargo.lock b/Cargo.lock index 0372622ac2..afbded5674 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1412,6 +1412,7 @@ dependencies = [ "fastwebsockets", "http", "hyper 0.14.26", + "once_cell", "serde", "tokio", "tokio-rustls", @@ -2788,9 +2789,9 @@ checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "libffi" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb06d5b4c428f3cd682943741c39ed4157ae989fffe1094a08eaf7c4014cf60" +checksum = "ce826c243048e3d5cec441799724de52e2d42f820468431fc3fceee2341871e2" dependencies = [ "libc", "libffi-sys", @@ -2798,9 +2799,9 @@ dependencies = [ [[package]] name = "libffi-sys" -version = "2.1.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c6f11e063a27ffe040a9d15f0b661bf41edc2383b7ae0e0ad5a7e7d53d9da3" +checksum = "dc65067b78c0fc069771e8b9a9e02df71e08858bec92c1f101377c67b9dca7c7" dependencies = [ "cc", ] diff --git a/cli/tests/unit_node/_fs/_fs_handle_test.ts b/cli/tests/unit_node/_fs/_fs_handle_test.ts index 165608e1ce..8cfbf64906 100644 --- a/cli/tests/unit_node/_fs/_fs_handle_test.ts +++ b/cli/tests/unit_node/_fs/_fs_handle_test.ts @@ -1,5 +1,7 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. import * as path from "../../../../test_util/std/path/mod.ts"; +import { Buffer } from "node:buffer"; +import * as fs from "node:fs/promises"; import { assert, assertEquals, @@ -7,14 +9,83 @@ import { const moduleDir = path.dirname(path.fromFileUrl(import.meta.url)); const testData = path.resolve(moduleDir, "testdata", "hello.txt"); +const decoder = new TextDecoder(); Deno.test("readFileSuccess", async function () { - const fs = await import("node:fs/promises"); const fileHandle = await fs.open(testData); const data = await fileHandle.readFile(); assert(data instanceof Uint8Array); - assertEquals(new TextDecoder().decode(data as Uint8Array), "hello world"); + assertEquals(decoder.decode(data as Uint8Array), "hello world"); await fileHandle.close(); }); + +Deno.test("read", async function () { + const fileHandle = await fs.open(testData); + const byteLength = "hello world".length; + + const buf = new Buffer(byteLength); + await fileHandle.read(buf, 0, byteLength, 0); + + assertEquals(decoder.decode(buf as Uint8Array), "hello world"); + + await fileHandle.close(); +}); + +Deno.test("read specify opt", async function () { + const fileHandle = await fs.open(testData); + const byteLength = "hello world".length; + + const opt = { + buffer: new Buffer(byteLength), + offset: 6, + length: 5, + }; + let res = await fileHandle.read(opt); + + assertEquals(res.bytesRead, byteLength); + assertEquals(new TextDecoder().decode(res.buffer as Uint8Array), "world"); + + const opt2 = { + buffer: new Buffer(byteLength), + length: 5, + position: 0, + }; + res = await fileHandle.read(opt2); + + assertEquals(res.bytesRead, byteLength); + assertEquals(decoder.decode(res.buffer as Uint8Array), "hello"); + + await fileHandle.close(); +}); + +Deno.test("[node/fs filehandle.write] Write from Buffer", async function () { + const tempFile: string = await Deno.makeTempFile(); + const fileHandle = await fs.open(tempFile, "a+"); + + const buffer = Buffer.from("hello world"); + const res = await fileHandle.write(buffer, 0, 5, 0); + + const data = Deno.readFileSync(tempFile); + await Deno.remove(tempFile); + await fileHandle.close(); + + assertEquals(res.bytesWritten, 5); + assertEquals(decoder.decode(data), "hello"); +}); + +Deno.test("[node/fs filehandle.write] Write from string", async function () { + const tempFile: string = await Deno.makeTempFile(); + const fileHandle = await fs.open(tempFile, "a+"); + + const str = "hello world"; + const res = await fileHandle.write(str); + + const data = Deno.readFileSync(tempFile); + await Deno.remove(tempFile); + await fileHandle.close(); + + assertEquals(res.bytesWritten, 11); + assertEquals(decoder.decode(data), "hello world"); +}); diff --git a/ext/ffi/Cargo.toml b/ext/ffi/Cargo.toml index a1d2a68c34..e5093c8a76 100644 --- a/ext/ffi/Cargo.toml +++ b/ext/ffi/Cargo.toml @@ -17,8 +17,8 @@ path = "lib.rs" deno_core.workspace = true dlopen.workspace = true dynasmrt = "1.2.3" -libffi = "=3.1.0" -libffi-sys = "=2.1.0" # temporary pin for downgrade to Rust 1.69 +libffi = "=3.2.0" +libffi-sys = "=2.2.1" serde.workspace = true serde-value = "0.7" serde_json = "1.0" diff --git a/ext/http/http_next.rs b/ext/http/http_next.rs index 900a956f49..93634ae3e2 100644 --- a/ext/http/http_next.rs +++ b/ext/http/http_next.rs @@ -73,13 +73,13 @@ type Request = hyper1::Request; type Response = hyper1::Response; static USE_WRITEV: Lazy = Lazy::new(|| { - let disable_writev = std::env::var("DENO_HYPER_USE_WRITEV").ok(); + let enable = std::env::var("DENO_USE_WRITEV").ok(); - if let Some(val) = disable_writev { - return val != "0"; + if let Some(val) = enable { + return !val.is_empty(); } - true + false }); /// All HTTP/2 connections start with this byte string. diff --git a/ext/napi/lib.rs b/ext/napi/lib.rs index 22d86e4a9b..ada9aa13fb 100644 --- a/ext/napi/lib.rs +++ b/ext/napi/lib.rs @@ -647,12 +647,13 @@ where // SAFETY: we are going blind, calling the register function on the other side. let maybe_exports = unsafe { - let init = library + let Ok(init) = library .get:: napi_value>(b"napi_register_module_v1") - .expect("napi_register_module_v1 not found"); + ) -> napi_value>(b"napi_register_module_v1") else { + return Err(type_error(format!("Unable to find napi_register_module_v1 symbol in {}", path))); + }; init( env_ptr, std::mem::transmute::, napi_value>(exports.into()), diff --git a/ext/node/polyfills/_fs/_fs_common.ts b/ext/node/polyfills/_fs/_fs_common.ts index 19f0d7d176..4e8bfc2858 100644 --- a/ext/node/polyfills/_fs/_fs_common.ts +++ b/ext/node/polyfills/_fs/_fs_common.ts @@ -33,6 +33,13 @@ export type BinaryOptionsArgument = | ({ encoding: BinaryEncodings } & FileOptions); export type FileOptionsArgument = Encodings | FileOptions; +export type ReadOptions = { + buffer: Buffer | Uint8Array; + offset: number; + length: number; + position: number | null; +}; + export interface WriteFileOptions extends FileOptions { mode?: number; } diff --git a/ext/node/polyfills/_fs/_fs_read.ts b/ext/node/polyfills/_fs/_fs_read.ts index bce7d334f4..2c840f07cd 100644 --- a/ext/node/polyfills/_fs/_fs_read.ts +++ b/ext/node/polyfills/_fs/_fs_read.ts @@ -3,6 +3,7 @@ import { Buffer } from "ext:deno_node/buffer.ts"; import { ERR_INVALID_ARG_TYPE } from "ext:deno_node/internal/errors.ts"; import * as io from "ext:deno_io/12_io.js"; import * as fs from "ext:deno_fs/30_fs.js"; +import { ReadOptions } from "ext:deno_node/_fs/_fs_common.ts"; import { validateOffsetLengthRead, validatePosition, @@ -12,13 +13,6 @@ import { validateInteger, } from "ext:deno_node/internal/validators.mjs"; -type readOptions = { - buffer: Buffer | Uint8Array; - offset: number; - length: number; - position: number | null; -}; - type readSyncOptions = { offset: number; length: number; @@ -35,7 +29,7 @@ type Callback = BinaryCallback; export function read(fd: number, callback: Callback): void; export function read( fd: number, - options: readOptions, + options: ReadOptions, callback: Callback, ): void; export function read( @@ -48,7 +42,7 @@ export function read( ): void; export function read( fd: number, - optOrBufferOrCb?: Buffer | Uint8Array | readOptions | Callback, + optOrBufferOrCb?: Buffer | Uint8Array | ReadOptions | Callback, offsetOrCallback?: number | Callback, length?: number, position?: number | null, @@ -86,7 +80,7 @@ export function read( length = buffer.byteLength; position = null; } else { - const opt = optOrBufferOrCb as readOptions; + const opt = optOrBufferOrCb as ReadOptions; if ( !(opt.buffer instanceof Buffer) && !(opt.buffer instanceof Uint8Array) ) { diff --git a/ext/node/polyfills/internal/fs/handle.ts b/ext/node/polyfills/internal/fs/handle.ts index a1ee263ead..fbe535840b 100644 --- a/ext/node/polyfills/internal/fs/handle.ts +++ b/ext/node/polyfills/internal/fs/handle.ts @@ -1,13 +1,24 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. import { EventEmitter } from "ext:deno_node/events.ts"; import { Buffer } from "ext:deno_node/buffer.ts"; -import { promises } from "ext:deno_node/fs.ts"; +import { promises, read, write } from "ext:deno_node/fs.ts"; import { BinaryOptionsArgument, FileOptionsArgument, + ReadOptions, TextOptionsArgument, } from "ext:deno_node/_fs/_fs_common.ts"; +interface WriteResult { + bytesWritten: number; + buffer: Buffer | string; +} + +interface ReadResult { + bytesRead: number; + buffer: Buffer; +} + export class FileHandle extends EventEmitter { #rid: number; constructor(rid: number) { @@ -19,12 +30,104 @@ export class FileHandle extends EventEmitter { return this.rid; } + read( + buffer: Buffer, + offset?: number, + length?: number, + position?: number | null, + ): Promise; + read(options?: ReadOptions): Promise; + read( + bufferOrOpt: Buffer | ReadOptions, + offset?: number, + length?: number, + position?: number | null, + ): Promise { + if (bufferOrOpt instanceof Buffer) { + return new Promise((resolve, reject) => { + read( + this.fd, + bufferOrOpt, + offset, + length, + position, + (err, bytesRead, buffer) => { + if (err) reject(err); + else resolve({ buffer: buffer, bytesRead: bytesRead }); + }, + ); + }); + } else { + return new Promise((resolve, reject) => { + read(this.fd, bufferOrOpt, (err, bytesRead, buffer) => { + if (err) reject(err); + else resolve({ buffer: buffer, bytesRead: bytesRead }); + }); + }); + } + } + readFile( opt?: TextOptionsArgument | BinaryOptionsArgument | FileOptionsArgument, ): Promise { return promises.readFile(this, opt); } + write( + buffer: Buffer, + offset: number, + length: number, + position: number, + ): Promise; + write( + str: string, + position: number, + encoding: string, + ): Promise; + write( + bufferOrStr: Buffer | string, + offsetOrPotition: number, + lengthOrEncoding: number | string, + position?: number, + ): Promise { + if (bufferOrStr instanceof Buffer) { + const buffer = bufferOrStr; + const offset = offsetOrPotition; + const length = lengthOrEncoding; + + return new Promise((resolve, reject) => { + write( + this.fd, + buffer, + offset, + length, + position, + (err, bytesWritten, buffer) => { + if (err) reject(err); + else resolve({ buffer, bytesWritten }); + }, + ); + }); + } else { + const str = bufferOrStr; + const position = offsetOrPotition; + const encoding = lengthOrEncoding; + + return new Promise((resolve, reject) => { + write( + this.fd, + str, + position, + encoding, + (err, bytesWritten, buffer) => { + if (err) reject(err); + else resolve({ buffer, bytesWritten }); + }, + ); + }); + } + } + close(): Promise { // Note that Deno.close is not async return Promise.resolve(Deno.close(this.fd)); diff --git a/ext/websocket/Cargo.toml b/ext/websocket/Cargo.toml index 1ac465f698..324a2b1a43 100644 --- a/ext/websocket/Cargo.toml +++ b/ext/websocket/Cargo.toml @@ -21,6 +21,7 @@ deno_tls.workspace = true fastwebsockets = { workspace = true, features = ["upgrade"] } http.workspace = true hyper = { workspace = true, features = ["backports"] } +once_cell.workspace = true serde.workspace = true tokio.workspace = true tokio-rustls.workspace = true diff --git a/ext/websocket/lib.rs b/ext/websocket/lib.rs index af987c1e4b..b492be0c02 100644 --- a/ext/websocket/lib.rs +++ b/ext/websocket/lib.rs @@ -27,6 +27,7 @@ use http::Method; use http::Request; use http::Uri; use hyper::Body; +use once_cell::sync::Lazy; use serde::Serialize; use std::borrow::Cow; use std::cell::Cell; @@ -50,9 +51,18 @@ use fastwebsockets::Frame; use fastwebsockets::OpCode; use fastwebsockets::Role; use fastwebsockets::WebSocket; - mod stream; +static USE_WRITEV: Lazy = Lazy::new(|| { + let enable = std::env::var("DENO_USE_WRITEV").ok(); + + if let Some(val) = enable { + return !val.is_empty(); + } + + false +}); + #[derive(Clone)] pub struct WsRootStoreProvider(Option>); @@ -360,7 +370,7 @@ pub fn ws_create_server_stream( ), Role::Server, ); - ws.set_writev(true); + ws.set_writev(*USE_WRITEV); ws.set_auto_close(true); ws.set_auto_pong(true); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index b2cf8c5f28..434cca4575 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.68.0" +channel = "1.68.2" components = ["rustfmt", "clippy"]