fix(cli/buffer): allow Buffer to store MAX_SIZE bytes (#6570)

cli/js/buffer.ts

@@ -11,7 +11,7 @@ import { assert } from "./util.ts";
 // buffer.ReadFrom. As long as the Buffer has at least MIN_READ bytes beyond
 // what is required to hold the contents of r, readFrom() will not grow the
 // underlying buffer.
-const MIN_READ = 512;
+const MIN_READ = 32 * 1024;
 const MAX_SIZE = 2 ** 32 - 2;
 
 // `off` is the offset into `dst` where it will at which to begin writing values
@@ -133,17 +133,17 @@ export class Buffer implements Reader, ReaderSync, Writer, WriterSync {
       // we instead let capacity get twice as large so we
       // don't spend all our time copying.
       copyBytes(this.#buf.subarray(this.#off), this.#buf);
-    } else if (c > MAX_SIZE - c - n) {
+    } else if (c + n > MAX_SIZE) {
       throw new Error("The buffer cannot be grown beyond the maximum size.");
     } else {
       // Not enough space anywhere, we need to allocate.
-      const buf = new Uint8Array(2 * c + n);
+      const buf = new Uint8Array(Math.min(2 * c + n, MAX_SIZE));
       copyBytes(this.#buf.subarray(this.#off), buf);
       this.#buf = buf;
     }
     // Restore this.#off and len(this.#buf).
     this.#off = 0;
-    this.#reslice(m + n);
+    this.#reslice(Math.min(m + n, MAX_SIZE));
     return m;
   };
 
@@ -157,30 +157,48 @@ export class Buffer implements Reader, ReaderSync, Writer, WriterSync {
 
   async readFrom(r: Reader): Promise<number> {
     let n = 0;
+    const tmp = new Uint8Array(MIN_READ);
     while (true) {
-      const i = this.#grow(MIN_READ);
+      const shouldGrow = this.capacity - this.length < MIN_READ;
-      this.#reslice(i);
+      // read into tmp buffer if there's not enough room
-      const fub = new Uint8Array(this.#buf.buffer, i);
+      // otherwise read directly into the internal buffer
-      const nread = await r.read(fub);
+      const buf = shouldGrow
+        ? tmp
+        : new Uint8Array(this.#buf.buffer, this.length);
+
+      const nread = await r.read(buf);
       if (nread === null) {
         return n;
       }
-      this.#reslice(i + nread);
+
+      // write will grow if needed
+      if (shouldGrow) this.writeSync(buf.subarray(0, nread));
+      else this.#reslice(this.length + nread);
+
       n += nread;
     }
   }
 
   readFromSync(r: ReaderSync): number {
     let n = 0;
+    const tmp = new Uint8Array(MIN_READ);
     while (true) {
-      const i = this.#grow(MIN_READ);
+      const shouldGrow = this.capacity - this.length < MIN_READ;
-      this.#reslice(i);
+      // read into tmp buffer if there's not enough room
-      const fub = new Uint8Array(this.#buf.buffer, i);
+      // otherwise read directly into the internal buffer
-      const nread = r.readSync(fub);
+      const buf = shouldGrow
+        ? tmp
+        : new Uint8Array(this.#buf.buffer, this.length);
+
+      const nread = r.readSync(buf);
       if (nread === null) {
         return n;
       }
-      this.#reslice(i + nread);
+
+      // write will grow if needed
+      if (shouldGrow) this.writeSync(buf.subarray(0, nread));
+      else this.#reslice(this.length + nread);
+
       n += nread;
     }
   }

cli/tests/unit/buffer_test.ts

@@ -11,11 +11,19 @@ import {
   unitTest,
 } from "./test_util.ts";
 
+const MAX_SIZE = 2 ** 32 - 2;
 // N controls how many iterations of certain checks are performed.
 const N = 100;
 let testBytes: Uint8Array | null;
 let testString: string | null;
 
+let ignoreMaxSizeTests = false;
+try {
+  new ArrayBuffer(MAX_SIZE);
+} catch (e) {
+  ignoreMaxSizeTests = true;
+}
+
 function init(): void {
   if (testBytes == null) {
     testBytes = new Uint8Array(N);
@@ -167,6 +175,100 @@ unitTest(async function bufferTooLargeByteWrites(): Promise<void> {
   );
 });
 
+unitTest(
+  { ignore: ignoreMaxSizeTests },
+  function bufferGrowWriteMaxBuffer(): void {
+    const bufSize = 16 * 1024;
+    const capacities = [MAX_SIZE, MAX_SIZE - 1];
+    for (const capacity of capacities) {
+      let written = 0;
+      const buf = new Deno.Buffer();
+      const writes = Math.floor(capacity / bufSize);
+      for (let i = 0; i < writes; i++)
+        written += buf.writeSync(repeat("x", bufSize));
+
+      if (written < capacity) {
+        written += buf.writeSync(repeat("x", capacity - written));
+      }
+
+      assertEquals(written, capacity);
+    }
+  }
+);
+
+unitTest(
+  { ignore: ignoreMaxSizeTests },
+  async function bufferGrowReadCloseMaxBufferPlus1(): Promise<void> {
+    const reader = new Deno.Buffer(new ArrayBuffer(MAX_SIZE + 1));
+    const buf = new Deno.Buffer();
+
+    await assertThrowsAsync(
+      async () => {
+        await buf.readFrom(reader);
+      },
+      Error,
+      "grown beyond the maximum size"
+    );
+  }
+);
+
+unitTest(
+  { ignore: ignoreMaxSizeTests },
+  function bufferGrowReadSyncCloseMaxBufferPlus1(): void {
+    const reader = new Deno.Buffer(new ArrayBuffer(MAX_SIZE + 1));
+    const buf = new Deno.Buffer();
+
+    assertThrows(
+      () => {
+        buf.readFromSync(reader);
+      },
+      Error,
+      "grown beyond the maximum size"
+    );
+  }
+);
+
+unitTest(
+  { ignore: ignoreMaxSizeTests },
+  function bufferGrowReadSyncCloseToMaxBuffer(): void {
+    const capacities = [MAX_SIZE, MAX_SIZE - 1];
+    for (const capacity of capacities) {
+      const reader = new Deno.Buffer(new ArrayBuffer(capacity));
+      const buf = new Deno.Buffer();
+      buf.readFromSync(reader);
+
+      assertEquals(buf.length, capacity);
+    }
+  }
+);
+
+unitTest(
+  { ignore: ignoreMaxSizeTests },
+  async function bufferGrowReadCloseToMaxBuffer(): Promise<void> {
+    const capacities = [MAX_SIZE, MAX_SIZE - 1];
+    for (const capacity of capacities) {
+      const reader = new Deno.Buffer(new ArrayBuffer(capacity));
+      const buf = new Deno.Buffer();
+      await buf.readFrom(reader);
+      assertEquals(buf.length, capacity);
+    }
+  }
+);
+
+unitTest(
+  { ignore: ignoreMaxSizeTests },
+  async function bufferReadCloseToMaxBufferWithInitialGrow(): Promise<void> {
+    const capacities = [MAX_SIZE, MAX_SIZE - 1, MAX_SIZE - 512];
+    for (const capacity of capacities) {
+      const reader = new Deno.Buffer(new ArrayBuffer(capacity));
+      const buf = new Deno.Buffer();
+      buf.grow(MAX_SIZE);
+      await buf.readFrom(reader);
+      assertEquals(buf.length, capacity);
+    }
+  }
+);
+
 unitTest(async function bufferLargeByteReads(): Promise<void> {
   init();
   assert(testBytes);