feat (std/encoding): add binary module (#4274)

2025-03-03 09:31:22 -05:00 · 2020-03-11 06:16:08 +11:00 · 2020-03-11 06:16:08 +11:00 · a309dcdd0f
commit a309dcdd0f
parent 55119aaee2
4 changed files with 444 additions and 1 deletions
--- a/std/encoding/README.md
+++ b/std/encoding/README.md
@ -1,4 +1,31 @@
-# Encoding
+# encoding
+
+Helper module for dealing with external data structures.
+
+- [`base32`](#base32)
+- [`binary`](#binary)
+- [`csv`](#csv)
+- [`toml`](#toml)
+- [`yaml`](#yaml)
+
+## Binary
+
+Implements equivalent methods to Go's `encoding/binary` package.
+
+Available Functions:
+
+```typescript
+sizeof(dataType: RawTypes): number
+getNBytes(r: Deno.Reader, n: number): Promise<Uint8Array>
+varnum(b: Uint8Array, o: VarnumOptions = {}): number | Deno.EOF
+varbig(b: Uint8Array, o: VarbigOptions = {}): bigint | Deno.EOF
+putVarnum(b: Uint8Array, x: number, o: VarnumOptions = {}): number
+putVarbig(b: Uint8Array, x: bigint, o: VarbigOptions = {}): number
+readVarnum(r: Deno.Reader, o: VarnumOptions = {}): Promise<number>
+readVarbig(r: Deno.Reader, o: VarbigOptions = {}): Promise<bigint>
+writeVarnum(w: Deno.Writer, x: number, o: VarnumOptions = {}): Promise<number>
+writeVarbig(w: Deno.Writer, x: bigint, o: VarbigOptions = {}): Promise<number>
+```

 ## CSV

--- a/std/encoding/binary.ts
+++ b/std/encoding/binary.ts
@ -0,0 +1,250 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+
+import { UnexpectedEOFError } from "../io/bufio.ts";
+
+type RawBaseTypes = "int8" | "int16" | "int32" | "uint8" | "uint16" | "uint32";
+type RawNumberTypes = RawBaseTypes | "float32" | "float64";
+type RawBigTypes = RawBaseTypes | "int64" | "uint64";
+type RawTypes = RawNumberTypes | RawBigTypes;
+
+/** How encoded binary data is ordered. */
+export type Endianness = "little" | "big";
+
+/** Options for working with the `number` type. */
+export interface VarnumOptions {
+  /** The binary format used. */
+  dataType?: RawNumberTypes;
+  /** The binary encoding order used. */
+  endian?: Endianness;
+}
+
+/** Options for working with the `bigint` type. */
+export interface VarbigOptions {
+  /** The binary format used. */
+  dataType?: RawBigTypes;
+  /** The binary encoding order used. */
+  endian?: Endianness;
+}
+
+const rawTypeSizes = {
+  int8: 1,
+  uint8: 1,
+  int16: 2,
+  uint16: 2,
+  int32: 4,
+  uint32: 4,
+  int64: 8,
+  uint64: 8,
+  float32: 4,
+  float64: 8
+};
+
+/** Returns the number of bytes required to store the given data-type. */
+export function sizeof(dataType: RawTypes): number {
+  return rawTypeSizes[dataType];
+}
+
+/** Reads `n` bytes from `r`.
+ *
+ * Returns it in a `Uint8Array`, or throws `UnexpectedEOFError` if `n` bytes cannot be read. */
+export async function getNBytes(
+  r: Deno.Reader,
+  n: number
+): Promise<Uint8Array> {
+  const scratch = new Uint8Array(n);
+  const nRead = await r.read(scratch);
+  if (nRead === Deno.EOF || nRead < n) throw new UnexpectedEOFError();
+  return scratch;
+}
+
+/** Decode a number from `b`, and return it as a `number`. Data-type defaults to `int32`.
+ * Returns `EOF` if `b` is too short for the data-type given in `o`. */
+export function varnum(
+  b: Uint8Array,
+  o: VarnumOptions = {}
+): number | Deno.EOF {
+  o.dataType = o.dataType ?? "int32";
+  const littleEndian = (o.endian ?? "big") === "little" ? true : false;
+  if (b.length < sizeof(o.dataType)) return Deno.EOF;
+  const view = new DataView(b.buffer);
+  switch (o.dataType) {
+    case "int8":
+      return view.getInt8(0);
+    case "uint8":
+      return view.getUint8(0);
+    case "int16":
+      return view.getInt16(0, littleEndian);
+    case "uint16":
+      return view.getUint16(0, littleEndian);
+    case "int32":
+      return view.getInt32(0, littleEndian);
+    case "uint32":
+      return view.getUint32(0, littleEndian);
+    case "float32":
+      return view.getFloat32(0, littleEndian);
+    case "float64":
+      return view.getFloat64(0, littleEndian);
+  }
+}
+
+/** Decode an integer from `b`, and return it as a `bigint`. Data-type defaults to `int64`.
+ * Returns `EOF` if `b` is too short for the data-type given in `o`. */
+export function varbig(
+  b: Uint8Array,
+  o: VarbigOptions = {}
+): bigint | Deno.EOF {
+  o.dataType = o.dataType ?? "int64";
+  const littleEndian = (o.endian ?? "big") === "little" ? true : false;
+  if (b.length < sizeof(o.dataType)) return Deno.EOF;
+  const view = new DataView(b.buffer);
+  switch (o.dataType) {
+    case "int8":
+      return BigInt(view.getInt8(0));
+    case "uint8":
+      return BigInt(view.getUint8(0));
+    case "int16":
+      return BigInt(view.getInt16(0, littleEndian));
+    case "uint16":
+      return BigInt(view.getUint16(0, littleEndian));
+    case "int32":
+      return BigInt(view.getInt32(0, littleEndian));
+    case "uint32":
+      return BigInt(view.getUint32(0, littleEndian));
+    case "int64":
+      return view.getBigInt64(0, littleEndian);
+    case "uint64":
+      return view.getBigUint64(0, littleEndian);
+  }
+}
+
+/** Encode a number `x` into `b`, and return the number of bytes used. Data-type defaults to `int32`.
+ * Returns 0 if `b` is too short for the data-type given in `o`. */
+export function putVarnum(
+  b: Uint8Array,
+  x: number,
+  o: VarnumOptions = {}
+): number {
+  o.dataType = o.dataType ?? "int32";
+  const littleEndian = (o.endian ?? "big") === "little" ? true : false;
+  if (b.length < sizeof(o.dataType)) return 0;
+  const view = new DataView(b.buffer);
+  switch (o.dataType) {
+    case "int8":
+      view.setInt8(0, x);
+      break;
+    case "uint8":
+      view.setUint8(0, x);
+      break;
+    case "int16":
+      view.setInt16(0, x, littleEndian);
+      break;
+    case "uint16":
+      view.setUint16(0, x, littleEndian);
+      break;
+    case "int32":
+      view.setInt32(0, x, littleEndian);
+      break;
+    case "uint32":
+      view.setUint32(0, x, littleEndian);
+      break;
+    case "float32":
+      view.setFloat32(0, x, littleEndian);
+      break;
+    case "float64":
+      view.setFloat64(0, x, littleEndian);
+      break;
+  }
+  return sizeof(o.dataType);
+}
+
+/** Encode an integer `x` into `b`, and return the number of bytes used. Data-type defaults to `int64`.
+ * Returns 0 if `b` is too short for the data-type given in `o`. */
+export function putVarbig(
+  b: Uint8Array,
+  x: bigint,
+  o: VarbigOptions = {}
+): number {
+  o.dataType = o.dataType ?? "int64";
+  const littleEndian = (o.endian ?? "big") === "little" ? true : false;
+  if (b.length < sizeof(o.dataType)) return 0;
+  const view = new DataView(b.buffer);
+  switch (o.dataType) {
+    case "int8":
+      view.setInt8(0, Number(x));
+      break;
+    case "uint8":
+      view.setUint8(0, Number(x));
+      break;
+    case "int16":
+      view.setInt16(0, Number(x), littleEndian);
+      break;
+    case "uint16":
+      view.setUint16(0, Number(x), littleEndian);
+      break;
+    case "int32":
+      view.setInt32(0, Number(x), littleEndian);
+      break;
+    case "uint32":
+      view.setUint32(0, Number(x), littleEndian);
+      break;
+    case "int64":
+      view.setBigInt64(0, x, littleEndian);
+      break;
+    case "uint64":
+      view.setBigUint64(0, x, littleEndian);
+      break;
+  }
+  return sizeof(o.dataType);
+}
+
+/** Reads a number from `r`, comsuming `sizeof(o.dataType)` bytes. Data-type defaults to `int32`.
+ *
+ * Returns it as `number`, or throws `UnexpectedEOFError` if not enough bytes can be read. */
+export async function readVarnum(
+  r: Deno.Reader,
+  o: VarnumOptions = {}
+): Promise<number> {
+  o.dataType = o.dataType ?? "int32";
+  const scratch = await getNBytes(r, sizeof(o.dataType));
+  return varnum(scratch, o) as number;
+}
+
+/** Reads an integer from `r`, comsuming `sizeof(o.dataType)` bytes. Data-type defaults to `int64`.
+ *
+ * Returns it as `bigint`, or throws `UnexpectedEOFError` if not enough bytes can be read. */
+export async function readVarbig(
+  r: Deno.Reader,
+  o: VarbigOptions = {}
+): Promise<bigint> {
+  o.dataType = o.dataType ?? "int64";
+  const scratch = await getNBytes(r, sizeof(o.dataType));
+  return varbig(scratch, o) as bigint;
+}
+
+/** Writes a number `x` to `w`. Data-type defaults to `int32`.
+ *
+ * Returns the number of bytes written. */
+export function writeVarnum(
+  w: Deno.Writer,
+  x: number,
+  o: VarnumOptions = {}
+): Promise<number> {
+  o.dataType = o.dataType ?? "int32";
+  const scratch = new Uint8Array(sizeof(o.dataType));
+  putVarnum(scratch, x, o);
+  return w.write(scratch);
+}
+
+/** Writes an integer `x` to `w`. Data-type defaults to `int64`.
+ *
+ * Returns the number of bytes written. */
+export function writeVarbig(
+  w: Deno.Writer,
+  x: bigint,
+  o: VarbigOptions = {}
+): Promise<number> {
+  o.dataType = o.dataType ?? "int64";
+  const scratch = new Uint8Array(sizeof(o.dataType));
+  putVarbig(scratch, x, o);
+  return w.write(scratch);
+}
--- a/std/encoding/binary_test.ts
+++ b/std/encoding/binary_test.ts
@ -0,0 +1,165 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+
+import { assertEquals, assertThrowsAsync } from "../testing/asserts.ts";
+import { UnexpectedEOFError } from "../io/bufio.ts";
+import {
+  getNBytes,
+  putVarbig,
+  putVarnum,
+  readVarbig,
+  readVarnum,
+  sizeof,
+  varbig,
+  varnum,
+  writeVarbig,
+  writeVarnum
+} from "./binary.ts";
+
+Deno.test(async function testGetNBytes(): Promise<void> {
+  const data = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+  const buff = new Deno.Buffer(data.buffer);
+  const rslt = await getNBytes(buff, 8);
+  assertEquals(rslt, data);
+});
+
+Deno.test(async function testGetNBytesThrows(): Promise<void> {
+  const data = new Uint8Array([1, 2, 3, 4]);
+  const buff = new Deno.Buffer(data.buffer);
+  assertThrowsAsync(async () => {
+    await getNBytes(buff, 8);
+  }, UnexpectedEOFError);
+});
+
+Deno.test(async function testPutVarbig(): Promise<void> {
+  const buff = new Uint8Array(8);
+  putVarbig(buff, 0xffeeddccbbaa9988n);
+  assertEquals(
+    buff,
+    new Uint8Array([0xff, 0xee, 0xdd, 0xcc, 0xbb, 0xaa, 0x99, 0x88])
+  );
+});
+
+Deno.test(async function testPutVarbigLittleEndian(): Promise<void> {
+  const buff = new Uint8Array(8);
+  putVarbig(buff, 0x8899aabbccddeeffn, { endian: "little" });
+  assertEquals(
+    buff,
+    new Uint8Array([0xff, 0xee, 0xdd, 0xcc, 0xbb, 0xaa, 0x99, 0x88])
+  );
+});
+
+Deno.test(async function testPutVarnum(): Promise<void> {
+  const buff = new Uint8Array(4);
+  putVarnum(buff, 0xffeeddcc);
+  assertEquals(buff, new Uint8Array([0xff, 0xee, 0xdd, 0xcc]));
+});
+
+Deno.test(async function testPutVarnumLittleEndian(): Promise<void> {
+  const buff = new Uint8Array(4);
+  putVarnum(buff, 0xccddeeff, { endian: "little" });
+  assertEquals(buff, new Uint8Array([0xff, 0xee, 0xdd, 0xcc]));
+});
+
+Deno.test(async function testReadVarbig(): Promise<void> {
+  const data = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+  const buff = new Deno.Buffer(data.buffer);
+  const rslt = await readVarbig(buff);
+  assertEquals(rslt, 0x0102030405060708n);
+});
+
+Deno.test(async function testReadVarbigLittleEndian(): Promise<void> {
+  const data = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+  const buff = new Deno.Buffer(data.buffer);
+  const rslt = await readVarbig(buff, { endian: "little" });
+  assertEquals(rslt, 0x0807060504030201n);
+});
+
+Deno.test(async function testReadVarnum(): Promise<void> {
+  const data = new Uint8Array([1, 2, 3, 4]);
+  const buff = new Deno.Buffer(data.buffer);
+  const rslt = await readVarnum(buff);
+  assertEquals(rslt, 0x01020304);
+});
+
+Deno.test(async function testReadVarnumLittleEndian(): Promise<void> {
+  const data = new Uint8Array([1, 2, 3, 4]);
+  const buff = new Deno.Buffer(data.buffer);
+  const rslt = await readVarnum(buff, { endian: "little" });
+  assertEquals(rslt, 0x04030201);
+});
+
+Deno.test(function testSizeof(): void {
+  assertEquals(1, sizeof("int8"));
+  assertEquals(1, sizeof("uint8"));
+  assertEquals(2, sizeof("int16"));
+  assertEquals(2, sizeof("uint16"));
+  assertEquals(4, sizeof("int32"));
+  assertEquals(4, sizeof("uint32"));
+  assertEquals(8, sizeof("int64"));
+  assertEquals(8, sizeof("uint64"));
+  assertEquals(4, sizeof("float32"));
+  assertEquals(8, sizeof("float64"));
+});
+
+Deno.test(function testVarbig(): void {
+  const data = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+  const rslt = varbig(data);
+  assertEquals(rslt, 0x0102030405060708n);
+});
+
+Deno.test(function testVarbigLittleEndian(): void {
+  const data = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+  const rslt = varbig(data, { endian: "little" });
+  assertEquals(rslt, 0x0807060504030201n);
+});
+
+Deno.test(function testVarnum(): void {
+  const data = new Uint8Array([1, 2, 3, 4]);
+  const rslt = varnum(data);
+  assertEquals(rslt, 0x01020304);
+});
+Deno.test(function testVarnumLittleEndian(): void {
+  const data = new Uint8Array([1, 2, 3, 4]);
+  const rslt = varnum(data, { endian: "little" });
+  assertEquals(rslt, 0x04030201);
+});
+
+Deno.test(async function testWriteVarbig(): Promise<void> {
+  const data = new Uint8Array(8);
+  const buff = new Deno.Buffer();
+  await writeVarbig(buff, 0x0102030405060708n);
+  await buff.read(data);
+  assertEquals(
+    data,
+    new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08])
+  );
+});
+
+Deno.test(async function testWriteVarbigLittleEndian(): Promise<void> {
+  const data = new Uint8Array(8);
+  const buff = new Deno.Buffer();
+  await writeVarbig(buff, 0x0807060504030201n, { endian: "little" });
+  await buff.read(data);
+  assertEquals(
+    data,
+    new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08])
+  );
+});
+
+Deno.test(async function testWriteVarnum(): Promise<void> {
+  const data = new Uint8Array(4);
+  const buff = new Deno.Buffer();
+  await writeVarnum(buff, 0x01020304);
+  await buff.read(data);
+  assertEquals(data, new Uint8Array([0x01, 0x02, 0x03, 0x04]));
+});
+
+Deno.test(async function testWriteVarnumLittleEndian(): Promise<void> {
+  const data = new Uint8Array(4);
+  const buff = new Deno.Buffer();
+  await writeVarnum(buff, 0x04030201, { endian: "little" });
+  await buff.read(data);
+  assertEquals(data, new Uint8Array([0x01, 0x02, 0x03, 0x04]));
+});
+
+Deno.runTests();
--- a/std/encoding/mod.ts
+++ b/std/encoding/mod.ts
@ -12,3 +12,4 @@ export {
 } from "./hex.ts";
 export { parse as parseToml, stringify as tomlStringify } from "./toml.ts";
 export { parse as parseYaml, stringify as yamlStringify } from "./yaml.ts";
+export * from "./binary.ts";