bugc: keep a slice of calldata in calldata

packages/bugc/CHANGELOG.md

@@ -109,6 +109,24 @@ support. Changes to the specification itself are tracked in the root
   function whose `return` is only inside a loop needs one after it.
   Before, such a function compiled, and the compiler ended it with a
   `return` that gave no value ([#352]).
+- A new type, `bytes calldata`, is bytes in calldata, as in Solidity.
+  `msg.data` and a slice of it have this type, and a slice of calldata
+  refers to the calldata instead of copying it, as in
+  `let text: bytes calldata = msg.data[68:68 + n];` or
+  `let text = msg.data[68:68 + n];`. A `let` typed `bytes` copies the
+  bytes to memory. `bytes calldata` can only be the type of a `let` or
+  of a cast. Such a value is one word, with the offset in its high 128
+  bits and the length in its low 128 bits. `.length`, `text[i]` (a
+  `CALLDATALOAD`), slicing and a cast to `bytesN` read the calldata. An
+  assignment to a `bytes` local or to storage, a cast to `bytes` or
+  `string`, a function argument or return value, and the argument of
+  `keccak256` copy the bytes to memory. A local's pointer names its word
+  and then a `calldata` region whose offset and length are `$quotient`
+  and `$remainder` of that word by 2^128, so it reads exactly the local's
+  bytes, even when its bounds are computed. Writing to bytes in
+  calldata, or assigning or casting memory bytes to `bytes calldata`, is
+  a type error. Before, every slice was copied to memory, and a local's
+  pointer read the copy ([#367]).
 
 ### Fixed
 
@@ -404,3 +422,4 @@ First publication.
 [#364]: https://github.com/ethdebug/format/pull/364
 [#365]: https://github.com/ethdebug/format/pull/365
 [#366]: https://github.com/ethdebug/format/pull/366
+[#367]: https://github.com/ethdebug/format/pull/367

packages/bugc/examples/advanced/token-registry.bug

@@ -128,7 +128,7 @@ code {
   }
 
   // Extract 4-byte function selector from calldata
-  let selector = msg.data[0:4];
+  let selector: bytes = msg.data[0:4];
 
   // Compute function selectors from signatures (optimizer should evaluate these as constants)
   // These keccak256 computations should be optimized out to constants

packages/bugc/src/ast/spec.ts

@@ -464,16 +464,29 @@ export namespace Type {
       return { id, kind: "type:elementary:bool", loc: loc ?? null };
     }
 
+    /** `bytesN`, `bytes`, or `bytes calldata` (`location`) */
     export interface Bytes extends Elementary.Base {
       kind: "type:elementary:bytes";
       size?: number;
+      location?: "calldata";
     }
 
     export const isBytes = (type: Type.Base): type is Bytes =>
       type.kind === "type:elementary:bytes";
 
-    export function bytes(id: Id, size?: number, loc?: SourceLocation): Bytes {
-      return { id, kind: "type:elementary:bytes", size, loc: loc ?? null };
+    export function bytes(
+      id: Id,
+      size?: number,
+      loc?: SourceLocation,
+      location?: "calldata",
+    ): Bytes {
+      return {
+        id,
+        kind: "type:elementary:bytes",
+        size,
+        ...(location ? { location } : {}),
+        loc: loc ?? null,
+      };
     }
 
     export interface String extends Elementary.Base {

packages/bugc/src/evmgen/bounds.test.ts

@@ -257,7 +257,7 @@ describe("memory bytes bounds", () => {
     "read and write the first and last bytes": [
       `storage { [0] r0: uint256; [1] r2: uint256; }
 code {
-  let b = msg.data[0:3];
+  let b: bytes = msg.data[0:3];
   b[0] = b[0] + 1 as uint8;
   b[2] = b[2] + 1 as uint8;
   r0 = b[0];
@@ -272,15 +272,15 @@ code {
     "read at the length": [
       `storage { [0] r: uint256; }
 code {
-  let b = msg.data[0:3];
+  let b: bytes = msg.data[0:3];
   r = b[3];
 }`,
       "0x0a0b0c",
     ],
     "write at the length": [
       `storage { [0] r: uint256; }
 code {
-  let b = msg.data[0:3];
+  let b: bytes = msg.data[0:3];
   r = 1;
   b[3] = 1 as uint8;
 }`,

packages/bugc/src/evmgen/calldata-slices.test.ts

@@ -0,0 +1,200 @@
+import { describe, it, expect } from "vitest";
+import { keccak256 } from "ethereum-cryptography/keccak";
+import { bytesToHex, hexToBytes } from "ethereum-cryptography/utils";
+
+import { executeProgram } from "#test/evm/behavioral";
+
+const levels = [0, 1, 2, 3] as const;
+
+const word = (n: bigint) => n.toString(16).padStart(64, "0");
+
+/** Calldata for `setMotd(string)`, ABI-encoded, without the 0x */
+function setMotd(text: string): string {
+  const data = Buffer.from(text).toString("hex");
+  const padded = data.padEnd(Math.ceil(data.length / 64) * 64, "0");
+  return "deadbeef" + word(32n) + word(BigInt(text.length)) + padded;
+}
+
+/** The slot of `out[i]`, with `[3] out: mapping<uint256, uint256>` */
+const outSlot = (i: number) =>
+  BigInt("0x" + bytesToHex(keccak256(hexToBytes(word(BigInt(i)) + word(3n)))));
+
+/** A string's slot as Solidity stores it, and its data slots */
+function stored(text: string, slot: bigint): Map<bigint, bigint> {
+  const bytes = Buffer.from(text);
+  const slots = new Map<bigint, bigint>();
+  if (bytes.length < 32) {
+    const data = Buffer.alloc(32);
+    bytes.copy(data);
+    data[31] = bytes.length * 2;
+    slots.set(slot, BigInt("0x" + data.toString("hex")));
+    return slots;
+  }
+  slots.set(slot, BigInt(bytes.length * 2 + 1));
+  const base = BigInt("0x" + bytesToHex(keccak256(hexToBytes(word(slot)))));
+  for (let i = 0; i * 32 < bytes.length; i++) {
+    const data = Buffer.alloc(32);
+    bytes.copy(data, 0, i * 32, i * 32 + 32);
+    slots.set(base + BigInt(i), BigInt("0x" + data.toString("hex")));
+  }
+  return slots;
+}
+
+/**
+ * Declare `text`, with an annotation (`: bytes calldata` or `: bytes`)
+ * or none. Its bounds come from its ABI offset and length words.
+ */
+const decodeAs = (annotation: string) => `
+  let offset = msg.data[4:36] as bytes32 as uint256;
+  let n = msg.data[4 + offset:36 + offset] as bytes32 as uint256;
+  let text${annotation} = msg.data[36 + offset:36 + offset + n];`;
+
+const decode = decodeAs("");
+
+// A calldata reference untyped or typed so, and a copy in memory
+const annotations = ["", ": bytes calldata", ": bytes"];
+
+const texts = [
+  "",
+  "hello",
+  "a".repeat(31),
+  "b".repeat(32),
+  "the quick brown fox jumps over the lazy dog, twice",
+];
+
+describe.each(levels)("a slice of calldata at O%i", (level) => {
+  for (const [text, annotation] of texts.flatMap((text) =>
+    annotations.map((annotation) => [text, annotation] as const),
+  )) {
+    const label = `${text.length} bytes, \`let text${annotation}\``;
+    const decode = decodeAs(annotation);
+
+    it(`reads its length and bytes from calldata, ${label}`, async () => {
+      const result = await executeProgram(
+        `name Read;
+storage {
+  [0] motd: string; [1] len: uint256; [3] out: mapping<uint256, uint256>;
+}
+code {${decode}
+  len = text.length;
+  for (let i = 0; i < text.length; i = i + 1) {
+    out[i] = text[i];
+  }
+}`,
+        { calldata: setMotd(text), optimizationLevel: level },
+      );
+      expect(result.callSuccess).toBe(true);
+      expect(await result.getStorage(1n)).toBe(BigInt(text.length));
+      for (let i = 0; i < text.length; i++) {
+        expect(await result.getStorage(outSlot(i)), `byte ${i}`).toBe(
+          BigInt(text.charCodeAt(i)),
+        );
+      }
+    });
+
+    it(`stores its bytes to storage, ${label}`, async () => {
+      const result = await executeProgram(
+        `name Store;
+storage { [0] motd: string; [1] raw: bytes; }
+code {${decode}
+  motd = text as string;
+  raw = text;
+}`,
+        { calldata: setMotd(text), optimizationLevel: level },
+      );
+      expect(result.callSuccess).toBe(true);
+      for (const slot of [0n, 1n]) {
+        for (const [at, value] of stored(text, slot)) {
+          expect(await result.getStorage(at), `slot ${at}`).toBe(value);
+        }
+      }
+    });
+  }
+
+  it("copies to a memory local, a cast and a parameter", async () => {
+    const text = "the quick brown fox jumps over the lazy dog, twice";
+    const result = await executeProgram(
+      `name Copy;
+define {
+  function sum(b: bytes) -> uint256 {
+    let total = 0;
+    for (let i = 0; i < b.length; i = i + 1) {
+      total = total + b[i];
+    }
+    return total;
+  };
+}
+storage {
+  [0] motd: string; [1] len: uint256; [2] total: uint256;
+  [3] out: mapping<uint256, uint256>;
+}
+code {${decode}
+  let m: bytes = text;
+  m[0] = 88;
+  len = m.length;
+  let c = text as bytes;
+  c[1] = 89;
+  out[0] = m[0];
+  out[1] = text[0];
+  out[2] = c[1];
+  out[3] = text[1];
+  total = sum(text);
+}`,
+      { calldata: setMotd(text), optimizationLevel: level },
+    );
+    expect(result.callSuccess).toBe(true);
+    expect(await result.getStorage(1n)).toBe(BigInt(text.length));
+    expect(await result.getStorage(outSlot(0))).toBe(0x58n);
+    expect(await result.getStorage(outSlot(1))).toBe(
+      BigInt(text.charCodeAt(0)),
+    );
+    expect(await result.getStorage(outSlot(2))).toBe(89n);
+    expect(await result.getStorage(outSlot(3))).toBe(
+      BigInt(text.charCodeAt(1)),
+    );
+    expect(await result.getStorage(2n)).toBe(
+      [...Buffer.from(text)].reduce((a, b) => a + BigInt(b), 0n),
+    );
+  });
+
+  it("slices a slice, and casts one to bytesN", async () => {
+    const result = await executeProgram(
+      `name SliceCast;
+storage {
+  [0] a: bytes32; [1] b: bytes4; [2] c: uint256;
+  [3] out: mapping<uint256, uint256>;
+}
+code {
+  let all = msg.data[2:msg.data.length];
+  let part = all[1:4];
+  a = part as bytes32;
+  b = msg.data[0:4] as bytes4;
+  c = part.length;
+  out[0] = part[0];
+  out[1] = part[2];
+}`,
+      { calldata: "0102030405060708", optimizationLevel: level },
+    );
+    expect(result.callSuccess).toBe(true);
+    // Bytes past the slice's length are zero
+    expect(await result.getStorage(0n)).toBe(0x040506n << 232n);
+    expect(await result.getStorage(1n)).toBe(0x01020304n);
+    expect(await result.getStorage(2n)).toBe(3n);
+    expect(await result.getStorage(outSlot(0))).toBe(4n);
+    expect(await result.getStorage(outSlot(1))).toBe(6n);
+  });
+
+  it("reverts on an index or slice past the end", async () => {
+    for (const access of ["text[5]", "text[2:6].length", "text[3:2].length"]) {
+      const result = await executeProgram(
+        `name Bounds;
+storage { [1] len: uint256; }
+code {${decode}
+  len = ${access};
+}`,
+        { calldata: setMotd("hello"), optimizationLevel: level },
+      );
+      expect(result.callSuccess, access).toBe(false);
+    }
+  });
+});

packages/bugc/src/evmgen/debug/local-variables.soundness.test.ts

@@ -408,6 +408,56 @@ code { r = g(${arg}, 0); }`,
  * Programs and the values their locals take, in execution order. A
  * value with `after` may be read only once that statement has run.
  */
+const motd = "hello world, this is longer than thirty-two bytes";
+
+/** Calldata for `setMotd(string)`, ABI-encoded (hex, no 0x) */
+function setMotd(text: string): string {
+  const word = (n: number) => n.toString(16).padStart(64, "0");
+  const data = Buffer.from(text).toString("hex");
+  return (
+    "deadbeef" +
+    word(32) +
+    word(text.length) +
+    data.padEnd(Math.ceil(data.length / 64) * 64, "0")
+  );
+}
+
+// `text` refers to the calldata, untyped or typed so, or is a copy in
+// memory
+const annotations = ["", ": bytes calldata", ": bytes"];
+
+/**
+ * A slice of calldata refers to the calldata. Its bounds here are
+ * computed from the ABI offset and length words of a string argument,
+ * and its pointer reads exactly its bytes (or, typed `bytes`, the
+ * copy's).
+ */
+function setMotdProgram(annotation: string): LocalsProgram {
+  return {
+    name: `a calldata slice with computed bounds, \`let text${annotation}\``,
+    source: `name SetMotd;
+storage { [0] motd: string; [1] r: uint256; }
+create { r = 1; }
+code {
+  let offset = msg.data[4:36] as bytes32 as uint256;
+  let n = msg.data[4 + offset:36 + offset] as bytes32 as uint256;
+  let text${annotation} = msg.data[36 + offset:36 + offset + n];
+  motd = text as string;
+  if (r > 0) { r = text.length + text[1]; }
+}`,
+    calldata: setMotd(motd),
+    locals: {
+      offset: { values: [32n] },
+      n: { values: [BigInt(motd.length)] },
+      text: {
+        shape: { kind: annotation === ": bytes" ? "bytes" : "calldata" },
+        values: [textBytes(motd)],
+        everyLevel: true,
+      },
+    },
+  };
+}
+
 const programs: LocalsProgram[] = [
   {
     name: "straight line",
@@ -1038,8 +1088,9 @@ code {
   r = r + b.length;
 }`,
     // Called with no calldata, so a longer slice would revert
-    locals: { b: { shape: { kind: "bytes" }, values: ["0x"] } },
+    locals: { b: { shape: { kind: "calldata" }, values: ["0x"] } },
   },
+  ...annotations.map(setMotdProgram),
   {
     // A slice longer than a word: its pointer reads every byte
     name: "a long bytes slice",