Skip to content

Commit 7af7d13

Browse files
authored
bugc: keep a slice of calldata in calldata (#367)
* bugc: keep a slice of calldata in calldata A slice of msg.data, or of another slice of calldata, now refers to the calldata instead of copying it to memory. Its value is one word: the offset in the high 128 bits, the length in the low 128 bits. A local's pointer reads that word and then a calldata region whose offset and length are computed from it. A let typed `bytes`, an assignment to memory or storage, a cast to `bytes` or `string`, a call argument, a return value and the argument of keccak256 copy the bytes to memory. * bugc: link the changelog entry to #367 * bugc: write bytes in calldata as `bytes calldata` `let text: bytes calldata = msg.data[a:b]` is a calldata reference, as an untyped slice of msg.data is; `let text: bytes = ...` copies to memory. `bytes calldata` can only be the type of a let or of a cast.
1 parent 1d45fea commit 7af7d13

35 files changed

Lines changed: 1144 additions & 146 deletions

‎packages/bugc/CHANGELOG.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,24 @@ support. Changes to the specification itself are tracked in the root
109109
function whose `return` is only inside a loop needs one after it.
110110
Before, such a function compiled, and the compiler ended it with a
111111
`return` that gave no value ([#352]).
112+
- A new type, `bytes calldata`, is bytes in calldata, as in Solidity.
113+
`msg.data` and a slice of it have this type, and a slice of calldata
114+
refers to the calldata instead of copying it, as in
115+
`let text: bytes calldata = msg.data[68:68 + n];` or
116+
`let text = msg.data[68:68 + n];`. A `let` typed `bytes` copies the
117+
bytes to memory. `bytes calldata` can only be the type of a `let` or
118+
of a cast. Such a value is one word, with the offset in its high 128
119+
bits and the length in its low 128 bits. `.length`, `text[i]` (a
120+
`CALLDATALOAD`), slicing and a cast to `bytesN` read the calldata. An
121+
assignment to a `bytes` local or to storage, a cast to `bytes` or
122+
`string`, a function argument or return value, and the argument of
123+
`keccak256` copy the bytes to memory. A local's pointer names its word
124+
and then a `calldata` region whose offset and length are `$quotient`
125+
and `$remainder` of that word by 2^128, so it reads exactly the local's
126+
bytes, even when its bounds are computed. Writing to bytes in
127+
calldata, or assigning or casting memory bytes to `bytes calldata`, is
128+
a type error. Before, every slice was copied to memory, and a local's
129+
pointer read the copy ([#367]).
112130

113131
### Fixed
114132

@@ -404,3 +422,4 @@ First publication.
404422
[#364]: https://github.com/ethdebug/format/pull/364
405423
[#365]: https://github.com/ethdebug/format/pull/365
406424
[#366]: https://github.com/ethdebug/format/pull/366
425+
[#367]: https://github.com/ethdebug/format/pull/367

‎packages/bugc/examples/advanced/token-registry.bug‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ code {
128128
}
129129

130130
// Extract 4-byte function selector from calldata
131-
let selector = msg.data[0:4];
131+
let selector: bytes = msg.data[0:4];
132132

133133
// Compute function selectors from signatures (optimizer should evaluate these as constants)
134134
// These keccak256 computations should be optimized out to constants

‎packages/bugc/src/ast/spec.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -464,16 +464,29 @@ export namespace Type {
464464
return { id, kind: "type:elementary:bool", loc: loc ?? null };
465465
}
466466

467+
/** `bytesN`, `bytes`, or `bytes calldata` (`location`) */
467468
export interface Bytes extends Elementary.Base {
468469
kind: "type:elementary:bytes";
469470
size?: number;
471+
location?: "calldata";
470472
}
471473

472474
export const isBytes = (type: Type.Base): type is Bytes =>
473475
type.kind === "type:elementary:bytes";
474476

475-
export function bytes(id: Id, size?: number, loc?: SourceLocation): Bytes {
476-
return { id, kind: "type:elementary:bytes", size, loc: loc ?? null };
477+
export function bytes(
478+
id: Id,
479+
size?: number,
480+
loc?: SourceLocation,
481+
location?: "calldata",
482+
): Bytes {
483+
return {
484+
id,
485+
kind: "type:elementary:bytes",
486+
size,
487+
...(location ? { location } : {}),
488+
loc: loc ?? null,
489+
};
477490
}
478491

479492
export interface String extends Elementary.Base {

‎packages/bugc/src/evmgen/bounds.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ describe("memory bytes bounds", () => {
257257
"read and write the first and last bytes": [
258258
`storage { [0] r0: uint256; [1] r2: uint256; }
259259
code {
260-
let b = msg.data[0:3];
260+
let b: bytes = msg.data[0:3];
261261
b[0] = b[0] + 1 as uint8;
262262
b[2] = b[2] + 1 as uint8;
263263
r0 = b[0];
@@ -272,15 +272,15 @@ code {
272272
"read at the length": [
273273
`storage { [0] r: uint256; }
274274
code {
275-
let b = msg.data[0:3];
275+
let b: bytes = msg.data[0:3];
276276
r = b[3];
277277
}`,
278278
"0x0a0b0c",
279279
],
280280
"write at the length": [
281281
`storage { [0] r: uint256; }
282282
code {
283-
let b = msg.data[0:3];
283+
let b: bytes = msg.data[0:3];
284284
r = 1;
285285
b[3] = 1 as uint8;
286286
}`,
Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
import { describe, it, expect } from "vitest";
2+
import { keccak256 } from "ethereum-cryptography/keccak";
3+
import { bytesToHex, hexToBytes } from "ethereum-cryptography/utils";
4+
5+
import { executeProgram } from "#test/evm/behavioral";
6+
7+
const levels = [0, 1, 2, 3] as const;
8+
9+
const word = (n: bigint) => n.toString(16).padStart(64, "0");
10+
11+
/** Calldata for `setMotd(string)`, ABI-encoded, without the 0x */
12+
function setMotd(text: string): string {
13+
const data = Buffer.from(text).toString("hex");
14+
const padded = data.padEnd(Math.ceil(data.length / 64) * 64, "0");
15+
return "deadbeef" + word(32n) + word(BigInt(text.length)) + padded;
16+
}
17+
18+
/** The slot of `out[i]`, with `[3] out: mapping<uint256, uint256>` */
19+
const outSlot = (i: number) =>
20+
BigInt("0x" + bytesToHex(keccak256(hexToBytes(word(BigInt(i)) + word(3n)))));
21+
22+
/** A string's slot as Solidity stores it, and its data slots */
23+
function stored(text: string, slot: bigint): Map<bigint, bigint> {
24+
const bytes = Buffer.from(text);
25+
const slots = new Map<bigint, bigint>();
26+
if (bytes.length < 32) {
27+
const data = Buffer.alloc(32);
28+
bytes.copy(data);
29+
data[31] = bytes.length * 2;
30+
slots.set(slot, BigInt("0x" + data.toString("hex")));
31+
return slots;
32+
}
33+
slots.set(slot, BigInt(bytes.length * 2 + 1));
34+
const base = BigInt("0x" + bytesToHex(keccak256(hexToBytes(word(slot)))));
35+
for (let i = 0; i * 32 < bytes.length; i++) {
36+
const data = Buffer.alloc(32);
37+
bytes.copy(data, 0, i * 32, i * 32 + 32);
38+
slots.set(base + BigInt(i), BigInt("0x" + data.toString("hex")));
39+
}
40+
return slots;
41+
}
42+
43+
/**
44+
* Declare `text`, with an annotation (`: bytes calldata` or `: bytes`)
45+
* or none. Its bounds come from its ABI offset and length words.
46+
*/
47+
const decodeAs = (annotation: string) => `
48+
let offset = msg.data[4:36] as bytes32 as uint256;
49+
let n = msg.data[4 + offset:36 + offset] as bytes32 as uint256;
50+
let text${annotation} = msg.data[36 + offset:36 + offset + n];`;
51+
52+
const decode = decodeAs("");
53+
54+
// A calldata reference untyped or typed so, and a copy in memory
55+
const annotations = ["", ": bytes calldata", ": bytes"];
56+
57+
const texts = [
58+
"",
59+
"hello",
60+
"a".repeat(31),
61+
"b".repeat(32),
62+
"the quick brown fox jumps over the lazy dog, twice",
63+
];
64+
65+
describe.each(levels)("a slice of calldata at O%i", (level) => {
66+
for (const [text, annotation] of texts.flatMap((text) =>
67+
annotations.map((annotation) => [text, annotation] as const),
68+
)) {
69+
const label = `${text.length} bytes, \`let text${annotation}\``;
70+
const decode = decodeAs(annotation);
71+
72+
it(`reads its length and bytes from calldata, ${label}`, async () => {
73+
const result = await executeProgram(
74+
`name Read;
75+
storage {
76+
[0] motd: string; [1] len: uint256; [3] out: mapping<uint256, uint256>;
77+
}
78+
code {${decode}
79+
len = text.length;
80+
for (let i = 0; i < text.length; i = i + 1) {
81+
out[i] = text[i];
82+
}
83+
}`,
84+
{ calldata: setMotd(text), optimizationLevel: level },
85+
);
86+
expect(result.callSuccess).toBe(true);
87+
expect(await result.getStorage(1n)).toBe(BigInt(text.length));
88+
for (let i = 0; i < text.length; i++) {
89+
expect(await result.getStorage(outSlot(i)), `byte ${i}`).toBe(
90+
BigInt(text.charCodeAt(i)),
91+
);
92+
}
93+
});
94+
95+
it(`stores its bytes to storage, ${label}`, async () => {
96+
const result = await executeProgram(
97+
`name Store;
98+
storage { [0] motd: string; [1] raw: bytes; }
99+
code {${decode}
100+
motd = text as string;
101+
raw = text;
102+
}`,
103+
{ calldata: setMotd(text), optimizationLevel: level },
104+
);
105+
expect(result.callSuccess).toBe(true);
106+
for (const slot of [0n, 1n]) {
107+
for (const [at, value] of stored(text, slot)) {
108+
expect(await result.getStorage(at), `slot ${at}`).toBe(value);
109+
}
110+
}
111+
});
112+
}
113+
114+
it("copies to a memory local, a cast and a parameter", async () => {
115+
const text = "the quick brown fox jumps over the lazy dog, twice";
116+
const result = await executeProgram(
117+
`name Copy;
118+
define {
119+
function sum(b: bytes) -> uint256 {
120+
let total = 0;
121+
for (let i = 0; i < b.length; i = i + 1) {
122+
total = total + b[i];
123+
}
124+
return total;
125+
};
126+
}
127+
storage {
128+
[0] motd: string; [1] len: uint256; [2] total: uint256;
129+
[3] out: mapping<uint256, uint256>;
130+
}
131+
code {${decode}
132+
let m: bytes = text;
133+
m[0] = 88;
134+
len = m.length;
135+
let c = text as bytes;
136+
c[1] = 89;
137+
out[0] = m[0];
138+
out[1] = text[0];
139+
out[2] = c[1];
140+
out[3] = text[1];
141+
total = sum(text);
142+
}`,
143+
{ calldata: setMotd(text), optimizationLevel: level },
144+
);
145+
expect(result.callSuccess).toBe(true);
146+
expect(await result.getStorage(1n)).toBe(BigInt(text.length));
147+
expect(await result.getStorage(outSlot(0))).toBe(0x58n);
148+
expect(await result.getStorage(outSlot(1))).toBe(
149+
BigInt(text.charCodeAt(0)),
150+
);
151+
expect(await result.getStorage(outSlot(2))).toBe(89n);
152+
expect(await result.getStorage(outSlot(3))).toBe(
153+
BigInt(text.charCodeAt(1)),
154+
);
155+
expect(await result.getStorage(2n)).toBe(
156+
[...Buffer.from(text)].reduce((a, b) => a + BigInt(b), 0n),
157+
);
158+
});
159+
160+
it("slices a slice, and casts one to bytesN", async () => {
161+
const result = await executeProgram(
162+
`name SliceCast;
163+
storage {
164+
[0] a: bytes32; [1] b: bytes4; [2] c: uint256;
165+
[3] out: mapping<uint256, uint256>;
166+
}
167+
code {
168+
let all = msg.data[2:msg.data.length];
169+
let part = all[1:4];
170+
a = part as bytes32;
171+
b = msg.data[0:4] as bytes4;
172+
c = part.length;
173+
out[0] = part[0];
174+
out[1] = part[2];
175+
}`,
176+
{ calldata: "0102030405060708", optimizationLevel: level },
177+
);
178+
expect(result.callSuccess).toBe(true);
179+
// Bytes past the slice's length are zero
180+
expect(await result.getStorage(0n)).toBe(0x040506n << 232n);
181+
expect(await result.getStorage(1n)).toBe(0x01020304n);
182+
expect(await result.getStorage(2n)).toBe(3n);
183+
expect(await result.getStorage(outSlot(0))).toBe(4n);
184+
expect(await result.getStorage(outSlot(1))).toBe(6n);
185+
});
186+
187+
it("reverts on an index or slice past the end", async () => {
188+
for (const access of ["text[5]", "text[2:6].length", "text[3:2].length"]) {
189+
const result = await executeProgram(
190+
`name Bounds;
191+
storage { [1] len: uint256; }
192+
code {${decode}
193+
len = ${access};
194+
}`,
195+
{ calldata: setMotd("hello"), optimizationLevel: level },
196+
);
197+
expect(result.callSuccess, access).toBe(false);
198+
}
199+
});
200+
});

‎packages/bugc/src/evmgen/debug/local-variables.soundness.test.ts‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,56 @@ code { r = g(${arg}, 0); }`,
408408
* Programs and the values their locals take, in execution order. A
409409
* value with `after` may be read only once that statement has run.
410410
*/
411+
const motd = "hello world, this is longer than thirty-two bytes";
412+
413+
/** Calldata for `setMotd(string)`, ABI-encoded (hex, no 0x) */
414+
function setMotd(text: string): string {
415+
const word = (n: number) => n.toString(16).padStart(64, "0");
416+
const data = Buffer.from(text).toString("hex");
417+
return (
418+
"deadbeef" +
419+
word(32) +
420+
word(text.length) +
421+
data.padEnd(Math.ceil(data.length / 64) * 64, "0")
422+
);
423+
}
424+
425+
// `text` refers to the calldata, untyped or typed so, or is a copy in
426+
// memory
427+
const annotations = ["", ": bytes calldata", ": bytes"];
428+
429+
/**
430+
* A slice of calldata refers to the calldata. Its bounds here are
431+
* computed from the ABI offset and length words of a string argument,
432+
* and its pointer reads exactly its bytes (or, typed `bytes`, the
433+
* copy's).
434+
*/
435+
function setMotdProgram(annotation: string): LocalsProgram {
436+
return {
437+
name: `a calldata slice with computed bounds, \`let text${annotation}\``,
438+
source: `name SetMotd;
439+
storage { [0] motd: string; [1] r: uint256; }
440+
create { r = 1; }
441+
code {
442+
let offset = msg.data[4:36] as bytes32 as uint256;
443+
let n = msg.data[4 + offset:36 + offset] as bytes32 as uint256;
444+
let text${annotation} = msg.data[36 + offset:36 + offset + n];
445+
motd = text as string;
446+
if (r > 0) { r = text.length + text[1]; }
447+
}`,
448+
calldata: setMotd(motd),
449+
locals: {
450+
offset: { values: [32n] },
451+
n: { values: [BigInt(motd.length)] },
452+
text: {
453+
shape: { kind: annotation === ": bytes" ? "bytes" : "calldata" },
454+
values: [textBytes(motd)],
455+
everyLevel: true,
456+
},
457+
},
458+
};
459+
}
460+
411461
const programs: LocalsProgram[] = [
412462
{
413463
name: "straight line",
@@ -1038,8 +1088,9 @@ code {
10381088
r = r + b.length;
10391089
}`,
10401090
// Called with no calldata, so a longer slice would revert
1041-
locals: { b: { shape: { kind: "bytes" }, values: ["0x"] } },
1091+
locals: { b: { shape: { kind: "calldata" }, values: ["0x"] } },
10421092
},
1093+
...annotations.map(setMotdProgram),
10431094
{
10441095
// A slice longer than a word: its pointer reads every byte
10451096
name: "a long bytes slice",

0 commit comments

Comments
 (0)