bugc: end each source range at its last token

packages/bugc/CHANGELOG.md

@@ -79,6 +79,11 @@ support. Changes to the specification itself are tracked in the root
 
 ### Fixed
 
+- A source range now ends at its node's last token. Before, a node
+  followed by whitespace or a comment included them, so most `code`
+  ranges had trailing whitespace: `block.number ` in `block.number + 1`,
+  the target `x ` in `x = 1`, a statement up to the next statement, and
+  the `create` and `code` blocks up to the next block ([#362]).
 - A call to a function with no return type now compiles as a statement,
   as in `bump();`, at every optimization level. The function's `return`
   context has no `data`. Before, IR generation failed with "Cannot convert
@@ -290,3 +295,4 @@ First publication.
 [#349]: https://github.com/ethdebug/format/pull/349
 [#351]: https://github.com/ethdebug/format/pull/351
 [#352]: https://github.com/ethdebug/format/pull/352
+[#362]: https://github.com/ethdebug/format/pull/362

packages/bugc/src/parser/locations.test.ts

@@ -0,0 +1,97 @@
+import { describe, it, expect } from "vitest";
+import { promises as fs } from "fs";
+import path from "path";
+import { glob } from "glob";
+
+import { parse } from "./parser.js";
+
+const EXAMPLES_DIR = path.resolve(__dirname, "../../examples");
+
+/**
+ * The spans of the comments in a source, skipping string literals
+ */
+function commentSpans(source: string): [number, number][] {
+  const spans: [number, number][] = [];
+  let i = 0;
+  while (i < source.length) {
+    if (source[i] === '"') {
+      const close = source.indexOf('"', i + 1);
+      i = close === -1 ? source.length : close + 1;
+    } else if (source.startsWith("//", i)) {
+      const newline = source.indexOf("\n", i);
+      const end = newline === -1 ? source.length : newline;
+      spans.push([i, end]);
+      i = end;
+    } else if (source.startsWith("/*", i)) {
+      const close = source.indexOf("*/", i + 2);
+      const end = close === -1 ? source.length : close + 2;
+      spans.push([i, end]);
+      i = end;
+    } else {
+      i++;
+    }
+  }
+  return spans;
+}
+
+/**
+ * Every source location in an AST, with the node's kind
+ */
+function locations(
+  node: unknown,
+  found: { kind: string; offset: number; length: number }[] = [],
+) {
+  if (Array.isArray(node)) {
+    for (const item of node) {
+      locations(item, found);
+    }
+  } else if (node instanceof Map) {
+    for (const value of node.values()) {
+      locations(value, found);
+    }
+  } else if (node && typeof node === "object") {
+    const { kind, loc } = node as {
+      kind?: string;
+      loc?: { offset: number; length: number } | null;
+    };
+    if (loc) {
+      found.push({
+        kind: kind ?? "?",
+        offset: Number(loc.offset),
+        length: Number(loc.length),
+      });
+    }
+    for (const value of Object.values(node)) {
+      locations(value, found);
+    }
+  }
+  return found;
+}
+
+describe("source locations", () => {
+  it("should end each node at its last token", async () => {
+    const files = await glob("**/*.bug", { cwd: EXAMPLES_DIR });
+    expect(files.length).toBeGreaterThan(0);
+
+    for (const file of files) {
+      const source = await fs.readFile(path.join(EXAMPLES_DIR, file), "utf-8");
+      const result = parse(source);
+      if (!result.success) {
+        continue;
+      }
+
+      const comments = commentSpans(source);
+      for (const { kind, offset, length } of locations(result.value)) {
+        const text = source.slice(offset, offset + length);
+        const end = offset + length;
+        const where = `${file}: ${kind} ${JSON.stringify(text)}`;
+
+        expect(text.trim(), where).toBe(text);
+        expect(
+          comments.some(([start, stop]) => start < end && end <= stop),
+          where,
+        ).toBe(false);
+      }
+    }
+  });
+});

packages/bugc/src/parser/parser.ts

@@ -16,6 +16,7 @@ const PENDING_ID = "<pending>" as Ast.Id;
  * Parse a BUG program and return Result
  */
 export function parse(input: string): Result<Ast.Program, ParseError> {
+  tokenEnds.clear();
   return runParser(parser, input);
 }
 
@@ -29,10 +30,22 @@ export function parse(input: string): Result<Ast.Program, ParseError> {
 function toSourceLocation(mark: P.Mark<unknown>): Ast.SourceLocation {
   return {
     offset: mark.start.offset,
-    length: mark.end.offset - mark.start.offset,
+    length: tokenEnd(mark.start.offset, mark.end.offset) - mark.start.offset,
   };
 }
 
+/**
+ * For the input being parsed: the end of each token, before the
+ * whitespace and comments after it, by the index after them. A node's
+ * location ends at its last token's end, not after the whitespace and
+ * comments that follow it.
+ */
+const tokenEnds = new Map<number, number>();
+
+function tokenEnd(start: number, end: number): number {
+  return Math.max(start, tokenEnds.get(end) ?? end);
+}
+
 /**
  * Basic parsers
  */
@@ -51,7 +64,11 @@ const _ = whitespaceOrComment.many().result(undefined);
 const __ = whitespaceOrComment.atLeast(1).result(undefined);
 
 // Lexeme: consume trailing whitespace/comments
-const lexeme = <T>(p: P.Parser<T>): P.Parser<T> => p.skip(_);
+const lexeme = <T>(p: P.Parser<T>): P.Parser<T> =>
+  P.seqMap(p, P.index, _, P.index, (value, end, _skipped, next) => {
+    tokenEnds.set(next.offset, end.offset);
+    return value;
+  });
 
 // Keywords - must not be followed by identifier chars
 const keyword = (name: string): P.Parser<string> =>
@@ -651,7 +668,7 @@ const postfixExpression = P.lazy(() => {
     suffixes.reduce<Ast.Expression>((obj, [suffix, end]) => {
       const loc = {
         offset: start.offset,
-        length: end.offset - start.offset,
+        length: tokenEnd(start.offset, end.offset) - start.offset,
       };
       const id = `${loc.offset}_${loc.length}` as Ast.Id;