Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions engine/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ def map(self):
f"`{SM.probe_for(c, url, param)}`")
lines.append("")
path = os.path.join(self.eng.dir, "arsenal.md")
open(path, "w").write("\n".join(lines))
open(path, "w", encoding="utf-8").write("\n".join(lines))
self.eng.log(f"map: wrote arsenal map -> {path} ← focus your 20% here (or run --hunt to auto-test)")

# ---------------- parallel runner ----------------
Expand Down Expand Up @@ -411,7 +411,7 @@ def report(self):
# Coverage caveats are generated from the ledger, never hand-written.
lines += ["", closure.render_caveats(self.eng.state, cov)]
path = os.path.join(self.eng.dir, "report.md")
open(path, "w").write("\n".join(lines))
open(path, "w", encoding="utf-8").write("\n".join(lines))
self.eng.log(f"report: wrote {len(c)} confirmed finding(s) -> {path}")
if not self.mock:
try:
Expand Down Expand Up @@ -449,6 +449,13 @@ def run(self, phases):


def main():
# Windows/non-UTF-8 locales default stdout to the code page (cp1252), which
# crashes on the '->' / em-dash characters the engine prints. Force UTF-8.
for _stream in (sys.stdout, sys.stderr):
try:
_stream.reconfigure(encoding="utf-8")
except (AttributeError, ValueError):
pass
ap = argparse.ArgumentParser()
ap.add_argument("--scope", required=True, help="engagement/scope JSON (name, in_scope, out_of_scope, seeds)")
ap.add_argument("--base", default="~/.bughunter-engagements")
Expand Down
8 changes: 4 additions & 4 deletions engine/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ def _write_rollup(host, tech_stack, tested, confirmed):
prev = {}
if os.path.exists(p):
try:
prev = json.load(open(p))
prev = json.load(open(p, encoding="utf-8"))
except Exception:
prev = {}
hc_conf = [f for f in confirmed if _host(f.get("url", "")) == host]
Expand All @@ -182,7 +182,7 @@ def _write_rollup(host, tech_stack, tested, confirmed):
for f in hc_conf],
}
tmp = p + ".tmp"
json.dump(doc, open(tmp, "w"), indent=2)
json.dump(doc, open(tmp, "w", encoding="utf-8"), indent=2)
os.replace(tmp, p)


Expand Down Expand Up @@ -235,7 +235,7 @@ def skip_decision(host, tech_stack, item):

def rollup(host):
p = os.path.join(_root(), "targets", f"{host}.json")
return json.load(open(p)) if os.path.exists(p) else None
return json.load(open(p, encoding="utf-8")) if os.path.exists(p) else None


def gc(action="report", max_mb=10, keep=KEEP, root=None):
Expand Down Expand Up @@ -273,7 +273,7 @@ def _selftest():
negs = _read(os.path.join(d, "negatives.jsonl"))
assert any(n["vuln_class"] == "xss" for n in negs) # xss tested, not confirmed -> negative
assert not any(n["vuln_class"] == "idor" for n in negs) # idor confirmed -> no negative
assert json.load(open(os.path.join(d, "targets", "a.com.json")))["sessions"] == 1
assert json.load(open(os.path.join(d, "targets", "a.com.json"), encoding="utf-8"))["sessions"] == 1
# skip_decision: known-confirmed carries the finding
skip_res = skip_decision("a.com", ["nextjs"], {"url": "http://a.com/x", "param": "id", "vuln_class": "idor"})
assert skip_res["skip"] and skip_res["reason"] == "known-confirmed" and skip_res["carry"]["verdict"]["real"] is True
Expand Down
2 changes: 1 addition & 1 deletion engine/scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ def __init__(self, in_scope, out_of_scope=None, seeds=None, name="engagement"):
@classmethod
def load(cls, path):
import json
d = json.load(open(path))
d = json.load(open(path, encoding="utf-8"))
return cls(d.get("in_scope", []), d.get("out_of_scope", []),
d.get("seeds", []), d.get("name", "engagement"))

Expand Down
13 changes: 9 additions & 4 deletions engine/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,22 +39,27 @@ def _fresh_defaults(self):

def _load(self):
if os.path.isfile(self.state_path):
s = json.load(open(self.state_path))
s = json.load(open(self.state_path, encoding="utf-8"))
for k, v in self._fresh_defaults().items(): # forward-compat with pre-ledger engagements
s.setdefault(k, v)
return s
return {"name": self.name, "created": _now(), "phase": "init", **self._fresh_defaults()}

def save(self):
tmp = self.state_path + ".tmp"
json.dump(self.state, open(tmp, "w"), indent=2)
json.dump(self.state, open(tmp, "w", encoding="utf-8"), indent=2)
os.replace(tmp, self.state_path)

def log(self, msg):
line = f"{_now()} {msg}"
with open(self.log_path, "a") as f:
with open(self.log_path, "a", encoding="utf-8") as f:
f.write(line + "\n")
print(" " + msg, flush=True)
try:
print(" " + msg, flush=True)
except UnicodeEncodeError:
# A non-UTF-8 console (Windows cp1252) must not crash the run; the
# full line is already in the UTF-8 log file.
print(" " + msg.encode("ascii", "replace").decode("ascii"), flush=True)

# ---- phase ----
def set_phase(self, phase):
Expand Down
8 changes: 4 additions & 4 deletions engine/test_coverage_ledger.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def _run(base, phases):
capture_output=True, text=True)
assert r.returncode == 0, f"engine failed:\n{r.stdout}\n{r.stderr}"
d = os.path.join(base, ".mock", "demo-local")
return json.load(open(os.path.join(d, "state.json"))), open(os.path.join(d, "report.md")).read()
return json.load(open(os.path.join(d, "state.json"), encoding="utf-8")), open(os.path.join(d, "report.md"), encoding="utf-8").read()


def test_full_run_records_every_pair():
Expand All @@ -52,13 +52,13 @@ def test_gap_reaches_the_report_file():
with tempfile.TemporaryDirectory() as t:
_run(t, "recon,rank,map,hunt,validate,report")
d = os.path.join(t, ".mock", "demo-local")
st = json.load(open(os.path.join(d, "state.json")))
st = json.load(open(os.path.join(d, "state.json"), encoding="utf-8"))
# mark one genuinely-discovered pair blocked, then re-report
host, cls = sorted(closure.expected_coverage(st))[0]
item = next(it for it in st["surface"]
if closure._host(it["url"]) == host and it["vuln_class"] == cls)
st["coverage_by_item"][Engagement._key(item)] = {"verdict": "blocked", "note": "edge 403 — needs allowlisted IP"}
json.dump(st, open(os.path.join(d, "state.json"), "w"))
json.dump(st, open(os.path.join(d, "state.json"), "w", encoding="utf-8"))
_, rep = _run(t, "report")
assert "Coverage: **INCOMPLETE**" in rep
assert "do not read as passing" in rep
Expand Down Expand Up @@ -93,7 +93,7 @@ def test_legacy_state_upgrades():
os.makedirs(os.path.join(t, "old"))
json.dump({"name": "old", "created": "x", "phase": "map", "targets": [], "surface": [],
"tested": [], "candidates": [], "confirmed": []},
open(os.path.join(t, "old", "state.json"), "w"))
open(os.path.join(t, "old", "state.json"), "w", encoding="utf-8"))
assert Engagement(t, "old").state["coverage"] == {}
print(" ok pre-ledger engagements load without migration")

Expand Down
4 changes: 2 additions & 2 deletions engine/test_mock_ledger.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def test_full_mock_run_ignores_engagement_ledger(self):
[sys.executable, str(ENGINE_DIR / "engine.py"), "--mock", "--hunt", "--use-memory",
"--scope", str(ENGINE_DIR / "engagement.example.json"), "--base", base],
env={**os.environ, "BUGHUNTER_MEMORY_DIR": str(ledger)},
capture_output=True, text=True, check=True,
capture_output=True, text=True, check=True, encoding="utf-8",
)
self.assertNotIn("skipped via ledger", result.stdout)
self.assertIn("CONFIRMED idor", result.stdout)
Expand All @@ -42,7 +42,7 @@ def test_mock_run_does_not_read_or_write_live_engagement_files(self):
report.write_text("live report", encoding="utf-8")
command = [sys.executable, str(ENGINE_DIR / "engine.py"), "--mock", "--hunt",
"--scope", str(ENGINE_DIR / "engagement.example.json"), "--base", base]
subprocess.run(command, check=True, capture_output=True, text=True)
subprocess.run(command, check=True, capture_output=True, text=True, encoding="utf-8")
self.assertEqual(state.read_text(encoding="utf-8"), '{"phase":"recon","marker":"live"}')
self.assertEqual(report.read_text(encoding="utf-8"), "live report")
self.assertFalse((live / "engine.log").exists())
Expand Down
35 changes: 35 additions & 0 deletions engine/test_windows_encoding.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
"""Regression test for issue #190 (engine UnicodeEncodeError on non-UTF-8 locales).
On Windows the default code page is cp1252, so text-mode open() calls without an
explicit encoding crash when the engine writes '->' / em-dash characters to the
log, arsenal map, or report. We can reproduce that deterministically on any host
by forcing an ASCII default encoding (PYTHONUTF8=0 + PYTHONCOERCECLOCALE=0 +
LC_ALL=C), which is what Windows hits with cp1252.
"""
import os
import subprocess
import sys
import tempfile
import unittest

_ENGINE = os.path.dirname(os.path.abspath(__file__))
_REPO = os.path.dirname(_ENGINE)
_ASCII_ENV = dict(os.environ, PYTHONUTF8="0", PYTHONCOERCECLOCALE="0", LC_ALL="C")


class WindowsEncodingTests(unittest.TestCase):
def test_mock_hunt_run_survives_ascii_locale(self):
"""Full --mock --hunt flow must finish without a UnicodeEncodeError."""
with tempfile.TemporaryDirectory() as base:
r = subprocess.run(
[sys.executable, "engine/engine.py",
"--scope", "engine/engagement.example.json",
"--base", base, "--mock", "--hunt"],
cwd=_REPO, env=_ASCII_ENV, capture_output=True, text=True, encoding="utf-8")
self.assertEqual(r.returncode, 0,
f"engine crashed under ASCII locale:\n{r.stderr}")
self.assertNotIn("UnicodeEncodeError", r.stderr)


if __name__ == "__main__":
unittest.main()
Loading