Skip to content

Commit cbc338d

Browse files
authored
test(cuda_core): capture machine state on the first CUDA OOM (#2458)
* test(cuda_core): capture machine state on the first CUDA OOM * rewrite and reason checker
1 parent f39d288 commit cbc338d

3 files changed

Lines changed: 843 additions & 0 deletions

File tree

‎cuda_core/tests/conftest.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,24 @@ def pytest_configure(config):
5454
config.pluginmanager.register(_CudaCoreParallelPlugin(), name="_cuda_core_parallel_plugin")
5555

5656

57+
@pytest.hookimpl(wrapper=True)
58+
def pytest_runtest_makereport(item, call):
59+
# Runs the OOM reason checker on the first CUDA OOM of a session; see
60+
# issue #2381 and helpers/oom_diagnostics.py for why this is latched and
61+
# what it checks (host VA exhaustion vs. physical device memory).
62+
report = yield
63+
from helpers import oom_diagnostics
64+
65+
oom_diagnostics.record_if_oom(item, call, report)
66+
return report
67+
68+
69+
def pytest_terminal_summary(terminalreporter):
70+
from helpers import oom_diagnostics
71+
72+
oom_diagnostics.report_terminal_summary(terminalreporter)
73+
74+
5775
@contextmanager
5876
def _init_cuda_context():
5977
# TODO: rename this to e.g. init_context

0 commit comments

Comments
 (0)