diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6f794ef58f..86bd18bed93 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -90,7 +90,7 @@ repos: name: Check tests do not create uncapped memory pools entry: python ./ci/tools/check_mempool_hygiene.py language: python - files: '^cuda_core/tests/.*\.py$' + files: '^(cuda_core|cuda_bindings_1[23])/tests/.*\.py$' - id: no-markdown-in-docs-source name: Prevent markdown files in docs/source directories diff --git a/ci/tools/check_mempool_hygiene.py b/ci/tools/check_mempool_hygiene.py index b200aba1ebb..f1837f2a919 100644 --- a/ci/tools/check_mempool_hygiene.py +++ b/ci/tools/check_mempool_hygiene.py @@ -9,7 +9,22 @@ address space, after which the rest of the session fails with CUDA_ERROR_OUT_OF_MEMORY on a device with free physical memory. -See cuda_core/tests/AGENTS.md for the rule this enforces. +Two rules are enforced: + +* cuda_core: ``DeviceMemoryResource`` / ``PinnedMemoryResource`` and their + options classes must be given ``max_size``. +* cuda.bindings: a pool properties object (``MemPoolProps_v1``, + ``CUmemPoolProps``, ``cudaMemPoolProps``) assigned to a name must have + ``max_size`` / ``maxSize`` set in the same function (or at module level, for + module-level code). Graph alloc-node ``params.poolProps`` is a field, not a + constructor call, so it is not checked; that pool cannot be capped anyway. + +Only ``.py`` files are checked. ``ast`` cannot parse Cython, so ``.pyx`` tests +(e.g. ``cuda_bindings_*/tests/cython``) are out of scope and must be capped by +hand. + +See cuda_core/tests/AGENTS.md and cuda_bindings_13/AGENTS.md for the rules this +enforces. """ from __future__ import annotations @@ -17,18 +32,39 @@ import argparse import ast import sys +from collections.abc import Iterator from pathlib import Path +from typing import NamedTuple ROOT = Path(__file__).resolve().parents[2] -DEFAULT_TREE = ROOT / "cuda_core" / "tests" +DEFAULT_TREES = ( + ROOT / "cuda_core" / "tests", + ROOT / "cuda_bindings_12" / "tests", + ROOT / "cuda_bindings_13" / "tests", +) # Managed pools cannot be right-sized: cuMemPoolCreate requires maxSize == 0 for # managed pools, so ManagedMemoryResourceOptions has no max_size to set. CAPPABLE_OPTIONS = frozenset({"DeviceMemoryResourceOptions", "PinnedMemoryResourceOptions"}) CAPPABLE_RESOURCES = frozenset({"DeviceMemoryResource", "PinnedMemoryResource"}) +# Pool properties types of the cuda.bindings APIs; the size cap is a plain +# attribute, spelled ``max_size`` on the v2 types and ``maxSize`` on the others. +POOL_PROPS_TYPES = frozenset( + {"MemPoolProps_v1", "CUmemPoolProps", "CUmemPoolProps_v1", "CUmemPoolProps_st", "cudaMemPoolProps"} +) +POOL_PROPS_CAP_ATTRIBUTES = frozenset({"max_size", "maxSize"}) + OPT_OUT_MARKER = "uncapped-pool-ok" +RULE_RESOURCE = "resource" +RULE_PROPS = "props" + + +class Violation(NamedTuple): + rule: str + message: str + def _callee_name(node: ast.Call) -> str: func = node.func @@ -60,12 +96,95 @@ def _opted_out(lines: list[str], node: ast.AST) -> bool: return any(OPT_OUT_MARKER in line for line in lines[start:end]) -def violations_in(path: Path) -> list[str]: - """Return one message per uncapped pool construction in ``path``.""" - source = path.read_text(encoding="utf-8") - lines = source.splitlines() +def _walk_scope(scope: ast.AST) -> Iterator[ast.AST]: + """Yield ``scope``'s descendants in source order, not entering nested function/lambda/class bodies.""" + # Push children reversed so the leftmost (earliest) child pops first. + pending = list(reversed(list(ast.iter_child_nodes(scope)))) + while pending: + node = pending.pop() + yield node + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef)): + pending.extend(reversed(list(ast.iter_child_nodes(node)))) + + +def _assign_targets(node: ast.AST) -> list[ast.expr]: + """The assignment targets of ``node``, or ``[]`` if it is not an assignment.""" + if isinstance(node, ast.Assign): + return node.targets + if isinstance(node, (ast.AugAssign, ast.AnnAssign)): + return [node.target] + return [] + + +def _pool_props_violations(path: Path, scope: ast.AST, lines: list[str]) -> list[Violation]: + """Report each ``var = PoolPropsType(...)`` with no ``var.max_size`` set in the same scope. + + Processes nodes in source order. When ``var`` is reassigned to a new + construction, any cap set on the previous object is reset; only a cap that + follows the new construction (in the same scope) excuses it. + + Recurses into nested scopes so a cap set in another function does not count. + + Handles both plain and type-annotated assignments (``ast.Assign`` and + ``ast.AnnAssign``). + """ + found: list[Violation] = [] + # Maps variable name -> (construction node, type name) for the latest + # uncapped construction. Removed when the variable is capped; overwritten + # (with a report of the old entry if it was still pending) on reassignment. + pending: dict[str, tuple[ast.AST, str]] = {} + + for node in _walk_scope(scope): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + found += _pool_props_violations(path, node, lines) + continue + # Cap: props.max_size = ... or props.maxSize = ... + for target in _assign_targets(node): + if ( + isinstance(target, ast.Attribute) + and target.attr in POOL_PROPS_CAP_ATTRIBUTES + and isinstance(target.value, ast.Name) + ): + pending.pop(target.value.id, None) + # Construction: props = PoolPropsType(...) or props: T = PoolPropsType(...) + con_target = con_value = None + if isinstance(node, ast.Assign) and len(node.targets) == 1: + con_target, con_value = node.targets[0], node.value + elif isinstance(node, ast.AnnAssign) and node.value is not None: + con_target, con_value = node.target, node.value + if ( + isinstance(con_target, ast.Name) + and isinstance(con_value, ast.Call) + and _callee_name(con_value) in POOL_PROPS_TYPES + ): + var, type_name = con_target.id, _callee_name(con_value) + if var in pending: + prev_node, prev_type = pending.pop(var) + if not _opted_out(lines, prev_node): + found.append( + Violation( + RULE_PROPS, + f"{path.as_posix()}:{prev_node.lineno}: {prev_type} assigned to '{var}' " + f"without {var}.max_size / {var}.maxSize", + ) + ) + pending[var] = (node, type_name) + + for var, (node, type_name) in pending.items(): + if not _opted_out(lines, node): + found.append( + Violation( + RULE_PROPS, + f"{path.as_posix()}:{node.lineno}: {type_name} assigned to '{var}' " + f"without {var}.max_size / {var}.maxSize", + ) + ) + return found + + +def _resource_violations(path: Path, tree: ast.AST, lines: list[str]) -> list[Violation]: found = [] - for node in ast.walk(ast.parse(source, filename=str(path))): + for node in ast.walk(tree): if not isinstance(node, ast.Call): continue name = _callee_name(node) @@ -78,33 +197,62 @@ def violations_in(path: Path) -> list[str]: else: continue if uncapped and not _opted_out(lines, node): - found.append(f"{path.as_posix()}:{node.lineno}: {name} without max_size") + found.append(Violation(RULE_RESOURCE, f"{path.as_posix()}:{node.lineno}: {name} without max_size")) return found +def _violations(path: Path) -> list[Violation]: + source = path.read_text(encoding="utf-8") + lines = source.splitlines() + tree = ast.parse(source, filename=str(path)) + return [*_resource_violations(path, tree, lines), *_pool_props_violations(path, tree, lines)] + + +def violations_in(path: Path) -> list[str]: + """Return one message per uncapped pool construction in ``path``.""" + return [violation.message for violation in _violations(path)] + + +def _report(header: str, messages: list[str], hint: str) -> None: + """Print ``header``, the bulleted ``messages``, and ``hint`` to stderr; a no-op if empty.""" + if not messages: + return + print(header, file=sys.stderr) + for message in messages: + print(f" - {message}", file=sys.stderr) + print(hint, file=sys.stderr) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "paths", nargs="*", type=Path, - help=f"Files to check. Defaults to every .py under {DEFAULT_TREE.relative_to(ROOT).as_posix()}.", + help="Files to check. Defaults to every .py under " + + ", ".join(tree.relative_to(ROOT).as_posix() for tree in DEFAULT_TREES) + + ".", ) args = parser.parse_args(argv) - paths = args.paths or sorted(DEFAULT_TREE.rglob("*.py")) - violations = sorted(v for path in paths if path.suffix == ".py" for v in violations_in(path)) + paths = args.paths or sorted(path for tree in DEFAULT_TREES for path in tree.rglob("*.py")) + violations = sorted(v for path in paths if path.suffix == ".py" for v in _violations(path)) if not violations: return 0 - print("error: memory pools created by tests must set max_size:", file=sys.stderr) - for violation in violations: - print(f" - {violation}", file=sys.stderr) - print( + _report( + "error: memory pools created by tests must set max_size:", + [v.message for v in violations if v.rule == RULE_RESOURCE], f"Use the suite-wide POOL_SIZE from cuda_core/tests/helpers/constants.py, or annotate a\n" f"deliberate exception with a '# {OPT_OUT_MARKER}: ' comment.\n" f"See cuda_core/tests/AGENTS.md.", - file=sys.stderr, + ) + _report( + "error: pool properties created by tests must set max_size / maxSize:", + [v.message for v in violations if v.rule == RULE_PROPS], + f"Set max_size (v2 bindings) or maxSize (other bindings) on the props object you constructed,\n" + f"using POOL_SIZE from cuda_python_test_helpers.memory, or annotate a deliberate exception\n" + f"with a '# {OPT_OUT_MARKER}: ' comment. See cuda_bindings_13/AGENTS.md.", ) return 1 diff --git a/ci/tools/tests/test_check_mempool_hygiene.py b/ci/tools/tests/test_check_mempool_hygiene.py index 5ff3562059b..54d2b2344b1 100644 --- a/ci/tools/tests/test_check_mempool_hygiene.py +++ b/ci/tools/tests/test_check_mempool_hygiene.py @@ -9,7 +9,7 @@ import pytest sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from check_mempool_hygiene import DEFAULT_TREE, main, violations_in +from check_mempool_hygiene import DEFAULT_TREES, main, violations_in def write(tmp_path, source): @@ -92,5 +92,128 @@ def test_main_ignores_non_python_files(tmp_path): def test_the_live_test_suite_is_clean(): # Without a default the hook would only ever see changed files, so a # violation could ride in on a rename or a merge. - assert DEFAULT_TREE.is_dir() + assert all(tree.is_dir() for tree in DEFAULT_TREES) assert main([]) == 0 + + +POOL_PROPS_CAPPED = [ + pytest.param("props = driver.MemPoolProps_v1()\nprops.max_size = POOL_SIZE\n", id="v2-max_size"), + pytest.param("props = cuda.CUmemPoolProps()\nprops.maxSize = POOL_SIZE\n", id="driver-maxSize"), + pytest.param("props = cudart.cudaMemPoolProps()\nprops.maxSize = POOL_SIZE\n", id="runtime-maxSize"), + pytest.param("def f():\n props = cuda.CUmemPoolProps_v1()\n props.maxSize = POOL_SIZE\n", id="in-function"), + # A field of an alloc-node params object is not a constructor call, so it is not checked. + pytest.param("params.poolProps.allocType = TYPE\nparams.pool_props = other\n", id="graph-params-fields"), + # Type-annotated assignment (ast.AnnAssign) must also be detected and treated as capped. + pytest.param( + "props: cuda.CUmemPoolProps = cuda.CUmemPoolProps()\nprops.maxSize = POOL_SIZE\n", + id="annotated-capped", + ), + # Second construction caps correctly; the first was already capped. + pytest.param( + "props = cuda.CUmemPoolProps()\nprops.maxSize = POOL_SIZE\nprops = cuda.CUmemPoolProps()\nprops.maxSize = POOL_SIZE\n", + id="two-constructions-both-capped", + ), +] + +POOL_PROPS_UNCAPPED = [ + pytest.param("props = driver.MemPoolProps_v1()\nprops.location = loc\n", id="v2"), + pytest.param("props = cuda.CUmemPoolProps()\n", id="driver"), + pytest.param("props = cudart.cudaMemPoolProps()\n", id="runtime"), + pytest.param( + "def f():\n props = cuda.CUmemPoolProps()\n other.maxSize = POOL_SIZE\n", + id="other-object", + ), + # The cap must be set in the function that constructed the props. + pytest.param( + "def make():\n return cuda.CUmemPoolProps()\n\n\ndef f():\n props = cuda.CUmemPoolProps()\n g(props)\n\n\ndef g(props):\n props.maxSize = POOL_SIZE\n", + id="capped-in-another-function", + ), + # Type-annotated assignment without a cap must also be reported (ast.AnnAssign, Finding 1). + pytest.param("props: cuda.CUmemPoolProps = cuda.CUmemPoolProps()\n", id="annotated-uncapped"), + # A cap on the first object does not cover a second construction of the same name (Finding 2). + pytest.param( + "props = cuda.CUmemPoolProps()\nprops.maxSize = POOL_SIZE\nprops = cuda.CUmemPoolProps()\n", + id="reassignment-second-uncapped", + ), +] + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +@pytest.mark.parametrize("source", POOL_PROPS_CAPPED) +def test_capped_pool_props_are_not_reported(tmp_path, source): + assert violations_in(write(tmp_path, source)) == [] + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +@pytest.mark.parametrize("source", POOL_PROPS_UNCAPPED) +def test_uncapped_pool_props_are_reported(tmp_path, source): + assert violations_in(write(tmp_path, source)) + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +def test_pool_props_marker_opts_out(tmp_path): + source = "# uncapped-pool-ok: pool is never created\nprops = cuda.CUmemPoolProps()\n" + + assert violations_in(write(tmp_path, source)) == [] + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +def test_pool_props_message_names_file_line_and_variable(tmp_path): + path = write(tmp_path, "x = 1\nprops = driver.MemPoolProps_v1()\n") + + (violation,) = violations_in(path) + + assert violation.startswith(path.as_posix()) + assert ":2:" in violation + assert "MemPoolProps_v1 assigned to 'props' without props.max_size / props.maxSize" in violation + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +def test_main_reports_pool_props_with_their_own_message(tmp_path, capsys): + path = write(tmp_path, "props = cuda.CUmemPoolProps()\n") + + assert main([str(path)]) == 1 + + err = capsys.readouterr().err + assert "pool properties created by tests must set max_size / maxSize" in err + assert "must set max_size:" not in err + + +@pytest.mark.agent_authored(model="claude-sonnet-5.5") +def test_main_reports_both_rules_separately(tmp_path, capsys): + path = write( + tmp_path, + "DeviceMemoryResource(dev, DeviceMemoryResourceOptions())\nprops = cuda.CUmemPoolProps()\n", + ) + + assert main([str(path)]) == 1 + + err = capsys.readouterr().err + assert "memory pools created by tests must set max_size:" in err + assert "pool properties created by tests must set max_size / maxSize" in err + + +@pytest.mark.agent_authored(model="claude-sonnet-4.6") +def test_annotated_pool_props_assignment_reports_correct_line(tmp_path): + # Finding 1: props: T = CUmemPoolProps() is ast.AnnAssign; it must be detected. + path = write(tmp_path, "x = 1\nprops: cuda.CUmemPoolProps = cuda.CUmemPoolProps()\n") + + (violation,) = violations_in(path) + + assert ":2:" in violation + + +@pytest.mark.agent_authored(model="claude-sonnet-4.6") +def test_reassignment_reports_only_the_uncapped_construction(tmp_path): + # Finding 2: a cap on the first object must not excuse a second construction + # of the same variable. Only the second (uncapped) object should be reported. + source = ( + "props = cuda.CUmemPoolProps()\n" # line 1 + "props.maxSize = POOL_SIZE\n" # line 2 — caps line 1 + "props = cuda.CUmemPoolProps()\n" # line 3 — new object, no cap + ) + path = write(tmp_path, source) + + (violation,) = violations_in(path) + + assert ":3:" in violation diff --git a/cuda_bindings_12/tests/cython/test_interoperability_cython.pyx b/cuda_bindings_12/tests/cython/test_interoperability_cython.pyx index f98660725f2..54dee295f58 100644 --- a/cuda_bindings_12/tests/cython/test_interoperability_cython.pyx +++ b/cuda_bindings_12/tests/cython/test_interoperability_cython.pyx @@ -7,6 +7,7 @@ import cuda.cuda as cuda import cuda.cudart as cudart import numpy as np import pytest +from cuda_python_test_helpers.memory import supports_memory_pools # TODO: update to new module once the old ones are removed, we use the # tests to cover backward compatibility. @@ -14,11 +15,6 @@ cimport cuda.ccuda as ccuda cimport cuda.ccudart as ccudart -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - def test_interop_stream(): err_dr, = cuda.cuInit(0) assert(err_dr == cuda.CUresult.CUDA_SUCCESS) @@ -140,7 +136,7 @@ def test_interop_graphNode(): assert(err_dr == cuda.CUresult.CUDA_SUCCESS) -@pytest.mark.skipif(not supportsMemoryPool(), reason='Requires mempool operations') +@pytest.mark.skipif(not supports_memory_pools(), reason='Requires mempool operations') def test_interop_memPool(): err_dr, = cuda.cuInit(0) assert(err_dr == cuda.CUresult.CUDA_SUCCESS) diff --git a/cuda_bindings_12/tests/test_cuda.py b/cuda_bindings_12/tests/test_cuda.py index 402f4eb9a5a..09a77646b6f 100644 --- a/cuda_bindings_12/tests/test_cuda.py +++ b/cuda_bindings_12/tests/test_cuda.py @@ -10,32 +10,14 @@ import pytest import cuda.cuda as cuda -import cuda.cudart as cudart from cuda.bindings import driver - - -def driverVersionLessThan(target): - (err,) = cuda.cuInit(0) - assert err == cuda.CUresult.CUDA_SUCCESS - err, version = cuda.cuDriverGetVersion() - assert err == cuda.CUresult.CUDA_SUCCESS - return version < target - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - -def supportsManagedMemory(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrManagedMemory, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - -# Cap for memory pools created by tests. A pool created with maxSize 0 reserves -# virtual address space of about twice the device memory, and that reservation -# can fail on systems with a limited virtual address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB +from cuda_python_test_helpers import driver_version_less_than +from cuda_python_test_helpers.memory import ( + POOL_SIZE, + supports_managed_memory, + supports_memory_pools, + xfail_graph_mempool_oom, +) def supportsCudaAPI(name): @@ -325,7 +307,7 @@ def test_cuda_CUstreamBatchMemOpParams(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_cuda_memPool_attr(): (err,) = cuda.cuInit(0) @@ -397,7 +379,7 @@ def test_cuda_memPool_attr(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_cuda_pointer_attr(): (err,) = cuda.cuInit(0) @@ -456,7 +438,7 @@ def test_cuda_pointer_attr(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_pointer_get_attributes_device_ordinal(): attributes = [ @@ -482,7 +464,7 @@ def test_pointer_allowed_handle_types_preserves_64_bits(): assert helper.pyObj() == expected -@pytest.mark.skipif(not supportsManagedMemory(), reason="When new attributes were introduced") +@pytest.mark.skipif(not supports_managed_memory(), reason="When new attributes were introduced") def test_cuda_mem_range_attr(): (err,) = cuda.cuInit(0) assert err == cuda.CUresult.CUDA_SUCCESS @@ -546,27 +528,11 @@ def test_cuda_mem_range_attr(): assert err == cuda.CUresult.CUDA_SUCCESS -@pytest.fixture -def xfail_on_graph_mempool_oom(): - """Return a check that xfails the calling test if the graph memory pool cannot be created. - - The graph memory pool reserves virtual address space of about twice the - device memory, and unlike pools from cuMemPoolCreate it cannot be capped. - On systems with a limited virtual address space this reservation can fail - with CUDA_ERROR_OUT_OF_MEMORY, especially when another uncapped pool - already exists in the process. - """ - - def check(err): - if err == cuda.CUresult.CUDA_ERROR_OUT_OF_MEMORY: - pytest.xfail("graph memory pool creation failed with CUDA_ERROR_OUT_OF_MEMORY") - - return check - - -@pytest.mark.skipif(driverVersionLessThan(11040) or not supportsMemoryPool(), reason="Mempool for graphs not supported") +@pytest.mark.skipif( + driver_version_less_than(11040) or not supports_memory_pools(), reason="Mempool for graphs not supported" +) @pytest.mark.thread_unsafe(reason="used high memory can be higher if threaded.") -def test_cuda_graphMem_attr(xfail_on_graph_mempool_oom): +def test_cuda_graphMem_attr(): (err,) = cuda.cuInit(0) assert err == cuda.CUresult.CUDA_SUCCESS err, device = cuda.cuDeviceGet(0) @@ -594,7 +560,11 @@ def test_cuda_graphMem_attr(xfail_on_graph_mempool_oom): assert destroy_err == cuda.CUresult.CUDA_SUCCESS (destroy_err,) = cuda.cuStreamDestroy(stream) assert destroy_err == cuda.CUresult.CUDA_SUCCESS - xfail_on_graph_mempool_oom(err) + # xfail raises, so the context must be destroyed first or it leaks + # into later tests as the current context. + (destroy_err,) = cuda.cuCtxDestroy(ctx) + assert destroy_err == cuda.CUresult.CUDA_SUCCESS + xfail_graph_mempool_oom() assert err == cuda.CUresult.CUDA_SUCCESS err, freeNode = cuda.cuGraphAddMemFreeNode(graph, [allocNode], 1, params.dptr) assert err == cuda.CUresult.CUDA_SUCCESS @@ -632,7 +602,7 @@ def test_cuda_graphMem_attr(xfail_on_graph_mempool_oom): @pytest.mark.skipif( - driverVersionLessThan(12010) + driver_version_less_than(12010) or not supportsCudaAPI("cuCoredumpSetAttributeGlobal") or not supportsCudaAPI("cuCoredumpGetAttributeGlobal"), reason="Coredump API not present", @@ -808,7 +778,7 @@ def test_device_get_name(): # TODO: cuStreamGetCaptureInfo_v2 -@pytest.mark.skipif(driverVersionLessThan(11030), reason="Driver too old for cuStreamGetCaptureInfo_v2") +@pytest.mark.skipif(driver_version_less_than(11030), reason="Driver too old for cuStreamGetCaptureInfo_v2") def test_stream_capture(): pass @@ -941,7 +911,7 @@ def test_invalid_repr_attribute(): @pytest.mark.skipif( - driverVersionLessThan(12020) + driver_version_less_than(12020) or not supportsCudaAPI("cuGraphAddNode") or not supportsCudaAPI("cuGraphNodeSetParams") or not supportsCudaAPI("cuGraphExecNodeSetParams"), @@ -1061,7 +1031,7 @@ def test_graph_poly(): @pytest.mark.skipif( - driverVersionLessThan(12040) or not supportsCudaAPI("cuDeviceGetDevResource"), + driver_version_less_than(12040) or not supportsCudaAPI("cuDeviceGetDevResource"), reason="Polymorphic graph APIs required", ) def test_cuDeviceGetDevResource(): @@ -1090,7 +1060,7 @@ def test_cuDeviceGetDevResource(): @pytest.mark.skipif( - driverVersionLessThan(12030) or not supportsCudaAPI("cuGraphConditionalHandleCreate"), + driver_version_less_than(12030) or not supportsCudaAPI("cuGraphConditionalHandleCreate"), reason="Conditional graph APIs required", ) def test_conditional(): @@ -1159,14 +1129,14 @@ def test_all_CUresult_codes(): assert num_good >= 76 # CTK 11.0.3_450.51.06 -@pytest.mark.skipif(driverVersionLessThan(12030), reason="Driver too old for cuKernelGetName") +@pytest.mark.skipif(driver_version_less_than(12030), reason="Driver too old for cuKernelGetName") def test_cuKernelGetName_failure(): err, name = cuda.cuKernelGetName(0) assert err == cuda.CUresult.CUDA_ERROR_INVALID_VALUE assert name is None -@pytest.mark.skipif(driverVersionLessThan(12030), reason="Driver too old for cuFuncGetName") +@pytest.mark.skipif(driver_version_less_than(12030), reason="Driver too old for cuFuncGetName") def test_cuFuncGetName_failure(): err, name = cuda.cuFuncGetName(0) assert err == cuda.CUresult.CUDA_ERROR_INVALID_VALUE @@ -1174,7 +1144,7 @@ def test_cuFuncGetName_failure(): @pytest.mark.skipif( - driverVersionLessThan(12080) or not supportsCudaAPI("cuCheckpointProcessGetState"), + driver_version_less_than(12080) or not supportsCudaAPI("cuCheckpointProcessGetState"), reason="When API was introduced", ) def test_cuCheckpointProcessGetState_failure(): @@ -1216,7 +1186,7 @@ def test_struct_pointer_comparison(target): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphGetId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphGetId"), reason="Requires CUDA 13.1+", ) def test_cuGraphGetId(device, ctx): @@ -1243,7 +1213,7 @@ def test_cuGraphGetId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphExecGetId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphExecGetId"), reason="Requires CUDA 13.1+", ) def test_cuGraphExecGetId(device, ctx): @@ -1290,7 +1260,7 @@ def test_cuGraphExecGetId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphNodeGetLocalId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphNodeGetLocalId"), reason="Requires CUDA 13.1+", ) def test_cuGraphNodeGetLocalId(device, ctx): @@ -1332,7 +1302,7 @@ def test_cuGraphNodeGetLocalId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphNodeGetToolsId"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphNodeGetToolsId"), reason="Requires CUDA 13.1+", ) def test_cuGraphNodeGetToolsId(device, ctx): @@ -1361,7 +1331,7 @@ def test_cuGraphNodeGetToolsId(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuGraphNodeGetContainingGraph"), + driver_version_less_than(13010) or not supportsCudaAPI("cuGraphNodeGetContainingGraph"), reason="Requires CUDA 13.1+", ) def test_cuGraphNodeGetContainingGraph(device, ctx): @@ -1408,7 +1378,7 @@ def test_cuGraphNodeGetContainingGraph(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuStreamGetDevResource"), + driver_version_less_than(13010) or not supportsCudaAPI("cuStreamGetDevResource"), reason="Requires CUDA 13.1+", ) def test_cuStreamGetDevResource(device, ctx): @@ -1427,7 +1397,7 @@ def test_cuStreamGetDevResource(device, ctx): @pytest.mark.skipif( - driverVersionLessThan(13010) or not supportsCudaAPI("cuDevSmResourceSplit"), + driver_version_less_than(13010) or not supportsCudaAPI("cuDevSmResourceSplit"), reason="Requires CUDA 13.1+", ) def test_cuDevSmResourceSplit(device, ctx): diff --git a/cuda_bindings_12/tests/test_cudart.py b/cuda_bindings_12/tests/test_cudart.py index 5c130fbe4bd..a4c75fe8c46 100644 --- a/cuda_bindings_12/tests/test_cudart.py +++ b/cuda_bindings_12/tests/test_cudart.py @@ -10,6 +10,7 @@ import cuda.cuda as cuda import cuda.cudart as cudart from cuda.bindings import runtime +from cuda_python_test_helpers.memory import POOL_SIZE, supports_memory_pools def isSuccess(err): @@ -26,17 +27,6 @@ def driverVersionLessThan(target): return version < target -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return isSuccess(err) and isSupported - - -# Cap for memory pools created by tests. A pool created with maxSize 0 reserves -# virtual address space of about twice the device memory, and that reservation -# can fail on systems with a limited virtual address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - - def supportsSparseTexturesDeviceFilter(): err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrSparseCudaArraySupported, 0) return isSuccess(err) and isSupported @@ -424,7 +414,7 @@ def test_cudart_cudaGetDeviceProperties(): @pytest.mark.skipif( - driverVersionLessThan(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driverVersionLessThan(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_cudart_MemPool_attr(): poolProps = cudart.cudaMemPoolProps() diff --git a/cuda_bindings_12/tests/test_interoperability.py b/cuda_bindings_12/tests/test_interoperability.py index 5262524d9a2..de6e536bedc 100644 --- a/cuda_bindings_12/tests/test_interoperability.py +++ b/cuda_bindings_12/tests/test_interoperability.py @@ -6,11 +6,7 @@ import cuda.cuda as cuda import cuda.cudart as cudart - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported +from cuda_python_test_helpers.memory import supports_memory_pools def test_interop_stream(): @@ -144,7 +140,7 @@ def test_interop_function(): assert err_dr == cuda.CUresult.CUDA_SUCCESS -@pytest.mark.skipif(not supportsMemoryPool(), reason="Requires mempool operations") +@pytest.mark.skipif(not supports_memory_pools(), reason="Requires mempool operations") def test_interop_memPool(): (err_dr,) = cuda.cuInit(0) assert err_dr == cuda.CUresult.CUDA_SUCCESS diff --git a/cuda_bindings_13/AGENTS.md b/cuda_bindings_13/AGENTS.md index a5c92098ced..6b3a3d442b7 100644 --- a/cuda_bindings_13/AGENTS.md +++ b/cuda_bindings_13/AGENTS.md @@ -54,6 +54,16 @@ and `nvrtc`. These test files should not be added to, only updated when necessary to fix test failures. The canonical set of tests are those outside of the `tests/legacy_api` subdirectory. +Tests that create memory pools must cap them: set `max_size` (v2 types) or +`maxSize` (other types) on the pool properties object to `POOL_SIZE` from +`cuda_python_test_helpers.memory`. A pool created with a size of 0 reserves +virtual address space of about twice the device memory, which fails on systems +with a limited virtual address space. `ci/tools/check_mempool_hygiene.py` +(a pre-commit hook) enforces this for `.py` tests. It cannot parse Cython, so +cap pools in `tests/cython/*.pyx` by hand. The same module holds the shared +`supports_memory_pools`, `supports_managed_memory`, and +`xfail_graph_mempool_oom` helpers; reuse them rather than defining copies. + ## Build and environment notes - `CUDA_HOME` or `CUDA_PATH` must point to a valid CUDA Toolkit for source diff --git a/cuda_bindings_13/tests/cython/test_interoperability_cython.pyx b/cuda_bindings_13/tests/cython/test_interoperability_cython.pyx index 62ef4abb312..944c0c21ae8 100644 --- a/cuda_bindings_13/tests/cython/test_interoperability_cython.pyx +++ b/cuda_bindings_13/tests/cython/test_interoperability_cython.pyx @@ -8,22 +8,11 @@ import cuda.bindings.driver as cuda import cuda.bindings.runtime as cudart import numpy as np import pytest +from cuda_python_test_helpers.memory import POOL_SIZE, supports_memory_pools cimport cuda.bindings.cydriver as ccuda cimport cuda.bindings.cyruntime as ccudart -# Cap for the memory pool created by test_interop_memPool. Touching the device's -# default pool (e.g. via cuDeviceGetDefaultMemPool, cudaDeviceGetDefaultMemPool, -# or cuDeviceGetMemPool / cudaDeviceGetMemPool before a pool has been set) -# reserves virtual address space of about twice the device memory, which -# cannot be satisfied in a 39-bit address space (e.g. riscv64 Sv39). -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - def test_interop_stream(): err_dr, = cuda.cuInit(0) @@ -147,7 +136,7 @@ def test_interop_graphNode(): @pytest.mark.thread_unsafe(reason="changes the device's current memory pool") -@pytest.mark.skipif(not supportsMemoryPool(), reason='Requires mempool operations') +@pytest.mark.skipif(not supports_memory_pools(), reason='Requires mempool operations') def test_interop_memPool(): err_dr, = cuda.cuInit(0) assert(err_dr == cuda.CUresult.CUDA_SUCCESS) diff --git a/cuda_bindings_13/tests/legacy_api/test_legacy_cuda.py b/cuda_bindings_13/tests/legacy_api/test_legacy_cuda.py index 8282f21673b..47043cf74f6 100644 --- a/cuda_bindings_13/tests/legacy_api/test_legacy_cuda.py +++ b/cuda_bindings_13/tests/legacy_api/test_legacy_cuda.py @@ -9,29 +9,19 @@ import numpy as np import pytest +from cuda_python_test_helpers.memory import ( + POOL_SIZE, + supports_managed_memory, + supports_memory_pools, + xfail_graph_mempool_oom, +) from cuda_python_test_helpers.subprocess_runner import run_python_snippet import cuda.bindings.driver as cuda -import cuda.bindings.runtime as cudart from cuda.bindings import driver from cuda.bindings._internal.utils import FunctionNotFoundError from cuda_python_test_helpers import driver_version_less_than -# Cap for memory pools created by tests. A pool created with maxSize 0 reserves -# virtual address space of about twice the device memory, and that reservation -# can fail on systems with a limited virtual address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - -def supportsManagedMemory(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrManagedMemory, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - def supportsCudaAPI(name): return name in dir(cuda) @@ -263,7 +253,7 @@ def test_cuda_CUstreamBatchMemOpParams(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_cuda_memPool_attr(): poolProps = cuda.CUmemPoolProps() @@ -326,7 +316,7 @@ def test_cuda_memPool_attr(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_cuda_pointer_attr(): err, ptr = cuda.cuMemAllocManaged(0x1000, cuda.CUmemAttach_flags.CU_MEM_ATTACH_GLOBAL.value) @@ -377,7 +367,7 @@ def test_cuda_pointer_attr(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_pointer_get_attributes_device_ordinal(): attributes = [ @@ -403,7 +393,7 @@ def test_pointer_allowed_handle_types_preserves_64_bits(): assert helper.pyObj() == expected -@pytest.mark.skipif(not supportsManagedMemory(), reason="When new attributes were introduced") +@pytest.mark.skipif(not supports_managed_memory(), reason="When new attributes were introduced") def test_cuda_mem_range_attr(device): size = 0x1000 location_device = cuda.CUmemLocation() @@ -466,29 +456,11 @@ def test_cuda_mem_range_attr(device): assert err == cuda.CUresult.CUDA_SUCCESS -@pytest.fixture -def xfail_on_graph_mempool_oom(): - """Return a check that xfails the calling test if the graph memory pool cannot be created. - - The graph memory pool reserves virtual address space of about twice the - device memory, and unlike pools from cuMemPoolCreate it cannot be capped. - On systems with a limited virtual address space this reservation can fail - with CUDA_ERROR_OUT_OF_MEMORY, especially when another uncapped pool - already exists in the process. - """ - - def check(err): - if err == cuda.CUresult.CUDA_ERROR_OUT_OF_MEMORY: - pytest.xfail("graph memory pool creation failed with CUDA_ERROR_OUT_OF_MEMORY") - - return check - - @pytest.mark.skipif( - driver_version_less_than(11040) or not supportsMemoryPool(), reason="Mempool for graphs not supported" + driver_version_less_than(11040) or not supports_memory_pools(), reason="Mempool for graphs not supported" ) @pytest.mark.thread_unsafe(reason="used high memory can be higher if threaded.") -def test_cuda_graphMem_attr(device, xfail_on_graph_mempool_oom): +def test_cuda_graphMem_attr(device): err, stream = cuda.cuStreamCreate(0) assert err == cuda.CUresult.CUDA_SUCCESS @@ -509,7 +481,7 @@ def test_cuda_graphMem_attr(device, xfail_on_graph_mempool_oom): assert destroy_err == cuda.CUresult.CUDA_SUCCESS (destroy_err,) = cuda.cuStreamDestroy(stream) assert destroy_err == cuda.CUresult.CUDA_SUCCESS - xfail_on_graph_mempool_oom(err) + xfail_graph_mempool_oom() assert err == cuda.CUresult.CUDA_SUCCESS err, freeNode = cuda.cuGraphAddMemFreeNode(graph, [allocNode], 1, params.dptr) assert err == cuda.CUresult.CUDA_SUCCESS diff --git a/cuda_bindings_13/tests/legacy_api/test_legacy_interoperability.py b/cuda_bindings_13/tests/legacy_api/test_legacy_interoperability.py index b8774af13fe..b268a3db842 100644 --- a/cuda_bindings_13/tests/legacy_api/test_legacy_interoperability.py +++ b/cuda_bindings_13/tests/legacy_api/test_legacy_interoperability.py @@ -3,22 +3,11 @@ import numpy as np import pytest +from cuda_python_test_helpers.memory import POOL_SIZE, supports_memory_pools import cuda.bindings.driver as cuda import cuda.bindings.runtime as cudart -# Cap for the memory pool created by test_interop_memPool. Touching the device's -# default pool (e.g. via cuDeviceGetDefaultMemPool, cudaDeviceGetDefaultMemPool, -# or cuDeviceGetMemPool / cudaDeviceGetMemPool before a pool has been set) -# reserves virtual address space of about twice the device memory, which -# cannot be satisfied in a 39-bit address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - def test_interop_stream(): # DRV to RT @@ -92,7 +81,7 @@ def test_interop_graphNode(): @pytest.mark.thread_unsafe(reason="changes the device's current memory pool") @pytest.mark.agent_authored(model="claude-sonnet-5.5") -@pytest.mark.skipif(not supportsMemoryPool(), reason="Requires mempool operations") +@pytest.mark.skipif(not supports_memory_pools(), reason="Requires mempool operations") def test_interop_memPool(): # Use a small, capped pool instead of the device's default pool. The pool # must be set before it is queried below because the getters create (and diff --git a/cuda_bindings_13/tests/test_cuda.py b/cuda_bindings_13/tests/test_cuda.py index 93a86a03986..c8fa0262a04 100644 --- a/cuda_bindings_13/tests/test_cuda.py +++ b/cuda_bindings_13/tests/test_cuda.py @@ -10,8 +10,13 @@ import numpy as np import pytest +from cuda_python_test_helpers.memory import ( + POOL_SIZE, + supports_managed_memory, + supports_memory_pools, + xfail_graph_mempool_oom, +) -import cuda.bindings.runtime as cudart from cuda.bindings._internal.utils import FunctionNotFoundError from cuda.bindings._v2 import driver from cuda_python_test_helpers import driver_version_less_than @@ -19,21 +24,6 @@ # NULL dependency array: the node has no dependencies. NO_DEPS = 0 -# Cap for memory pools created by tests. A pool created with maxSize 0 reserves -# virtual address space of about twice the device memory, and that reservation -# can fail on systems with a limited virtual address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - - -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - -def supportsManagedMemory(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrManagedMemory, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - def supportsCudaAPI(name): return name in dir(driver) @@ -279,7 +269,7 @@ def test_cuda_CUstreamBatchMemOpParams(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_cuda_memPool_attr(): poolProps = driver.MemPoolProps_v1() @@ -339,7 +329,7 @@ def get_attr(attr): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_cuda_pointer_attr(): ptr = driver.mem_alloc_managed(0x1000, driver.MemAttachFlags.GLOBAL.value) @@ -381,7 +371,7 @@ def test_cuda_pointer_attr(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_pointer_get_attributes_device_ordinal(): (value,) = driver.pointer_get_attributes([driver.PointerAttribute.DEVICE_ORDINAL], 0) @@ -393,7 +383,7 @@ def test_pointer_get_attributes_device_ordinal(): @pytest.mark.agent_authored(model="gpt-5.6-sol") @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsManagedMemory(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_managed_memory(), reason="When new attributes were introduced" ) def test_pointer_allowed_handle_types_preserves_64_bits(): ptr = driver.mem_alloc_managed(0x1000, driver.MemAttachFlags.GLOBAL.value) @@ -403,7 +393,7 @@ def test_pointer_allowed_handle_types_preserves_64_bits(): driver.mem_free_v2(ptr) -@pytest.mark.skipif(not supportsManagedMemory(), reason="When new attributes were introduced") +@pytest.mark.skipif(not supports_managed_memory(), reason="When new attributes were introduced") def test_cuda_mem_range_attr(device): size = 0x1000 location_device = driver.MemLocation_v1() @@ -452,29 +442,11 @@ def test_cuda_mem_range_attr(device): driver.mem_free_v2(ptr) -@pytest.fixture -def xfail_on_graph_mempool_oom(): - """Return a check that xfails the calling test if the graph memory pool cannot be created. - - The graph memory pool reserves virtual address space of about twice the - device memory, and unlike pools from cuMemPoolCreate it cannot be capped. - On systems with a limited virtual address space this reservation can fail - with CUDA_ERROR_OUT_OF_MEMORY, especially when another uncapped pool - already exists in the process. - """ - - def check(exc_type): - if exc_type is driver.OutOfMemoryError: - pytest.xfail("graph memory pool creation failed with CUDA_ERROR_OUT_OF_MEMORY") - - return check - - @pytest.mark.skipif( - driver_version_less_than(11040) or not supportsMemoryPool(), reason="Mempool for graphs not supported" + driver_version_less_than(11040) or not supports_memory_pools(), reason="Mempool for graphs not supported" ) @pytest.mark.thread_unsafe(reason="used high memory can be higher if threaded.") -def test_cuda_graphMem_attr(device, xfail_on_graph_mempool_oom): +def test_cuda_graphMem_attr(device): stream = driver.stream_create(0) graph = driver.graph_create(0) @@ -492,8 +464,7 @@ def test_cuda_graphMem_attr(device, xfail_on_graph_mempool_oom): except driver.OutOfMemoryError: driver.graph_destroy(graph) driver.stream_destroy_v2(stream) - xfail_on_graph_mempool_oom(driver.OutOfMemoryError) - raise + xfail_graph_mempool_oom() deps = node_array([allocNode]) freeNode = driver.graph_add_mem_free_node(graph, ctypes.addressof(deps), 1, params.dptr) @@ -594,7 +565,7 @@ def test_ffi_coverage_mapping_base_addr(): @pytest.mark.agent_authored(model="claude-sonnet-4-6") @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_ffi_coverage_new_mempool_attributes(): """Smoke-test all newly added MemPoolAttribute queries (driver errors for @@ -622,7 +593,7 @@ def test_ffi_coverage_new_mempool_attributes(): @pytest.mark.agent_authored(model="claude-sonnet-4-6") -@pytest.mark.skipif(not supportsManagedMemory(), reason="When new attributes were introduced") +@pytest.mark.skipif(not supports_managed_memory(), reason="When new attributes were introduced") def test_ffi_coverage_new_mem_range_attributes(): """Smoke-test all newly added MemRangeAttribute queries (scalar int).""" size = 0x1000 diff --git a/cuda_bindings_13/tests/test_cudart.py b/cuda_bindings_13/tests/test_cudart.py index f1c263f7c13..98aaa9dd501 100644 --- a/cuda_bindings_13/tests/test_cudart.py +++ b/cuda_bindings_13/tests/test_cudart.py @@ -6,6 +6,7 @@ import numpy as np import pytest +from cuda_python_test_helpers.memory import POOL_SIZE, supports_memory_pools import cuda.bindings.driver as cuda import cuda.bindings.runtime as cudart @@ -13,11 +14,6 @@ from cuda.bindings import runtime from cuda_python_test_helpers import driver_version_less_than -# Cap for memory pools created by tests. A pool created with maxSize 0 reserves -# virtual address space of about twice the device memory, and that reservation -# can fail on systems with a limited virtual address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - def isSuccess(err): return err == cudart.cudaError_t.cudaSuccess @@ -27,11 +23,6 @@ def assertSuccess(err): assert isSuccess(err) -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return isSuccess(err) and isSupported - - def supportsSparseTexturesDeviceFilter(): err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrSparseCudaArraySupported, 0) return isSuccess(err) and isSupported @@ -518,7 +509,7 @@ def test_cudart_cudaGetDeviceProperties(): @pytest.mark.skipif( - driver_version_less_than(11030) or not supportsMemoryPool(), reason="When new attributes were introduced" + driver_version_less_than(11030) or not supports_memory_pools(), reason="When new attributes were introduced" ) def test_cudart_MemPool_attr(): poolProps = cudart.cudaMemPoolProps() diff --git a/cuda_bindings_13/tests/test_interoperability.py b/cuda_bindings_13/tests/test_interoperability.py index 20503e8310e..e6fbc8e7d10 100644 --- a/cuda_bindings_13/tests/test_interoperability.py +++ b/cuda_bindings_13/tests/test_interoperability.py @@ -3,26 +3,15 @@ import numpy as np import pytest +from cuda_python_test_helpers.memory import POOL_SIZE, supports_memory_pools import cuda.bindings.runtime as cudart from cuda.bindings._v2 import driver -# Cap for the memory pool created by test_interop_memPool. Touching the device's -# default pool (e.g. via cuDeviceGetDefaultMemPool, cudaDeviceGetDefaultMemPool, -# or cuDeviceGetMemPool / cudaDeviceGetMemPool before a pool has been set) -# reserves virtual address space of about twice the device memory, which -# cannot be satisfied in a 39-bit address space. -POOL_SIZE = 2 * 1024 * 1024 # 2 MiB - # NULL dependency array: the node has no dependencies. NO_DEPS = 0 -def supportsMemoryPool(): - err, isSupported = cudart.cudaDeviceGetAttribute(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, 0) - return err == cudart.cudaError_t.cudaSuccess and isSupported - - def test_interop_stream(): # DRV to RT stream = driver.stream_create(0) @@ -86,7 +75,7 @@ def test_interop_graphNode(): @pytest.mark.thread_unsafe(reason="changes the device's current memory pool") @pytest.mark.agent_authored(model="claude-sonnet-5.5") -@pytest.mark.skipif(not supportsMemoryPool(), reason="Requires mempool operations") +@pytest.mark.skipif(not supports_memory_pools(), reason="Requires mempool operations") def test_interop_memPool(): # Use a small, capped pool instead of the device's default pool. The pool # must be set before it is queried below because the getters create (and diff --git a/cuda_core/tests/AGENTS.md b/cuda_core/tests/AGENTS.md index 8bf0a6c7162..732ad1baad5 100644 --- a/cuda_core/tests/AGENTS.md +++ b/cuda_core/tests/AGENTS.md @@ -22,8 +22,10 @@ from helpers.constants import POOL_SIZE mr = DeviceMemoryResource(dev, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) ``` -Use a larger value only if a test genuinely requires it, and prefer adding a -shared constant to `helpers/constants.py` over redefining one per module. +`helpers/constants.py` re-exports `POOL_SIZE` from +`cuda_python_test_helpers.memory`, where the `cuda.bindings` tests import it +too. Use a larger value only if a test genuinely requires it, and prefer adding +a shared constant over redefining one per module. ### Passing no options is different from passing empty options @@ -105,6 +107,10 @@ Follow these rules when adding or moving shared test code: when the test needs to communicate with a running child or inspect its lifetime; use `kill_subprocesses` before assertions and `track_child_processes` for fixture cleanup so a stuck child cannot block IPC resource teardown. +- Use the skip guards in `helpers/memory.py` instead of per-module copies: + `skip_if_mempool_unsupported`, `skip_if_managed_mempool_unsupported` (a + property check only; it never constructs a managed resource), and + `is_pool_resource` to tell pool-backed resources from VMM ones. - Search `tests/helpers/` and `cuda_python_test_helpers/` for prior art before adding a new helper; consolidate duplicates across diff --git a/cuda_core/tests/graph/test_graph_definition.py b/cuda_core/tests/graph/test_graph_definition.py index 16d44fb6034..041790a25eb 100644 --- a/cuda_core/tests/graph/test_graph_definition.py +++ b/cuda_core/tests/graph/test_graph_definition.py @@ -12,6 +12,7 @@ import pytest from helpers.graph_kernels import compile_common_kernels +from helpers.memory import skip_if_managed_mempool_unsupported, skip_if_mempool_unsupported from helpers.misc import try_create_condition from cuda.core import Device, LaunchConfig @@ -41,17 +42,6 @@ ALLOC_SIZE = 1024 -def _skip_if_no_mempool(): - if not Device(0).properties.memory_pools_supported: - pytest.skip("Device does not support mempool operations") - - -def _skip_if_no_managed_mempool(): - _skip_if_no_mempool() - if not Device(0).properties.concurrent_managed_access: - pytest.skip("Device does not support managed memory pool operations") - - def _has_node_get_params(): from cuda.core._utils.version import BUILD_CUDA_MAJOR, driver_version @@ -204,13 +194,13 @@ def _build_disconnected(): @pytest.fixture(params=_ALL_BUILDERS) def graph_spec(request, init_cuda): if request.param is not _build_empty: - _skip_if_no_mempool() + skip_if_mempool_unsupported() return request.param() @pytest.fixture(params=_NONEMPTY_BUILDERS) def nonempty_graph_spec(request, init_cuda): - _skip_if_no_mempool() + skip_if_mempool_unsupported() return request.param() @@ -276,7 +266,7 @@ def _build_alloc_node(g): def _build_alloc_managed_node(g): - _skip_if_no_managed_mempool() + skip_if_managed_mempool_unsupported() device_id = Device().device_id entry = g.allocate(ALLOC_SIZE) node = entry.allocate(ALLOC_SIZE, memory_type=GraphMemoryType.MANAGED) @@ -564,7 +554,7 @@ def _build_switch_node(g): def node_spec(request, init_cuda): spec = request.param if spec.needs_mempool: - _skip_if_no_mempool() + skip_if_mempool_unsupported() g = GraphDefinition() node, expected_attrs = spec.builder(g) return spec, g, node, expected_attrs @@ -839,7 +829,7 @@ def test_graphdef_entry_is_virtual(init_cuda): def test_alloc_zero_size_fails(init_cuda): """Alloc with zero size raises error (CUDA limitation).""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() from cuda.core._utils.cuda_utils import CUDAError with pytest.raises(CUDAError): @@ -849,7 +839,7 @@ def test_alloc_zero_size_fails(init_cuda): def test_free_creates_dependency(init_cuda): """Free node depends on its predecessor.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) free = alloc.deallocate(alloc.dptr) assert alloc in free.pred @@ -858,7 +848,7 @@ def test_free_creates_dependency(init_cuda): def test_alloc_free_chain(init_cuda): """Alloc and free can be chained.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() a1 = sample_graphdef.allocate(ALLOC_SIZE) a2 = a1.allocate(ALLOC_SIZE) f2 = a2.deallocate(a2.dptr) @@ -890,7 +880,7 @@ def test_alloc_memory_type_invalid(init_cuda): def test_alloc_device_option(init_cuda, device_spec): """Device can be specified as int or Device object.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() device = Device() node = sample_graphdef.allocate(ALLOC_SIZE, device=device_spec(device)) assert node.dptr != 0 @@ -907,7 +897,7 @@ def test_alloc_peer_access(mempool_device_x2): @pytest.mark.agent_authored(model="gpt-5.6-sol") def test_alloc_memory_type_host(init_cuda): """HOST graph alloc nodes reconstruct memory_type from the driver, not the Python argument.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() from cuda.core._utils.cuda_utils import CUDAError g = GraphDefinition() @@ -940,7 +930,7 @@ def test_alloc_memory_type_host(init_cuda): def test_join_merges_branches(init_cuda, num_branches): """join() with multiple branches creates correct dependencies.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() branches = [sample_graphdef.allocate(ALLOC_SIZE) for _ in range(num_branches)] joined = sample_graphdef.join(*branches) assert isinstance(joined, EmptyNode) @@ -1037,7 +1027,7 @@ def test_instantiate_empty_graph(init_cuda, inst_kwargs): def test_instantiate_with_nodes(init_cuda, inst_kwargs): """Graph with nodes can be instantiated.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() sample_graphdef.allocate(ALLOC_SIZE) sample_graphdef.allocate(ALLOC_SIZE) graph = _instantiate(sample_graphdef, inst_kwargs) @@ -1081,7 +1071,7 @@ def test_instantiate_and_execute_kernel(init_cuda, inst_kwargs): def test_instantiate_and_execute_alloc_free(init_cuda, inst_kwargs): """Graph with alloc/free can be executed.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) alloc.deallocate(alloc.dptr) @@ -1095,7 +1085,7 @@ def test_instantiate_and_execute_alloc_free(init_cuda, inst_kwargs): def test_instantiate_and_execute_memset(init_cuda, inst_kwargs): """Graph with alloc/memset/free can be executed.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) ms = alloc.memset(alloc.dptr, 0xAB, ALLOC_SIZE) ms.deallocate(alloc.dptr) @@ -1110,7 +1100,7 @@ def test_instantiate_and_execute_memset(init_cuda, inst_kwargs): def test_instantiate_and_execute_memcpy(init_cuda, inst_kwargs): """Graph with alloc/memset/memcpy/free can be executed and data is copied.""" sample_graphdef = GraphDefinition() - _skip_if_no_mempool() + skip_if_mempool_unsupported() import ctypes src_alloc = sample_graphdef.allocate(ALLOC_SIZE) @@ -1336,7 +1326,7 @@ def test_instantiate_and_execute_if_then(init_cuda): """If-conditional node: body executes only when condition is non-zero.""" sample_graphdef = GraphDefinition() _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() import ctypes from helpers.graph_kernels import compile_conditional_kernels @@ -1369,7 +1359,7 @@ def test_instantiate_and_execute_if_else(init_cuda): """If-else node: then or else branch executes based on condition.""" sample_graphdef = GraphDefinition() _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() import ctypes from helpers.graph_kernels import compile_conditional_kernels @@ -1404,7 +1394,7 @@ def test_instantiate_and_execute_switch(init_cuda): """Switch node: selected branch executes based on condition value.""" sample_graphdef = GraphDefinition() _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() import ctypes from helpers.graph_kernels import compile_conditional_kernels @@ -1456,7 +1446,7 @@ def test_debug_dot_print_creates_file(init_cuda, tmp_path): """debug_dot_print writes a DOT file.""" sample_graphdef = GraphDefinition() dot_file = tmp_path / "graph.dot" - _skip_if_no_mempool() + skip_if_mempool_unsupported() sample_graphdef.allocate(ALLOC_SIZE) sample_graphdef.debug_dot_print(str(dot_file)) assert dot_file.exists() @@ -1468,7 +1458,7 @@ def test_debug_dot_print_with_options(init_cuda, tmp_path): """debug_dot_print accepts GraphDebugPrintOptions.""" sample_graphdef = GraphDefinition() dot_file = tmp_path / "graph.dot" - _skip_if_no_mempool() + skip_if_mempool_unsupported() sample_graphdef.allocate(ALLOC_SIZE) options = GraphDebugPrintOptions(verbose=True, handles=True) sample_graphdef.debug_dot_print(str(dot_file), options) @@ -1479,7 +1469,7 @@ def test_debug_dot_print_invalid_options(init_cuda, tmp_path): """debug_dot_print rejects invalid options type.""" sample_graphdef = GraphDefinition() dot_file = tmp_path / "graph.dot" - _skip_if_no_mempool() + skip_if_mempool_unsupported() sample_graphdef.allocate(ALLOC_SIZE) with pytest.raises(TypeError, match="options must be a GraphDebugPrintOptions"): sample_graphdef.debug_dot_print(str(dot_file), "invalid") diff --git a/cuda_core/tests/graph/test_graph_definition_errors.py b/cuda_core/tests/graph/test_graph_definition_errors.py index 5069dacf4b8..46ad533741d 100644 --- a/cuda_core/tests/graph/test_graph_definition_errors.py +++ b/cuda_core/tests/graph/test_graph_definition_errors.py @@ -7,6 +7,7 @@ import pytest from helpers.graph_kernels import compile_common_kernels +from helpers.memory import skip_if_mempool_unsupported from helpers.misc import try_create_condition from cuda.core import Device, LaunchConfig @@ -20,11 +21,6 @@ SIZEOF_INT = ctypes.sizeof(ctypes.c_int) -def _skip_if_no_mempool(): - if not Device(0).properties.memory_pools_supported: - pytest.skip("Device does not support mempool operations") - - # ============================================================================= # Type validation — wrong types for conditional node methods # ============================================================================= @@ -85,7 +81,7 @@ def test_free_null_pointer(init_cuda): def test_memset_invalid_value_size(init_cuda): """memset with 3-byte value (not 1, 2, or 4) raises ValueError.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() g = GraphDefinition() alloc = g.allocate(1024) with pytest.raises(ValueError): @@ -129,7 +125,7 @@ def test_empty_node(init_cuda): def test_join_single_predecessor(init_cuda): """node.join() with no extra args creates a single-dep empty node.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() g = GraphDefinition() a = g.allocate(1024) joined = a.join() @@ -152,7 +148,7 @@ def test_multiple_instantiation(init_cuda): def test_unmatched_alloc_succeeds(init_cuda): """Alloc without corresponding free is valid (graph-scoped lifetime).""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() g = GraphDefinition() g.allocate(1024) graph = g.instantiate() @@ -184,7 +180,7 @@ def _skip_unless_cc_90(): def test_while_loop_zero_iterations(init_cuda): """While loop with default_value=0 never executes its body.""" _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() mod = compile_common_kernels() add_one = mod.get_kernel("add_one") @@ -212,7 +208,7 @@ def test_while_loop_zero_iterations(init_cuda): def test_if_then_false_skips_body(init_cuda): """If conditional with default_value=0 does not execute its body.""" _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() mod = compile_common_kernels() add_one = mod.get_kernel("add_one") @@ -240,7 +236,7 @@ def test_if_then_false_skips_body(init_cuda): def test_switch_oob_skips_all_branches(init_cuda): """Switch with out-of-range condition value does not execute any branch.""" _skip_unless_cc_90() - _skip_if_no_mempool() + skip_if_mempool_unsupported() mod = compile_common_kernels() add_one = mod.get_kernel("add_one") diff --git a/cuda_core/tests/graph/test_graph_definition_integration.py b/cuda_core/tests/graph/test_graph_definition_integration.py index a352ffba2de..5a74b739679 100644 --- a/cuda_core/tests/graph/test_graph_definition_integration.py +++ b/cuda_core/tests/graph/test_graph_definition_integration.py @@ -8,6 +8,7 @@ import numpy as np import pytest from helpers.graph_kernels import skip_if_nvrtc_lacks_conditional_handle +from helpers.memory import skip_if_mempool_unsupported from cuda.core import Device, EventOptions, LaunchConfig, Program, ProgramOptions from cuda.core._utils.cuda_utils import CUDAError, driver, handle_return @@ -17,11 +18,6 @@ SIZEOF_INT = 4 -def _skip_if_no_mempool(): - if not Device(0).properties.memory_pools_supported: - pytest.skip("Device does not support mempool operations") - - # =================================================================== # Kernel sources # =================================================================== @@ -200,7 +196,7 @@ def test_heat_diffusion(init_cuda): if dev.compute_capability < (9, 0): pytest.skip("Conditional nodes require compute capability >= 9.0") - _skip_if_no_mempool() + skip_if_mempool_unsupported() k_heat, k_countdown = _compile_heat_kernels() @@ -310,7 +306,7 @@ def test_bisection_root(init_cuda): if dev.compute_capability < (9, 0): pytest.skip("Conditional nodes require compute capability >= 9.0") - _skip_if_no_mempool() + skip_if_mempool_unsupported() k_eval, k_hi, k_lo, k_cd, k_check, k_newton = _compile_bisect_kernels() @@ -412,7 +408,7 @@ def test_switch_dispatch(init_cuda, mode, expected): if dev.compute_capability < (9, 0): pytest.skip("Conditional nodes require compute capability >= 9.0") - _skip_if_no_mempool() + skip_if_mempool_unsupported() k_negate, k_double, k_square = _compile_switch_kernels() diff --git a/cuda_core/tests/graph/test_graph_definition_lifetime.py b/cuda_core/tests/graph/test_graph_definition_lifetime.py index e049d76f500..5a3a99b65ea 100644 --- a/cuda_core/tests/graph/test_graph_definition_lifetime.py +++ b/cuda_core/tests/graph/test_graph_definition_lifetime.py @@ -13,7 +13,7 @@ import pytest from cuda_python_test_helpers.subprocess_runner import run_python_snippet from helpers.graph_kernels import compile_common_kernels -from helpers.memory import VMM_HANDLE_TYPE +from helpers.memory import VMM_HANDLE_TYPE, skip_if_managed_mempool_unsupported, skip_if_mempool_unsupported from helpers.misc import try_create_condition from cuda_python_test_helpers import under_compute_sanitizer @@ -102,13 +102,8 @@ def _wait_until(predicate, timeout=None, interval=0.02): from cuda.core.typing import GraphMemoryType -def _skip_if_no_mempool(): - if not Device(0).properties.memory_pools_supported: - pytest.skip("Device does not support mempool operations") - - def _device_memory_resource(dev): - _skip_if_no_mempool() + skip_if_mempool_unsupported() return DeviceMemoryResource(dev) @@ -126,12 +121,6 @@ def _virtual_memory_resource(dev): ] -def _skip_if_no_managed_mempool(): - _skip_if_no_mempool() - if not Device(0).properties.concurrent_managed_access: - pytest.skip("Device does not support managed memory pool operations") - - # ============================================================================= # Conditional body graph lifetime # ============================================================================= @@ -592,7 +581,7 @@ def callback(): def test_event_record_node_keeps_event_alive(init_cuda): """EventRecordNode should keep the Event alive after original is deleted.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() g = GraphDefinition() alloc = g.allocate(1024) @@ -609,7 +598,7 @@ def test_event_record_node_keeps_event_alive(init_cuda): def test_event_wait_node_keeps_event_alive(init_cuda): """EventWaitNode should keep the Event alive after original is deleted.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() g = GraphDefinition() alloc = g.allocate(1024) @@ -1305,7 +1294,7 @@ def _pred_chain_memcpy_host(g, bufs): def _pred_chain_memcpy_device(g, bufs): - _skip_if_no_mempool() + skip_if_mempool_unsupported() src = g.allocate(8) dst = g.allocate(8) return _make_pred_chain_memcpy(g.join(src, dst), src.dptr, dst.dptr) @@ -1336,7 +1325,7 @@ def check(reconstructed): def _pred_chain_alloc_peer_access(g, bufs): - _skip_if_no_mempool() + skip_if_mempool_unsupported() device = Device() node = g.allocate(64, device=device, peer_access=[device]) expected_peer_access = (device.device_id,) @@ -1350,7 +1339,7 @@ def check(reconstructed): def _pred_chain_alloc_managed(g, bufs): - _skip_if_no_managed_mempool() + skip_if_managed_mempool_unsupported() node = g.allocate(64, memory_type=GraphMemoryType.MANAGED) expected_dptr = node.dptr succ = node.record(Device().create_event()) @@ -1365,7 +1354,7 @@ def check(reconstructed): def _pred_chain_free(g, bufs): - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = g.allocate(64) node = alloc.deallocate(alloc.dptr) free_dptr = node.dptr @@ -1435,7 +1424,7 @@ def test_kernel_args_buffer_lifetime(init_cuda): """ from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) add_one = compile_common_kernels().get_kernel("add_one") @@ -1472,7 +1461,7 @@ def test_kernel_args_survive_graph_clone(init_cuda): """ from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) add_one = compile_common_kernels().get_kernel("add_one") @@ -1510,7 +1499,7 @@ def test_memset_buffer_lifetime(init_cuda): """Memset retains the Buffer allocation after the wrapper is collected.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1537,7 +1526,7 @@ def test_memset_buffer_survives_close(init_cuda): """Memset retains the allocation when the Buffer wrapper is closed.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1562,7 +1551,7 @@ def test_memcpy_buffer_lifetime(init_cuda): """Memcpy retains operand allocations after the Buffer wrappers are collected.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1591,7 +1580,7 @@ def test_memcpy_buffer_survives_close(init_cuda): """Memcpy retains allocations when Buffer wrappers are closed.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1627,7 +1616,7 @@ def test_memcpy_buffer_allocations_released_after_graph_destroyed(init_cuda): """ from cuda.core._utils._weak_handles import weak_handle - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1654,7 +1643,7 @@ def test_memcpy_buffers_survive_graph_clone(init_cuda): """Cloned graph keeps memcpy operand allocations alive via CUDA user objects.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1692,7 +1681,7 @@ def test_memset_raw_ptr_with_dst_owner(init_cuda): """Raw dst plus Buffer dst_owner retains the allocation after close.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1722,7 +1711,7 @@ def test_attachment_owners_released_after_graph_destroyed(init_cuda): This exercises the same teardown that releases Buffer device-pointer handles retained for ``dst`` and ``src``. """ - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(8, stream=dev.default_stream) @@ -1751,7 +1740,7 @@ def test_attachment_owners_released_after_graph_destroyed(init_cuda): @pytest.mark.agent_authored(model="gpt-5.6") def test_deleted_memcpy_bundle_survives_in_existing_exec(init_cuda): """Both memcpy owners remain together until an existing exec releases them.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(8, stream=dev.default_stream) @@ -1782,7 +1771,7 @@ def test_memcpy_raw_ptrs_with_owners(init_cuda): """Raw src/dst plus Buffer owners retain allocations after close.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1811,7 +1800,7 @@ def test_memcpy_mixed_buffer_and_raw_owner(init_cuda): """Buffer dst and raw src with src_owner retain allocations after close.""" from cuda.core._utils.cuda_utils import driver, handle_return - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) src = mr.allocate(4, stream=dev.default_stream) @@ -1838,7 +1827,7 @@ def test_memcpy_mixed_buffer_and_raw_owner(init_cuda): @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memset_closed_buffer_rejected(init_cuda): """Memset rejects a closed Buffer.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1853,7 +1842,7 @@ def test_memset_closed_buffer_rejected(init_cuda): @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memset_closed_buffer_dst_owner_rejected(init_cuda): """Memset rejects a closed Buffer passed as dst_owner.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1869,7 +1858,7 @@ def test_memset_closed_buffer_dst_owner_rejected(init_cuda): @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memcpy_closed_buffer_src_owner_rejected(init_cuda): """Memcpy rejects a closed Buffer passed as src_owner.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1885,7 +1874,7 @@ def test_memcpy_closed_buffer_src_owner_rejected(init_cuda): @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memcpy_buffer_and_dst_owner_rejected(init_cuda): """dst_owner cannot be combined with a Buffer dst operand.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) @@ -1899,7 +1888,7 @@ def test_memcpy_buffer_and_dst_owner_rejected(init_cuda): @pytest.mark.agent_authored(model="claude-opus-4.8") def test_memcpy_buffer_and_src_owner_rejected(init_cuda): """src_owner cannot be combined with a Buffer src operand.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() dev = Device() mr = DeviceMemoryResource(dev) buf = mr.allocate(4, stream=dev.default_stream) diff --git a/cuda_core/tests/helpers/constants.py b/cuda_core/tests/helpers/constants.py index f4ea61b1938..6a9459ec587 100644 --- a/cuda_core/tests/helpers/constants.py +++ b/cuda_core/tests/helpers/constants.py @@ -3,12 +3,8 @@ """Constants shared across the cuda_core test suite.""" -# Cap for memory pools created by tests. A pool created without an explicit -# max_size instead reserves a system-dependent window that scales with -# installed device memory -- hundreds of GiB on large-memory GPUs. The -# per-process virtual address budget is bounded (~1 TB on Windows MCDM), and a -# reservation is not returned until the pool is torn down and its -# stream-ordered frees retire, so oversized windows accumulate across a session -# and eventually starve later pool creations with CUDA_ERROR_OUT_OF_MEMORY -# (issue #2381). See AGENTS.md in the tests directory. -POOL_SIZE = 2097152 # 2 MiB +# Cap for memory pools created by tests; see cuda_python_test_helpers.memory +# for why it matters (issue #2381) and AGENTS.md in the tests directory. +from cuda_python_test_helpers.memory import POOL_SIZE + +__all__ = ["POOL_SIZE"] diff --git a/cuda_core/tests/helpers/memory.py b/cuda_core/tests/helpers/memory.py index 9f7de044bf5..77a879189a9 100644 --- a/cuda_core/tests/helpers/memory.py +++ b/cuda_core/tests/helpers/memory.py @@ -5,7 +5,7 @@ import pytest -from cuda.core import Device, ManagedMemoryResource, PinnedMemoryResource +from cuda.core import Device, DeviceMemoryResource, ManagedMemoryResource, PinnedMemoryResource from cuda.core._utils.cuda_utils import CUDAError from cuda_python_test_helpers import IS_WINDOWS @@ -21,6 +21,33 @@ class BaselineMemoryResourceReleasedWarning(Warning): """An ``init_cuda`` test released an owned memory pool or VA reservation that existed before it started.""" +def skip_if_mempool_unsupported(device=None): + """Skip unless ``device`` (default: device 0) supports memory pools.""" + if device is None: + device = Device(0) + if not device.properties.memory_pools_supported: + pytest.skip("Device does not support mempool operations") + + +def skip_if_managed_mempool_unsupported(device=None): + """Skip unless ``device`` (default: device 0) supports managed memory pools. + + This is a property check only. Unlike ``skip_if_managed_memory_unsupported`` + it never constructs a ``ManagedMemoryResource``, so it is also usable by + tests that allocate managed memory through graph nodes. + """ + if device is None: + device = Device(0) + skip_if_mempool_unsupported(device) + if not device.properties.concurrent_managed_access: + pytest.skip("Device does not support managed memory pool operations") + + +def is_pool_resource(mr): + """Whether ``mr`` has the pool half of the IPC protocol (allocation handle, registry, uuid).""" + return isinstance(mr, (DeviceMemoryResource, PinnedMemoryResource)) + + def skip_if_pinned_memory_unsupported(device): try: if not device.properties.host_memory_pools_supported: diff --git a/cuda_core/tests/memory/test_resource_live_count.py b/cuda_core/tests/memory/test_resource_live_count.py index 75cc6d956ca..892f11395c7 100644 --- a/cuda_core/tests/memory/test_resource_live_count.py +++ b/cuda_core/tests/memory/test_resource_live_count.py @@ -21,7 +21,7 @@ import pytest from helpers.constants import POOL_SIZE -from helpers.memory import VMM_HANDLE_TYPE +from helpers.memory import VMM_HANDLE_TYPE, skip_if_mempool_unsupported from cuda.core import ( DeviceMemoryResource, @@ -34,11 +34,6 @@ _THREAD_UNSAFE_MARKER = pytest.mark.thread_unsafe(reason="asserts on the process-global owned-resource counts") -def _skip_without_mempools(device): - if not device.properties.memory_pools_supported: - pytest.skip("Device does not support memory pools") - - def _owned_pool(device): """An owning DeviceMemoryResource: passing options creates its own pool.""" return DeviceMemoryResource(device, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) @@ -47,7 +42,7 @@ def _owned_pool(device): @_THREAD_UNSAFE_MARKER @pytest.mark.agent_authored(model="claude-opus-4.8") def test_owned_pool_is_counted_until_freed(init_cuda): - _skip_without_mempools(init_cuda) + skip_if_mempool_unsupported(init_cuda) before = _live_owned_mempool_count() mr = _owned_pool(init_cuda) assert _live_owned_mempool_count() == before + 1 @@ -61,7 +56,7 @@ def test_borrowed_default_pool_is_not_counted(init_cuda): # A no-options DeviceMemoryResource borrows the device default pool # (cuDeviceGetMemPool), which this layer does not own or destroy, so it must # not move the owned-pool counter. - _skip_without_mempools(init_cuda) + skip_if_mempool_unsupported(init_cuda) before = _live_owned_mempool_count() mr = DeviceMemoryResource(init_cuda) assert _live_owned_mempool_count() == before @@ -72,7 +67,7 @@ def test_borrowed_default_pool_is_not_counted(init_cuda): @_THREAD_UNSAFE_MARKER @pytest.mark.agent_authored(model="claude-opus-4.8") def test_owned_pool_in_cycle_stays_counted_until_collect(init_cuda): - _skip_without_mempools(init_cuda) + skip_if_mempool_unsupported(init_cuda) # Disable automatic collection so a background gc cannot free the cycle # before the still-counted assertion is checked. enabled = gc.isenabled() @@ -116,7 +111,7 @@ def test_concurrent_owned_pool_construct_and_drop_returns_to_start(init_cuda): # On a free-threaded build this guards against a non-atomic counter; on a # GIL build it passes trivially. Every thread sets the device current, then # creates and drops owning pools; after join the count is back at the start. - _skip_without_mempools(init_cuda) + skip_if_mempool_unsupported(init_cuda) before = _live_owned_mempool_count() n_threads = 4 n_per_thread = 10 diff --git a/cuda_core/tests/memory_ipc/test_errors.py b/cuda_core/tests/memory_ipc/test_errors.py index dd03c59d773..dffb410134f 100644 --- a/cuda_core/tests/memory_ipc/test_errors.py +++ b/cuda_core/tests/memory_ipc/test_errors.py @@ -11,6 +11,7 @@ import pytest from helpers.child_processes import child_timeout_sec, kill_subprocesses from helpers.constants import POOL_SIZE +from helpers.memory import is_pool_resource from cuda.core import ( Buffer, @@ -32,11 +33,6 @@ pytestmark = pytest.mark.parallel_threads_limit(2) -def _is_pool(mr): - """Whether ``mr`` has the pool half of the IPC protocol (allocation handle, registry, is_mapped).""" - return isinstance(mr, (DeviceMemoryResource, PinnedMemoryResource)) - - def test_outer_timeout_marker_is_applied(request): """Verify that memory_ipc/conftest.py applies the outer pytest-timeout marker. @@ -64,7 +60,7 @@ def test_ipc_types_cannot_be_constructed_directly(): def test_import_truncated_buffer_descriptor(ipc_device, ipc_memory_resource): """Truncated IPC buffer descriptor payload is rejected before driver import.""" desc = IPCBufferDescriptor._init(b"\x00" * 8, NBYTES) - if _is_pool(ipc_memory_resource): + if is_pool_resource(ipc_memory_resource): expected = pytest.raises(ValueError, match=r"payload is 8 bytes; expected at least 64") else: # A pool descriptor cannot be imported with a VirtualMemoryResource at all. @@ -114,7 +110,7 @@ def test_ipc_allocation_handle_state_tracks_close(): @pytest.mark.agent_authored(model="gpt-5.6") def test_closed_ipc_allocation_handle_rejected_before_registry_hit(ipc_device, ipc_memory_resource): mr = ipc_memory_resource - if not _is_pool(mr): + if not is_pool_resource(mr): pytest.skip("allocation handles and the registry are pool-only") handle = IPCAllocationHandle._init(os.dup(int(mr.allocation_handle)), mr.uuid) assert mr.register(mr.uuid) is mr @@ -149,7 +145,7 @@ def test_main(self, ipc_device, ipc_memory_resource): self.device = ipc_device self.mr = ipc_memory_resource self._extra_mrs = [] - if self.POOL_ONLY and not _is_pool(self.mr): + if self.POOL_ONLY and not is_pool_resource(self.mr): pytest.skip("this scenario needs the pool half of the IPC protocol") # Clear stale 'buffer' left by a previous flaky-rerun attempt. # Its backing DeviceMemoryResource may already be closed, which causes @@ -210,7 +206,7 @@ def PARENT_ACTION(self, queue): stream = self.device.default_stream self.buffer = self.mr.allocate(NBYTES, stream=stream) desc = self.buffer.ipc_descriptor - if _is_pool(self.mr): + if is_pool_resource(self.mr): payload, _ = desc.__reduce__()[1] oversized = IPCBufferDescriptor._init(payload, NBYTES * 100) else: @@ -227,7 +223,7 @@ def CHILD_ACTION(self, queue): def ASSERT(self, exc_type, exc_msg): assert exc_type is ValueError assert "exceeds" in exc_msg - if _is_pool(self.mr): + if is_pool_resource(self.mr): assert "mapped allocation extent" in exc_msg else: assert "exported allocations" in exc_msg @@ -245,7 +241,7 @@ def CHILD_ACTION(self, queue): buffer.close() def ASSERT(self, exc_type, exc_msg): - if _is_pool(self.mr): + if is_pool_resource(self.mr): assert exc_type is TypeError assert exc_msg == "Cannot allocate from a mapped IPC-enabled memory resource" else: @@ -258,7 +254,7 @@ class TestImportWrongMR(ChildErrorHarness): """Importing a buffer with a resource other than the exporter's: an error for pools, fine for VMM.""" def PARENT_ACTION(self, queue): - if _is_pool(self.mr): + if is_pool_resource(self.mr): options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True) mr2 = DeviceMemoryResource(self.device, options=options) self._extra_mrs.append(mr2) @@ -274,7 +270,7 @@ def CHILD_ACTION(self, queue): Buffer.from_ipc_descriptor(mr, buffer_desc, stream=self.device.default_stream).close() def ASSERT(self, exc_type, exc_msg): - if _is_pool(self.mr): + if is_pool_resource(self.mr): assert exc_type is CUDAError assert "CUDA_ERROR_INVALID_VALUE" in exc_msg else: diff --git a/cuda_core/tests/memory_ipc/test_memory_ipc.py b/cuda_core/tests/memory_ipc/test_memory_ipc.py index 477abc93da9..34b9de73405 100644 --- a/cuda_core/tests/memory_ipc/test_memory_ipc.py +++ b/cuda_core/tests/memory_ipc/test_memory_ipc.py @@ -6,8 +6,9 @@ import pytest from helpers.buffers import PatternGen from helpers.child_processes import child_timeout_sec, kill_subprocesses +from helpers.memory import is_pool_resource -from cuda.core import Buffer, DeviceMemoryResource, PinnedMemoryResource, VirtualMemoryResource +from cuda.core import Buffer, DeviceMemoryResource, VirtualMemoryResource CHILD_TIMEOUT_SEC = child_timeout_sec() NBYTES = 64 @@ -19,14 +20,9 @@ pytestmark = pytest.mark.parallel_threads_limit(2) -def _is_pool(mr): - """Whether ``mr`` has the pool half of the IPC protocol (allocation handle, registry, is_mapped).""" - return isinstance(mr, (DeviceMemoryResource, PinnedMemoryResource)) - - def _share_token(mr): """What a child needs to rebuild ``mr``: a pool's allocation handle, or the VMM resource itself.""" - return mr.allocation_handle if _is_pool(mr) else mr + return mr.allocation_handle if is_pool_resource(mr) else mr def _import_token(device, token): @@ -43,7 +39,7 @@ def test_main(self, ipc_device, ipc_memory_resource): # Set up the IPC-enabled memory pool and share it. device = ipc_device mr = ipc_memory_resource - if _is_pool(mr): + if is_pool_resource(mr): assert not mr.is_mapped stream = device.default_stream pgen = PatternGen(device, NBYTES, stream=stream) @@ -75,7 +71,7 @@ def test_main(self, ipc_device, ipc_memory_resource): def child_main(self, device, mr, queue): device.set_current() - if _is_pool(mr): + if is_pool_resource(mr): assert mr.is_mapped buffer = queue.get(timeout=CHILD_TIMEOUT_SEC) assert buffer.is_mapped diff --git a/cuda_core/tests/memory_ipc/test_serialize.py b/cuda_core/tests/memory_ipc/test_serialize.py index a44f480a8df..46858df1339 100644 --- a/cuda_core/tests/memory_ipc/test_serialize.py +++ b/cuda_core/tests/memory_ipc/test_serialize.py @@ -8,6 +8,7 @@ import pytest from helpers.buffers import PatternGen from helpers.child_processes import child_timeout_sec, kill_subprocesses +from helpers.memory import is_pool_resource from cuda.core import Buffer, Device, DeviceMemoryResource, PinnedMemoryResource, VirtualMemoryResource @@ -19,11 +20,6 @@ pytestmark = pytest.mark.parallel_threads_limit(2) -def _is_pool(mr): - """Whether ``mr`` has the pool half of the IPC protocol (allocation handle, registry, uuid).""" - return isinstance(mr, (DeviceMemoryResource, PinnedMemoryResource)) - - class TestObjectSerializationDirect: """ Test the low-level interface for sharing memory resources. @@ -46,8 +42,8 @@ def test_main(self, ipc_device, ipc_memory_resource): # Send a memory resource: a pool by its allocation handle over the raw # fd channel, a VirtualMemoryResource as the object itself (it holds no # driver object; a VMM descriptor carries its fds per allocation). - parent_conn.send(_is_pool(mr)) - if _is_pool(mr): + parent_conn.send(is_pool_resource(mr)) + if is_pool_resource(mr): alloc_handle = mr.allocation_handle mp.reduction.send_handle(parent_conn, alloc_handle.handle, process.pid) else: @@ -151,7 +147,7 @@ def child_main(self, pipe, _): # Buffer. buffer = pipe[0].get(timeout=CHILD_TIMEOUT_SEC) - if _is_pool(mr): + if is_pool_resource(mr): assert buffer.memory_resource.handle == mr.handle else: # A VMM resource pickles as (device, options); the buffer's rebuilt @@ -178,7 +174,7 @@ def test_main(self, ipc_device, ipc_memory_resource): # Define the objects. device = ipc_device mr = ipc_memory_resource - alloc_handle = mr.allocation_handle if _is_pool(mr) else None + alloc_handle = mr.allocation_handle if is_pool_resource(mr) else None stream = device.default_stream buffer = mr.allocate(NBYTES, stream=stream) buffer_desc = buffer.ipc_descriptor diff --git a/cuda_core/tests/test_object_protocols.py b/cuda_core/tests/test_object_protocols.py index 022ee456fd9..1e1e410f807 100644 --- a/cuda_core/tests/test_object_protocols.py +++ b/cuda_core/tests/test_object_protocols.py @@ -15,6 +15,7 @@ import pytest from helpers.constants import POOL_SIZE from helpers.graph_kernels import compile_common_kernels +from helpers.memory import skip_if_mempool_unsupported from helpers.misc import try_create_condition from cuda.core import ( @@ -31,12 +32,6 @@ from cuda.core._program import _can_load_generated_ptx from cuda.core.graph import GraphDefinition - -def _skip_if_no_mempool(): - if not Device(0).properties.memory_pools_supported: - pytest.skip("Device does not support mempool operations") - - # ============================================================================= # Fixtures - Primary samples # ============================================================================= @@ -280,7 +275,7 @@ def sample_root_node_alt(sample_graphdef_alt): @pytest.fixture def sample_empty_node(sample_graphdef): """An EmptyNode created by merging two branches.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() a = sample_graphdef.allocate(ALLOC_SIZE) b = sample_graphdef.allocate(ALLOC_SIZE) return sample_graphdef.join(a, b) @@ -289,7 +284,7 @@ def sample_empty_node(sample_graphdef): @pytest.fixture def sample_empty_node_alt(sample_graphdef): """An alternate EmptyNode from same graph.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() c = sample_graphdef.allocate(ALLOC_SIZE) d = sample_graphdef.allocate(ALLOC_SIZE) return sample_graphdef.join(c, d) @@ -298,14 +293,14 @@ def sample_empty_node_alt(sample_graphdef): @pytest.fixture def sample_alloc_node(sample_graphdef): """An AllocNode.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() return sample_graphdef.allocate(ALLOC_SIZE) @pytest.fixture def sample_alloc_node_alt(sample_graphdef): """An alternate AllocNode from same graph.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() return sample_graphdef.allocate(ALLOC_SIZE) @@ -330,7 +325,7 @@ def sample_kernel_node_alt(sample_graphdef, init_cuda): @pytest.fixture def sample_free_node(sample_graphdef): """A FreeNode.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) return alloc.deallocate(alloc.dptr) @@ -338,7 +333,7 @@ def sample_free_node(sample_graphdef): @pytest.fixture def sample_free_node_alt(sample_graphdef): """An alternate FreeNode from same graph.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) return alloc.deallocate(alloc.dptr) @@ -346,7 +341,7 @@ def sample_free_node_alt(sample_graphdef): @pytest.fixture def sample_memset_node(sample_graphdef): """A MemsetNode.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) return alloc.memset(alloc.dptr, 0, ALLOC_SIZE) @@ -354,7 +349,7 @@ def sample_memset_node(sample_graphdef): @pytest.fixture def sample_memset_node_alt(sample_graphdef): """An alternate MemsetNode from same graph.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() alloc = sample_graphdef.allocate(ALLOC_SIZE) return alloc.memset(alloc.dptr, 0, ALLOC_SIZE) @@ -362,7 +357,7 @@ def sample_memset_node_alt(sample_graphdef): @pytest.fixture def sample_memcpy_node(sample_graphdef): """A MemcpyNode.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() src = sample_graphdef.allocate(ALLOC_SIZE) dst = sample_graphdef.allocate(ALLOC_SIZE) dep = sample_graphdef.join(src, dst) @@ -372,7 +367,7 @@ def sample_memcpy_node(sample_graphdef): @pytest.fixture def sample_memcpy_node_alt(sample_graphdef): """An alternate MemcpyNode from same graph.""" - _skip_if_no_mempool() + skip_if_mempool_unsupported() src = sample_graphdef.allocate(ALLOC_SIZE) dst = sample_graphdef.allocate(ALLOC_SIZE) dep = sample_graphdef.join(src, dst) diff --git a/cuda_core/tests/test_rlcompleter_patch.py b/cuda_core/tests/test_rlcompleter_patch.py index 92a77efee8e..de7306a4b10 100644 --- a/cuda_core/tests/test_rlcompleter_patch.py +++ b/cuda_core/tests/test_rlcompleter_patch.py @@ -22,6 +22,7 @@ import pytest from cuda_python_test_helpers.subprocess_runner import run_python_snippet +from helpers.memory import skip_if_mempool_unsupported from cuda.core import Device @@ -30,9 +31,7 @@ def _gpu_with_mempool_or_skip(): """Skip when no GPU or no mempool support — test mirrors the bug repro.""" if len(Device.get_all_devices()) == 0: pytest.skip("Test requires a CUDA device") - dev = Device(0) - if not dev.properties.memory_pools_supported: - pytest.skip("Device 0 does not support mempool operations") + skip_if_mempool_unsupported() # Probe script: reproduces the bug-report repro literally, then runs diff --git a/cuda_python_test_helpers/cuda_python_test_helpers/memory.py b/cuda_python_test_helpers/cuda_python_test_helpers/memory.py new file mode 100644 index 00000000000..cb4ca161f12 --- /dev/null +++ b/cuda_python_test_helpers/cuda_python_test_helpers/memory.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Memory-pool helpers shared by the cuda.bindings and cuda.core test suites.""" + +import pytest + +from cuda.bindings import runtime as cudart + +__all__ = [ + "POOL_SIZE", + "supports_managed_memory", + "supports_memory_pools", + "xfail_graph_mempool_oom", +] + +# Cap for memory pools created by tests. A pool created without an explicit +# maximum size instead reserves a system-dependent window that scales with +# installed device memory: about twice the device memory, which is hundreds of +# GiB on large-memory GPUs. That reservation can fail where the virtual address +# space is limited (e.g. riscv64 Sv39, a 39-bit address space). The per-process +# virtual address budget is also bounded (~1 TB on Windows MCDM), and a +# reservation is not returned until the pool is torn down and its +# stream-ordered frees retire, so oversized windows accumulate across a session +# and eventually starve later pool creations with CUDA_ERROR_OUT_OF_MEMORY +# (issue #2381). +POOL_SIZE = 2 * 1024 * 1024 # 2 MiB + + +def _device_attribute_is_set(attribute, device): + err, value = cudart.cudaDeviceGetAttribute(attribute, device) + return err == cudart.cudaError_t.cudaSuccess and bool(value) + + +def supports_memory_pools(device=0): + """Return True if ``device`` supports stream-ordered memory pools. + + A failing attribute query (e.g. no device) counts as unsupported, so this + is safe to evaluate in a ``skipif`` condition at collection time. + """ + return _device_attribute_is_set(cudart.cudaDeviceAttr.cudaDevAttrMemoryPoolsSupported, device) + + +def supports_managed_memory(device=0): + """Return True if ``device`` supports managed memory. + + A failing attribute query (e.g. no device) counts as unsupported. + """ + return _device_attribute_is_set(cudart.cudaDeviceAttr.cudaDevAttrManagedMemory, device) + + +def xfail_graph_mempool_oom(): + """Mark the calling test as an expected failure; never returns. + + Call this after the graph memory pool failed with an out-of-memory error. + The graph memory pool reserves virtual address space of about twice the + device memory, and unlike pools from ``cuMemPoolCreate`` it cannot be + capped. On systems with a limited virtual address space this reservation + can fail with CUDA_ERROR_OUT_OF_MEMORY, especially when another uncapped + pool already exists in the process. + + The caller does its own out-of-memory check and resource cleanup first. + """ + pytest.xfail("graph memory pool creation failed with CUDA_ERROR_OUT_OF_MEMORY")