Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ Tack is a Python-first GPU compute framework inspired by Taichi. Kernels are dec

The measurements behind these choices (wait times, launch counts, thread schedules) are in `docs/design/backend-implementations.md` ("How each backend queues and waits") and `docs/design/memory-and-aliasing.md` ("New arrays, fills and one-element reads").

`tack.empty()` (under every array constructor) calls `backend.allocate_array()` through `_allocate`, which, on a `MemoryError` (every GPU backend raises `runtime.backend.DeviceMemoryError` when an allocation runs out), collects Python's garbage, waits for the device and tries once more: a dropped array in a reference cycle (a topology's cached locator holds a dataset of it) keeps its device memory until the cycle collector runs (`test_out_of_memory.py`). A new array's contents are unspecified on every backend; `tack.zeros` fills it, and `Array.fill` first offers the fill to `Backend.fill_array`, which the CPU backend answers for 256 KiB or more by running `tack.algorithms.copy._fill_kernel` through the normal dispatch, so the workers fill (and first-touch) the pages. On Linux `TACK_CPU_MALLOC=1` tells glibc to keep freed pages (`_tune_malloc`; off by default, since it is process-wide). `Array.__getitem__` reads one element through `DeviceBuffer.read_range(start, count)`, which copies just that range after the queued launches (each GPU backend checks the range against the allocation); `__setitem__` and `__iter__` raise.
`tack.empty()` (under every array constructor) calls `backend.allocate_array()` through `_allocate`, which, on a `MemoryError` (every GPU backend raises `runtime.backend.DeviceMemoryError` when an allocation runs out), collects Python's garbage, waits for the device and tries once more: a dropped array in a reference cycle (a topology's cached locator holds a dataset of it) keeps its device memory until the cycle collector runs (`test_out_of_memory.py`). Every buffer `_allocate` and `_empty_for_mpi` make is counted on its device (`runtime/memory_stats`, private: `live`, `peak`, `reset_peak`; its bytes come back through a weak reference's callback when the buffer is collected, about 0.9 us an array): views once, with their buffer; memory from outside (`wrap_ptr`, DLPack, `array_from_ptr`) not at all; a pooled block while an array holds it. The root conftest's `peak_memory` fixture measures one call's peak with it, on any backend (`test_memory_stats.py`; the cut filters' memory plan, D1). A new array's contents are unspecified on every backend; `tack.zeros` fills it, and `Array.fill` first offers the fill to `Backend.fill_array`, which the CPU backend answers for 256 KiB or more by running `tack.algorithms.copy._fill_kernel` through the normal dispatch, so the workers fill (and first-touch) the pages. On Linux `TACK_CPU_MALLOC=1` tells glibc to keep freed pages (`_tune_malloc`; off by default, since it is process-wide). `Array.__getitem__` reads one element through `DeviceBuffer.read_range(start, count)`, which copies just that range after the queued launches (each GPU backend checks the range against the allocation); `__setitem__` and `__iter__` raise.

### Backend contract

Expand Down
21 changes: 17 additions & 4 deletions benchmarks/vtk_compare/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@
before it (temporaries and output);
- ``kept``: what it holds after the call while the output is alive (the
output, and whatever the filter derived and kept on the input);
- the call's time.
- the call's time;
- for Tack, ``counted``: the most its arrays held during the call beyond what
they held before (`tack.runtime.memory_stats`, the count of every array Tack
makes), which sees any device and none of NumPy's or the allocator's own.

The memory measured is what the allocator has handed out and not taken back
(``malloc_zone_statistics`` on macOS, ``mallinfo2`` on Linux): every live
Expand Down Expand Up @@ -180,17 +183,25 @@ def run(data, mesh):
mesh = build(m.SIZES[size])
source, input_bytes = prepare(mesh)
_release_free_memory()
counted = None
if system == "tack":
from tack.runtime import memory_stats

counted = memory_stats.reset_peak()
before = read()
with _Sampler(read) as sampler:
start = time.perf_counter()
result = run(source, mesh)
seconds = time.perf_counter() - start
if counted is not None:
counted = memory_stats.peak() - counted
_release_free_memory()
kept = read() - before
del result
return {"mesh": mesh_name, "size": size, "cells": mesh.num_cells, "filter": filter_name,
"system": system, "threads": threads, "input": input_bytes,
"peak": sampler.peak - before, "kept": kept, "seconds": seconds}
"peak": sampler.peak - before, "kept": kept, "seconds": seconds,
"counted": counted}


def _child(args):
Expand Down Expand Up @@ -228,7 +239,7 @@ def main():
names = [q.name for q in p.PAIRS if not args.filters or q.name in args.filters.split(",")]
records = []
print(f"{'mesh':>4} {'size':>5} {'filter':<15} {'system':<5} {'input MB':>8} "
f"{'peak MB':>8} {'kept MB':>8} {'s':>7} peak/input", flush=True)
f"{'peak MB':>8} {'kept MB':>8} {'s':>7} peak/input counted MB", flush=True)
for mesh in args.meshes.split(","):
for size in args.sizes.split(","):
for name in names:
Expand All @@ -242,7 +253,9 @@ def main():
continue
print(f"{mesh:>4} {size:>5} {name:<15} {system:<5} {_gb(r['input'])} "
f"{_gb(r['peak'])} {_gb(r['kept'])} {r['seconds']:7.3f} "
f"{r['peak'] / max(r['input'], 1):5.2f}", flush=True)
f"{r['peak'] / max(r['input'], 1):5.2f}"
+ (f" {_gb(r['counted'])}" if r.get("counted") is not None else ""),
flush=True)
if args.output:
with open(args.output, "w") as f:
json.dump(records, f, indent=1)
Expand Down
14 changes: 14 additions & 0 deletions benchmarks/vtk_compare/pairs.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,16 @@ def _threshold(grid, mesh, form):
return f


def _table_clip(grid, mesh, form):
from vtkmodules.vtkFiltersGeneral import vtkTableBasedClipDataSet

# Keeps p above the value, as td.clip keeps it at or above (off every
# point's value, as the contour is).
f = vtkTableBasedClipDataSet(input_data=grid, value=CONTOUR_ISOVALUE)
f.SetInputArrayToProcess(0, 0, 0, 0, "p")
return f


def _geometry_filter(grid, mesh, form):
from vtkmodules.vtkFiltersGeometry import vtkGeometryFilter

Expand Down Expand Up @@ -208,6 +218,10 @@ def _tack():
vtk=_plane_cutter,
tack=lambda d, m: _tack()[0].slice_plane(d, tuple(_center(m)), tuple(NORMAL)),
point_fields=("p",), output="surface"),
Pair("clip",
vtk=_table_clip,
tack=lambda d, m: _tack()[0].clip(d, "p", CONTOUR_ISOVALUE),
point_fields=("p",), forms=("shape",), output="volume"),
Pair("threshold",
vtk=_threshold,
tack=lambda d, m: _tack()[0].threshold(d, "c", *THRESHOLD),
Expand Down
8 changes: 8 additions & 0 deletions changelog.d/clip-point-map.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
### Changed

- `clip` numbers the input points its cells keep through a map of one entry per
input point, and sorts only the crossings on cut edges, as VTK's
`vtkTableBasedClipDataSet` does: it made a record for every kept corner and
sorted them all. The output is the same bits. On 5M cells (CPU) its peak fell
from 364 to 256 MiB on tetrahedra and from 574 to 315 MiB on hexahedra, and a
warm clip from 43 to 30 ms and from 61 to 54 ms.
21 changes: 21 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,27 @@ def no_debug():
tack.set_debug(was)


@pytest.fixture
def peak_memory():
"""A function measuring the most memory Tack's arrays held on a device during
one call, above what they held before it: ``result, nbytes = peak_memory(fn,
*args, device=None, **kwargs)``. It reads the count of every array Tack makes
(`tack.runtime.memory_stats`), the same on every backend; garbage is collected
first, so arrays dropped before the call are not in it. For a test that holds a
filter's peak to a multiple of its output (the cut filters' memory plan)."""
import gc

from tack.runtime import memory_stats

def measure(fn, *args, device=None, **kwargs):
gc.collect()
start = memory_stats.reset_peak(device)
result = fn(*args, **kwargs)
return result, memory_stats.peak(device) - start

return measure


@pytest.fixture
def no_disk_cache():
"""For a test that counts compile work in this process, which a disk-cache
Expand Down
13 changes: 8 additions & 5 deletions packages/tack-core/src/tack/lang/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
mat,
vec,
)
from tack.runtime import memory_stats


class DeviceBuffer:
Expand Down Expand Up @@ -1102,16 +1103,18 @@ def _allocate(backend, dtype, shape, **options):
memory: a 16 GB GPU ran out with dropped meshes still on it. The wait
lets frees a backend defers (Level Zero's, a device without memory
pools) complete before the retry, and buffers the collection returned to
a backend's cache (HIP's, Metal's) are freed (``release_memory``)."""
a backend's cache (HIP's, Metal's) are freed (``release_memory``). The
buffer is counted on its device (`tack.runtime.memory_stats`)."""
try:
return backend.allocate_array(dtype, shape, **options)
buffer = backend.allocate_array(dtype, shape, **options)
except MemoryError:
import gc

gc.collect()
backend.synchronize()
backend.release_memory() # what the collection gave back to a cache
return backend.allocate_array(dtype, shape, **options)
buffer = backend.allocate_array(dtype, shape, **options)
return memory_stats.counted(backend, buffer)


def _empty_for_mpi(shape, dtype, device) -> Array:
Expand All @@ -1124,8 +1127,8 @@ def _empty_for_mpi(shape, dtype, device) -> Array:
scalar = dtype.scalar_dtype if isinstance(dtype, ElementType) else dtype
_check_dtype(backend, scalar)
storage = (int(math.prod(shape)) * dtype.size,) if isinstance(dtype, ElementType) else shape
return _array_view(dtype, shape, backend.allocate_mpi_array(scalar, storage),
device=backend)
buffer = memory_stats.counted(backend, backend.allocate_mpi_array(scalar, storage))
return _array_view(dtype, shape, buffer, device=backend)


def zeros(shape, dtype=None, *, device=None, layout="aos") -> Array:
Expand Down
105 changes: 105 additions & 0 deletions packages/tack-core/src/tack/runtime/memory_stats.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
# SPDX-FileCopyrightText: Copyright (c) Kitware Inc.
# SPDX-License-Identifier: BSD-3-Clause

"""The memory Tack's own arrays hold, counted per device, with a peak (private).

Every array Tack makes is counted where it is allocated (`tack.empty`'s
``_allocate`` and ``_empty_for_mpi``, which every backend's ``allocate_array``
goes through), and its bytes are given back when its buffer is collected. So
the CPU, Metal, CUDA, HIP, Level Zero and Warp are counted alike, which the
process's allocator statistics (the CPU only) and the drivers (behind CUDA's
pool, HIP's buffer cache and Metal's pool) cannot do. What is counted:

- the bytes an array asked for (``DeviceBuffer.nbytes``), not what a pool
rounded them up to;
- memory from outside (``wrap_ptr``, ``array_from_ptr``, a DLPack or NumPy
import without a copy) is not Tack's, and is not counted;
- a view shares its array's buffer, and is counted once, with the buffer: the
memory is held while any view of it is;
- a block a backend's cache hands to a new array is counted while that array
holds it, not while it waits in the cache (``release_memory`` empties the
caches);
- the few bytes of a compiled kernel's scalar packs are not counted.

`reset_peak` makes the peak what is live, so a test measures one call:
``reset_peak(d); start = live(d); f(); peak(d) - start``. Each function takes a
`tack.Device`, a spec (``"cuda:1"``), a backend, or None for the default
device. The cut filters' memory plan (``docs/design/plans/cut-filters-memory.md``,
D1) is why it exists; spilling to host memory under a budget (#387) will read
it too.
"""

import threading
import weakref

# A backend's count, kept while the backend lives. What a buffer gives back is
# held with its weak reference, the count and not the backend, so a buffer
# never keeps its backend alive here.
_counts = weakref.WeakKeyDictionary()
# Each counted buffer's weak reference: (count, bytes). A weak reference with a
# callback costs a third of `weakref.finalize` (0.2 against 0.8 us), and every
# array pays it.
_held = {}
# Reentrant: a buffer's finalizer can run inside a count's update, when the
# update's allocation sets off the garbage collector.
_lock = threading.RLock()


class _Count:
__slots__ = ("live", "peak")

def __init__(self):
self.live = 0
self.peak = 0


def _count_of(backend):
count = _counts.get(backend)
if count is None:
with _lock:
count = _counts.setdefault(backend, _Count())
return count


def _released(ref):
count, nbytes = _held.pop(ref)
with _lock:
count.live -= nbytes


def counted(backend, buffer):
"""``buffer``, just allocated on ``backend``, added to the backend's count until
it is collected."""
nbytes = int(buffer.nbytes)
count = _count_of(backend)
with _lock:
count.live += nbytes
if count.live > count.peak:
count.peak = count.live
_held[weakref.ref(buffer, _released)] = (count, nbytes)
return buffer


def _backend(device):
from tack.lang.array import _target

return _target(device)


def live(device=None) -> int:
"""The bytes Tack's arrays hold on ``device`` now."""
return _count_of(_backend(device)).live


def peak(device=None) -> int:
"""The most bytes Tack's arrays have held on ``device`` at once since it was
first used or `reset_peak` was called."""
return _count_of(_backend(device)).peak


def reset_peak(device=None) -> int:
"""Make ``device``'s peak what is live now, and return it."""
count = _count_of(_backend(device))
with _lock:
count.peak = count.live
return count.peak
Loading
Loading