libfork/bench/memory_run.py at main · ConorWilliams/libfork · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
# A small python utility to measure memory usage using the GNU time utility


import os
import argparse
import subprocess
import re
from statistics import median, stdev
from math import sqrt

# Parse location of benchmark and regex base.

parser = argparse.ArgumentParser(
    description="Measure memory usage of libfork's benchmarks."
)


parser.add_argument("binary", type=str, help="The benchmark binary to run")
parser.add_argument("bench", type=str, help="The regex benchmark")
parser.add_argument("cores", type=int, help="max number of cores")

args = parser.parse_args()

bench = args.bench

if not bench.startswith("T"):
    libfork = [
        "libfork.*lazy.*fan",
        "libfork.*busy.*fan",
        "libfork.*lazy.*seq",
        "libfork.*busy.*seq",
    ]
else:
    libfork = [
        "libfork.*_alloc_.*lazy.*fan",
        "libfork.*_alloc_.*busy.*fan",
        "libfork.*_alloc_.*lazy.*seq",
        "libfork.*_alloc_.*busy.*seq",
        "libfork.*_coalloc_.*lazy.*fan",
        "libfork.*_coalloc_.*busy.*fan",
        "libfork.*_coalloc_.*lazy.*seq",
        "libfork.*_coalloc_.*busy.*seq",
    ]

with open(f"memory.{bench.strip()}.csv", "w") as file:
    for kind in [
        "zero",
        "calibrate",
        "serial",
        *libfork,
        "omp",
        "tbb",
        "taskflow",
    ]:
        print(f"Running {kind} {bench.strip()}")

        is_serial = kind == "serial" or kind == "calibrate" or kind == "zero"

        for i in [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112]:
            #
            if i > args.cores:
                break

            if kind == "zero":
                reg = "NONNAMEDTHIS"
            elif kind == "calibrate":
                reg = "calibrate"
            elif bench.startswith("T"):
                reg = f"uts.*{kind}.*{bench}"
            else:
                reg = f"{bench}.*{kind}"

            if not is_serial:
                reg += f".*/{i}/"
            elif i > 1:
                break

            mem = []

            for r in range(25 if is_serial else 5):
                command = f'/usr/bin/time -f"MEMORY=%M"  -- {args.binary} --benchmark_filter="{reg}" --benchmark_time_unit=ms'

                # print(f"{command=}")

                output = subprocess.run(
                    command, shell=True, check=True, stderr=subprocess.PIPE
                ).stderr

                match = re.search(".*MEMORY=([1-9][0-9]*)", str(output))

                if match:
                    val = int(match.group(1))
                    mem.append(val)
                else:
                    raise "No memory found"

            x = median(mem)
            e = stdev(mem) / sqrt(len(mem))

            print(f"mems={mem} -> {x}, {e}")

            file.write(f"{kind},{i},{x},{e}\n")
            file.flush()