CI benchmarking suite take 2 (#636)

* create simple benchmark suite

* more hierarchy in benchmark

* create AirspeedVelocity github action for benchmarks

* fix imports in benchmark

* fix default branch name in CI

* benchmarks: more hierarchy

* timing + benchmark

* benchmarks: include gcbench in runs

* benchmarks: remove unused code

* benchmarks: modularize

* benchmarks: give more time to GC benchmark

* benchmarks: simplify naming scheme

* benchmarks: avoid issue of `tune`ing away from `evals=1`

* use getptr a lot less

* ci: update to new ASV

* ci: switch to job summary version

* ci: bench on PR sha

* ci: run on PR to any branch, not just main

* ci: run tuned benchmarks

* undo changes from merge

---------

Co-authored-by: Eric Hanson <[email protected]>
Co-authored-by: Christopher Doris <github.com/cjdoris>

Author: MilesCranmer

.github/workflows/benchmark_pr.yml

@@ -0,0 +1,16 @@
+name: Benchmark PR
+on:
+  pull_request:
+
+jobs:
+  bench:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: MilesCranmer/AirspeedVelocity.jl@action-v1
+        with:
+          julia-version: "1"
+          tune: "true"
+          # Post to "summary" tab of workflow run:
+          job-summary: "true"
+          # Run benchmark using PR's version of the script:
+          bench-on: ${{ github.event.pull_request.head.sha }}

benchmark/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"

benchmark/benchmarks.jl

@@ -0,0 +1,62 @@
+using BenchmarkTools
+using PythonCall
+using PythonCall: pydel!, pyimport, pydict, pystr, pyrange
+
+const SUITE = BenchmarkGroup()
+
+function test_pydict_init()
+    random = pyimport("random").random
+    x = pydict()
+    for i in pyrange(1000)
+        x[pystr(i)] = i + random()
+    end
+    return x
+end
+
+SUITE["basic"]["julia"]["pydict"]["init"] = @benchmarkable test_pydict_init()
+
+function test_pydict_pydel()
+    random = pyimport("random").random
+    x = pydict()
+    for i in pyrange(1000)
+        k = pystr(i)
+        r = random()
+        v = i + r
+        x[k] = v
+        pydel!(k)
+        pydel!(r)
+        pydel!(v)
+        pydel!(i)
+    end
+    return x
+end
+
+SUITE["basic"]["julia"]["pydict"]["pydel"] = @benchmarkable test_pydict_pydel()
+
+@generated function test_atpy(::Val{use_pydel}) where {use_pydel}
+    quote
+        @py begin
+            import random: random
+            x = {}
+            for i in range(1000)
+                x[str(i)] = i + random()
+                $(use_pydel ? :(@jl PythonCall.pydel!(i)) : :(nothing))
+            end
+            x
+        end
+    end
+end
+
+SUITE["basic"]["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false))
+SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true))
+
+
+include("gcbench.jl")
+using .GCBench: append_lots
+
+SUITE["gc"]["full"] = @benchmarkable(
+    GC.gc(true),
+    setup=(GC.gc(true); append_lots(size=159)),
+    seconds=30,
+    evals=1,
+)

benchmark/gcbench.jl

@@ -0,0 +1,13 @@
+module GCBench
+
+using PythonCall
+
+function append_lots(; iters=100 * 1024, size=1596)
+    v = pylist()
+    for i = 1:iters
+        v.append(pylist(rand(size)))
+    end
+    return v
+end
+
+end