remove Pkg.add from Benchmark (#711)

* remove Pkg.add from Benchmark * update * use median * disable retune * increase time tolerance * robustify * try forcing tuning * only run benchmark ci manually
JuliaStats · Mar 5, 2024 · cfd3023 · cfd3023
1 parent ff82b8f
commit cfd3023
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 17 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -3,16 +3,17 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 on:
-  pull_request:
-    branches:
-      - main
-    paths-ignore:
-      - 'LICENSE.md'
-      - 'README.md'
-      - 'docs/**'
-      - 'issues/**'
-      - 'format/**'
-      - 'test/**'
+    workflow_dispatch:
+#   pull_request:
+#     branches:
+#       - main
+#     paths-ignore:
+#       - 'LICENSE.md'
+#       - 'README.md'
+#       - 'docs/**'
+#       - 'issues/**'
+#       - 'format/**'
+#       - 'test/**'
 jobs:
     benchmarks:
         runs-on: ubuntu-latest

diff --git a/benchmark/Project.toml b/benchmark/Project.toml
@@ -2,7 +2,7 @@
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 MixedModels = "ff71e718-51f3-5ec2-a782-8ffcbfa3c316"
 PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
-StatsModels = "3eaba693-59b7-5ba5-a881-562e759f1c8d"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [compat]
 BenchmarkTools = "1"

diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
@@ -1,4 +1,4 @@
-using BenchmarkTools, MixedModels, StatsModels
+using BenchmarkTools, MixedModels
 using MixedModels: dataset
 
 const SUITE = BenchmarkGroup()
@@ -85,7 +85,7 @@ function fitbobyqa(dsnm::Symbol, i::Integer)
     return fit(MixedModel, fms[dsnm][i], dataset(dsnm); contrasts, progress=false)
 end
 
-# these tests are so fast that they can be very noisy because the denominator is so small, 
+# these tests are so fast that they can be very noisy because the denominator is so small,
 # so we disable them by default for auto-benchmarking
 # SUITE["simplescalar"] = BenchmarkGroup(["single", "simple", "scalar"])
 # for (ds, i) in [

diff --git a/benchmark/run.jl b/benchmark/run.jl
@@ -1,9 +1,10 @@
 using Pkg
 Pkg.develop(PackageSpec(path=dirname(@__DIR__)))
 Pkg.instantiate()
-using PkgBenchmark, MixedModels
-# explicit `Pkg.add` is a crutch until we've got a good base on main
+using PkgBenchmark, MixedModels, Statistics
 # Pkg.update() allows us to benchmark even when dependencies/compat requirements change
-juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update(); Pkg.add([\"BenchmarkTools\", \"StatsModels\"])"`
+juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"`
 config = BenchmarkConfig(; id="origin/HEAD", juliacmd)
-export_markdown("benchmark.md", judge(MixedModels, config; verbose=true))
+# for many of the smaller models, we get a lot of noise at the default 5% tolerance
+# TODO: specify a tune.json with per model time tolerances
+export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=median, judgekwargs=(;time_tolerance=0.1, memory_tolerance=0.05)))
diff --git a/benchmark/tune.json b/benchmark/tune.json
@@ -0,0 +1 @@
+[{"Julia":"1.9.2","BenchmarkTools":"1.0.0"},[["BenchmarkGroup",{"data":{"singlevector":["BenchmarkGroup",{"data":{"sleepstudy:3":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"sleepstudy:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"sleepstudy:4":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["single","vector"]}],"crossed":["BenchmarkGroup",{"data":{"kb07:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"ml1m:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"insteval:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"insteval:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"machines:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"mrk17_exp1:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"penicillin:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","crossed","scalar"]}],"crossedvector":["BenchmarkGroup",{"data":{"kb07:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"d3:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"kb07:3":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"mrk17_exp1:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","crossed","vector"]}],"nested":["BenchmarkGroup",{"data":{"pastes:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","nested","scalar"]}]},"tags":[]}]]]