Merge pull request #44 from MilesCranmer/defaults

Nicer default behavior
MilesCranmer · Jun 18, 2024 · e36d467 · e36d467 · MilesCranmer · Jun 18, 2024
2 parents f27b6bc + 5904de7
commit e36d467
Show file tree

Hide file tree

Showing 9 changed files with 525 additions and 334 deletions.
diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "AirspeedVelocity"
 uuid = "1c8270ee-6884-45cc-9545-60fa71ec23e4"
 authors = ["Miles Cranmer <miles.cranmer@gmail.com>"]
-version = "0.5.3"
+version = "0.6.0"
 
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
@@ -16,6 +16,7 @@ PlotlyLight = "ca7969ec-10b3-423e-8d99-40f33abb42bf"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 
 [compat]
 BenchmarkTools = "1"
@@ -24,7 +25,10 @@ DispatchDoctor = "0.4"
 FilePathsBase = "0.9"
 JSON3 = "1"
 OrderedCollections = "1"
+Pkg = "1"
 PlotlyKaleido = "2"
 PlotlyLight = "0.6"
 REPL = "1"
+Statistics = "1"
+TOML = "1"
 julia = "1.8"
diff --git a/README.md b/README.md
@@ -30,21 +30,32 @@ This will install two executables at `~/.julia/bin` - make sure to have it on yo
 
 ## Examples
 
-You may then use the CLI to generate benchmarks for any package with, e.g.,
+You may use the CLI to generate benchmarks for any package with, e.g.,
+
+```bash
+benchpkg
+```
+
+This will benchmark the package defined in the current directory
+at the current dirty state, against the default branch (i.e., `main` or `master`),
+over all benchmarks defined in `benchmark/benchmarks.jl`. It will then print
+a markdown table of the results while also saving the JSON results to the current directory.
+
+
+You can configure all options with the CLI flags. For example, to benchmark
+the registered package `Transducers.jl` at the revisions `v0.4.20`, `v0.4.70`, and `master`,
+you can use:
 
 ```bash
 benchpkg Transducers \
     --rev=v0.4.20,v0.4.70,master \
     --bench-on=v0.4.20
 ```
 
-which will benchmark `Transducers.jl`,
-at the revisions `v0.4.20`, `v0.4.70`, and `master`,
-using the benchmark script `benchmark/benchmarks.jl` as it was defined at `v0.4.20`,
+This will further use the benchmark script `benchmark/benchmarks.jl` as it was defined at `v0.4.20`,
 and then save the JSON results in the current directory.
 
-We can view the results of the benchmark as a table
-with `benchpkgtable`:
+We can explicitly view the results of the benchmark as a table with `benchpkgtable`:
 
 ```bash
 benchpkgtable Transducers \
@@ -131,31 +142,35 @@ For running benchmarks, you can use the `benchpkg` command, which is
 built into the `~/.julia/bin` folder:
 
 ```markdown
-    benchpkg package_name [-r --rev <arg>]
-                          [--url <arg>]
-                          [--path <arg>]
-                          [-o, --output-dir <arg>]
-                          [-e, --exeflags <arg>]
-                          [-a, --add <arg>]
-                          [-s, --script <arg>]
-                          [--bench-on <arg>]
-                          [-f, --filter <arg>]
-                          [--nsamples-load-time <arg>]
-                          [--tune]
+    benchpkg [package_name] [-r --rev <arg>]
+                            [--url <arg>]
+                            [--path <arg>]
+                            [-o, --output-dir <arg>]
+                            [-e, --exeflags <arg>]
+                            [-a, --add <arg>]
+                            [-s, --script <arg>]
+                            [--bench-on <arg>]
+                            [-f, --filter <arg>]
+                            [--nsamples-load-time <arg>]
+                            [--tune]
+                            [--dont-print]
 
 Benchmark a package over a set of revisions.
 
 # Arguments
 
-- `package_name`: Name of the package.
+- `package_name`: Name of the package. If not given, the package is assumed to be
+  the current directory.
 
 # Options
 
 - `-r, --rev <arg>`: Revisions to test (delimit by comma). Use `dirty` to
   benchmark the current state of the package at `path` (and not a git commit).
+  The default is `{DEFAULT},dirty`, which will attempt to find the default branch
+  of the package.
 - `--url <arg>`: URL of the package.
-- `--path <arg>`: Path of the package.
-- `-o, --output-dir <arg>`: Where to save the JSON results.
+- `--path <arg>`: Path of the package. The default is `.` if other arguments are not given.
+- `-o, --output-dir <arg>`: Where to save the JSON results. The default is `.`.
 - `-e, --exeflags <arg>`: CLI flags for Julia (default: none).
 - `-a, --add <arg>`: Extra packages needed (delimit by comma).
 - `-s, --script <arg>`: The benchmark script. Default: `benchmark/benchmarks.jl` downloaded from `stable`.
@@ -164,17 +179,22 @@ Benchmark a package over a set of revisions.
 - `-f, --filter <arg>`: Filter the benchmarks to run (delimit by comma).
 - `--nsamples-load-time <arg>`: Number of samples to take when measuring load time of
     the package (default: 5). (This means starting a Julia process for each sample.)
+- `--dont-print`: Don't print the table.
 
 # Flags
 
-- `--tune`: Whether to run benchmarks with tuning (default: false). 
+- `--tune`: Whether to run benchmarks with tuning (default: false).
 ```
 
 You can also just generate a table:
 
 ```markdown
-    benchpkgtable package_name [-r --rev <arg>] [-i --input-dir <arg>]
-                               [--ratio]
+    benchpkgtable [package_name] [-r --rev <arg>]
+                                 [-i --input-dir <arg>]
+                                 [--ratio]
+                                 [--mode <arg>]
+                                 [--url <arg>]
+                                 [--path <arg>]
 
 Print a table of the benchmarks of a package as created with `benchpkg`.
 
@@ -185,19 +205,29 @@ Print a table of the benchmarks of a package as created with `benchpkg`.
 # Options
 
 - `-r, --rev <arg>`: Revisions to test (delimit by comma).
+  The default is `{DEFAULT},dirty`, which will attempt to find the default branch
+  of the package.
 - `-i, --input-dir <arg>`: Where the JSON results were saved (default: ".").
+- `--url <arg>`: URL of the package. Only used to get the package name.
+- `--path <arg>`: Path of the package. The default is `.` if other arguments are not given.
+   Only used to get the package name.
 
 # Flags
 
 - `--ratio`: Whether to include the ratio (default: false). Only applies when
     comparing two revisions.
+- `--mode`: Table mode(s). Valid values are "time" (default), to print the
+    benchmark time, or "memory", to print the allocation and memory usage.
+    Both options can be passed, if delimited by comma.
 ```
 
 For plotting, you can use the `benchpkgplot` function:
 
 ```markdown
-    benchpkgplot package_name [-r --rev <arg>] [-i --input-dir <arg>]
-                              [-o --output-dir <arg>] [-n --npart <arg>]
+    benchpkgplot package_name [-r --rev <arg>]
+                              [-i --input-dir <arg>]
+                              [-o --output-dir <arg>]
+                              [-n --npart <arg>]
                               [--format <arg>]
 
 Plot the benchmarks of a package as created with `benchpkg`.

diff --git a/src/BenchPkg.jl b/src/BenchPkg.jl
@@ -1,35 +1,40 @@
 module BenchPkg
 
-using ..Utils: benchmark
+using ..TableUtils: create_table, format_memory
+using ..Utils: benchmark, get_package_name_defaults, parse_rev, load_results
 using Comonicon
 using Comonicon: @main
 
 """
-    benchpkg package_name [-r --rev <arg>]
-                          [--url <arg>]
-                          [--path <arg>]
-                          [-o, --output-dir <arg>]
-                          [-e, --exeflags <arg>]
-                          [-a, --add <arg>]
-                          [-s, --script <arg>]
-                          [--bench-on <arg>]
-                          [-f, --filter <arg>]
-                          [--nsamples-load-time <arg>]
-                          [--tune]
+    benchpkg [package_name] [-r --rev <arg>]
+                            [--url <arg>]
+                            [--path <arg>]
+                            [-o, --output-dir <arg>]
+                            [-e, --exeflags <arg>]
+                            [-a, --add <arg>]
+                            [-s, --script <arg>]
+                            [--bench-on <arg>]
+                            [-f, --filter <arg>]
+                            [--nsamples-load-time <arg>]
+                            [--tune]
+                            [--dont-print]
 
 Benchmark a package over a set of revisions.
 
 # Arguments
 
-- `package_name`: Name of the package.
+- `package_name`: Name of the package. If not given, the package is assumed to be
+  the current directory.
 
 # Options
 
 - `-r, --rev <arg>`: Revisions to test (delimit by comma). Use `dirty` to
   benchmark the current state of the package at `path` (and not a git commit).
+  The default is `{DEFAULT},dirty`, which will attempt to find the default branch
+  of the package.
 - `--url <arg>`: URL of the package.
-- `--path <arg>`: Path of the package.
-- `-o, --output-dir <arg>`: Where to save the JSON results.
+- `--path <arg>`: Path of the package. The default is `.` if other arguments are not given.
+- `-o, --output-dir <arg>`: Where to save the JSON results. The default is `.`.
 - `-e, --exeflags <arg>`: CLI flags for Julia (default: none).
 - `-a, --add <arg>`: Extra packages needed (delimit by comma).
 - `-s, --script <arg>`: The benchmark script. Default: `benchmark/benchmarks.jl` downloaded from `stable`.
@@ -38,15 +43,16 @@ Benchmark a package over a set of revisions.
 - `-f, --filter <arg>`: Filter the benchmarks to run (delimit by comma).
 - `--nsamples-load-time <arg>`: Number of samples to take when measuring load time of
     the package (default: 5). (This means starting a Julia process for each sample.)
+- `--dont-print`: Don't print the table.
 
 # Flags
 
 - `--tune`: Whether to run benchmarks with tuning (default: false).
 
 """
 @main function benchpkg(
-    package_name::String;
-    rev::String,
+    package_name::String="";
+    rev::String="{DEFAULT},dirty",
     output_dir::String=".",
     script::String="",
     exeflags::String="",
@@ -57,21 +63,39 @@ Benchmark a package over a set of revisions.
     bench_on::String="",
     filter::String="",
     nsamples_load_time::Int=5,
+    dont_print::Bool=false,
 )
     revs = convert(Vector{String}, split(rev, ","))
-    Base.filter!(x -> length(x) > 0, revs)
+    Base.filter!(!isempty, revs)
 
     filtered = convert(Vector{String}, split(filter, ","))
     Base.filter!(x -> length(x) > 0, filtered)
 
     @assert length(revs) > 0 "No revisions specified."
     @assert nsamples_load_time > 0 "nsamples_load_time must be positive."
 
+    package_name, url, path = get_package_name_defaults(package_name, url, path)
+
+    if path != ""
+        revs = map(Base.Fix2(parse_rev, path), revs)
+    else
+        if any(==("{DEFAULT}"), revs)
+            error("You must explicitly set `--revs` for this set of options.")
+        end
+    end
+
+    _script = if bench_on == "dirty" && path != "" && script == ""
+        bench_on = nothing
+        joinpath(path, "benchmark", "benchmarks.jl")
+    else
+        script
+    end
+
     benchmark(
         package_name,
         revs;
         output_dir=output_dir,
-        script=(length(script) > 0 ? script : nothing),
+        script=(length(_script) > 0 ? _script : nothing),
         tune=tune,
         exeflags=(length(exeflags) > 0 ? `$(Cmd(split(exeflags, " ") .|> String))` : ``),
         extra_pkgs=convert(Vector{String}, split(add, ",")),
@@ -82,6 +106,13 @@ Benchmark a package over a set of revisions.
         nsamples_load_time=nsamples_load_time,
     )
 
+    if !dont_print
+        combined_results = load_results(package_name, revs; input_dir=output_dir)
+        println(
+            create_table(combined_results; add_ratio_col=length(revs) == 2, key="median")
+        )
+    end
+
     return nothing
 end
 

diff --git a/src/BenchPkgTable.jl b/src/BenchPkgTable.jl
@@ -1,13 +1,17 @@
 module BenchPkgTable
 
 using ..TableUtils: create_table, format_memory
-using ..Utils: load_results
+using ..Utils: get_package_name_defaults, parse_rev, load_results
 using Comonicon
 using Comonicon: @main
 
 """
-    benchpkgtable package_name [-r --rev <arg>] [-i --input-dir <arg>]
-                               [--ratio] [--mode <arg>]
+    benchpkgtable [package_name] [-r --rev <arg>]
+                                 [-i --input-dir <arg>]
+                                 [--ratio]
+                                 [--mode <arg>]
+                                 [--url <arg>]
+                                 [--path <arg>]
 
 Print a table of the benchmarks of a package as created with `benchpkg`.
 
@@ -18,7 +22,12 @@ Print a table of the benchmarks of a package as created with `benchpkg`.
 # Options
 
 - `-r, --rev <arg>`: Revisions to test (delimit by comma).
+  The default is `{DEFAULT},dirty`, which will attempt to find the default branch
+  of the package.
 - `-i, --input-dir <arg>`: Where the JSON results were saved (default: ".").
+- `--url <arg>`: URL of the package. Only used to get the package name.
+- `--path <arg>`: Path of the package. The default is `.` if other arguments are not given.
+   Only used to get the package name.
 
 # Flags
 
@@ -29,16 +38,29 @@ Print a table of the benchmarks of a package as created with `benchpkg`.
     Both options can be passed, if delimited by comma.
 """
 @main function benchpkgtable(
-    package_name::String;
-    rev::String,
+    package_name::String="";
+    rev::String="dirty,{DEFAULT}",
     input_dir::String=".",
     ratio::Bool=false,
     mode::String="time",
+    url::String="",
+    path::String="",
 )
     revs = convert(Vector{String}, split(rev, ","))
-    # Filter empty strings:
-    revs = filter(x -> length(x) > 0, revs)
+    Base.filter!(!isempty, revs)
+
     @assert length(revs) > 0 "No revisions specified."
+
+    package_name, url, path = get_package_name_defaults(package_name, url, path)
+
+    if path != ""
+        revs = map(Base.Fix2(parse_rev, path), revs)
+    else
+        if any(==("{DEFAULT}"), revs)
+            error("You must explicitly set `--revs` for this set of options.")
+        end
+    end
+
     combined_results = load_results(package_name, revs; input_dir=input_dir)
 
     modes = split(mode, ",")