Goal
Compare yaml12 and yaml for reading and
writing YAML over a range of document sizes.
We intentionally avoid testing semantic differences between YAML 1.1 and 1.2, as well as less-used YAML features like tags and streams. Instead, we focus on the simple default usage of both.
Operations:
- read: YAML file → R object
-
write: R object → YAML (written to
nullfile())
Setup
Generate inputs
To benchmark, we create YAML files consisting of a repeating sequence
of a small, fixed “mixed” node, repeated 2^(1:15) times.
This node is designed to exercise the full YAML 1.2 core schema (every
core type is represented). We generate these files with
yaml12::write_yaml().
Each file is written to a temporary file and then we measure read
time. Note that on macOS and Linux, tempfile() paths often
live on a tmpfs in RAM, so this typically won’t trigger a
write to disk under most circumstances.
# A YAML node that exercises every core schema type:
# seq, map, str, bool, int, float, null
mixed_node <- list(
str = c(
"Lorem ipsum dolor sit amet, vel accumsan vitae faucibus ultrices leo",
"neque? Et cursus lacinia, ut, sit donec facilisi eu interdum. Dui",
"ipsum, vitae ligula commodo convallis ac sed nunc. Ipsum at nec lacus",
"eros suscipit vitae."
),
block_str = "lorem \n ipsum \n dolor\n",
bools = c(TRUE, FALSE),
ints = c(123L, -123L),
floats = c(123.456, -123.456),
null = NULL
)
make_yaml_doc <- function(size = 1) {
path <- tempfile(fileext = ".yaml")
yaml12::write_yaml(rep(list(mixed_node), size), path)
path
}
docs <- sapply(2^(1:15), make_yaml_doc)Read performance
read_results <- lapply(docs, function(doc) {
result <- bench::mark(
yaml12::read_yaml(doc),
yaml::read_yaml(doc),
check = FALSE
)
result$file_size <- file.info(doc)$size
result
})
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.Read results summary
read_results_df <- bind_rows(read_results) |>
mutate(expression = as.factor(sapply(expression, deparse1)))
read_results_df |>
ggplot(aes(x = file_size, y = median, color = expression)) +
labs(
y = "Median Run Time",
x = "File Size",
title = "Read Performance"
) +
geom_point() +
geom_smooth(linewidth = .5) +
scale_x_log10(labels = scales::label_bytes()) +
scale_y_continuous(trans = bench::bench_time_trans())
#> `geom_smooth()` using method = 'loess' and formula = 'y ~ x'
Read results (detailed)
invisible(lapply(read_results, \(result) {
print(plot(result) + ggplot2::ggtitle(scales::label_bytes()(result$file_size[1])))
print(summary(result, filter_gc = FALSE))
}))
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <dbl> <bch:t> <bch:t> <dbl> <bch:byt> <dbl>
#> 1 yaml12::read_… 840 32.1µs 33.3µs 28742. 2.75KB 0
#> 2 yaml::read_ya… 840 102.4µs 116.3µs 8225. 34.78KB 4.00
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <dbl> <bch:t> <bch:t> <dbl> <bch:byt> <dbl>
#> 1 yaml12::read_… 1676 56.7µs 59.8µs 16304. 0B 0
#> 2 yaml::read_ya… 1676 147.3µs 153.7µs 6220. 10.8KB 4.00
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <dbl> <bch> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::re… 3348 108µs 113µs 8475. 0B 2.00 4236
#> 2 yaml::read… 3348 237µs 245µs 3947. 13.7KB 4.00 1974
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <dbl> <bch> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::re… 6692 215µs 223µs 4352. 0B 2.00 2176
#> 2 yaml::read… 6692 416µs 438µs 2243. 19.3KB 4.00 1122
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <dbl> <bch> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::re… 13380 417µs 447µs 2225. 304B 0 1113
#> 2 yaml::read… 13380 780µs 814µs 1199. 30.9KB 4.00 600
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <dbl> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 yaml12::rea… 26756 859.64µs 886.16µs 1112. 560B 2.00
#> 2 yaml::read_… 26756 1.54ms 1.58ms 618. 69.4KB 4.00
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <dbl> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::r… 53508 1.75ms 1.81ms 553. 1.05KB 0 277
#> 2 yaml::rea… 53508 3.13ms 3.19ms 306. 146.29KB 3.99 153
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <dbl> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::r… 107012 3.42ms 3.49ms 283. 2.05KB 1.99 142
#> 2 yaml::rea… 107012 6.74ms 6.83ms 144. 300.09KB 1.98 73
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <dbl> <bch:t> <bch:> <dbl> <bch:byt> <dbl>
#> 1 yaml12::read_y… 214020 6.54ms 6.7ms 149. 4.05KB 0
#> 2 yaml::read_yam… 214020 15.99ms 16.3ms 60.0 607.63KB 3.87
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <dbl> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::r… 428036 12.8ms 13ms 76.0 8.05KB 2.00 38
#> 2 yaml::rea… 428036 42.5ms 43ms 23.0 1.19MB 1.92 12
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <dbl> <bch:t> <bch:t> <dbl> <bch:byt> <dbl>
#> 1 yaml12::read_… 856068 25.4ms 25.8ms 38.0 16KB 2.00
#> 2 yaml::read_ya… 856068 142.7ms 149.3ms 6.71 2.4MB 1.68
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <dbl> <bch:t> <bch:t> <dbl> <bch:byt> <dbl>
#> 1 yaml12::read_… 1712132 53.1ms 53.8ms 18.2 32KB 1.82
#> 2 yaml::read_ya… 1712132 887.6ms 887.6ms 1.13 4.8MB 0
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <dbl> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 yaml12::rea… 3424260 106.29ms 109.49ms 9.13 64KB 0
#> 2 yaml::read_… 3424260 4.28s 4.28s 0.234 9.6MB 0.234
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <dbl> <bch> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::re… 6848516 212ms 218ms 4.59 128KB 1.53 3
#> 2 yaml::read… 6848516 18s 18s 0.0556 19.2MB 0.223 1
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <dbl> <bch:t> <bch:t> <dbl> <bch:byt> <dbl>
#> 1 yaml12::read_… 13697028 431.7ms 462.8ms 2.16 256KB 3.24
#> 2 yaml::read_ya… 13697028 57.7s 57.7s 0.0173 38.4MB 0.0867
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>Write performance
objs <- lapply(2^(1:15), function(n) rep(list(mixed_node), n))
write_results <- lapply(objs, function(obj) {
result <- bench::mark(
yaml12::write_yaml(obj, nullfile()),
yaml::write_yaml(obj, nullfile()),
check = FALSE
)
result$obj_size <- object.size(obj)
result
})
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.Write results summary:
write_results_df <- bind_rows(write_results) |>
mutate(expression = as.factor(sapply(expression, deparse1)))
write_results_df |>
ggplot(aes(x = obj_size, y = median, color = expression)) +
labs(
y = "Median Run Time",
x = "Object Size",
title = "Write Performance"
) +
geom_point() +
geom_smooth(linewidth = .5) +
scale_x_log10(labels = scales::label_bytes()) +
scale_y_continuous(trans = bench::bench_time_trans())
#> `geom_smooth()` using method = 'loess' and formula = 'y ~ x'
Write results (detailed)
invisible(lapply(write_results, \(result) {
print(plot(result) + ggplot2::ggtitle(scales::label_bytes()(as.numeric(result$obj_size[1]))))
print(summary(result, filter_gc = FALSE))
}))
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wr… 3360 by… 18µs 18.5µs 51478. 3.02KB 5.15 10000
#> 2 yaml::writ… 3360 by… 68.6µs 72.8µs 12716. 33.68KB 4.00 6355
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wr… 6672 by… 30.1µs 30.8µs 31166. 0B 3.12 10000
#> 2 yaml::writ… 6672 by… 90.8µs 96.8µs 9310. 1.56KB 4.00 4654
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <objct_> <bch:t> <bch:t> <dbl> <bch:byt> <dbl>
#> 1 yaml12::write_… 13296 b… 53.9µs 54.9µs 17816. 0B 0
#> 2 yaml::write_ya… 13296 b… 135.5µs 146.4µs 6271. 3.07KB 6.00
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wri… 26544 b… 101µs 103µs 9458. 0B 0 4728
#> 2 yaml::write… 26544 b… 224µs 238µs 3889. 6.09KB 5.99 1949
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wri… 53040 b… 198µs 204µs 4821. 0B 0 2411
#> 2 yaml::write… 53040 b… 411µs 443µs 2133. 12.1KB 4.00 1067
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wri… 106032 … 392µs 406µs 2440. 0B 0 1220
#> 2 yaml::write… 106032 … 786µs 828µs 1118. 24.2KB 6.00 559
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <objct_> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 yaml12::writ… 212016 … 793.51µs 814.61µs 1223. 0B 0
#> 2 yaml::write_… 212016 … 1.53ms 1.58ms 596. 48.3KB 4.00
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wr… 423984 … 1.56ms 1.58ms 628. 0B 0 314
#> 2 yaml::writ… 423984 … 3.03ms 3.11ms 299. 96.6KB 5.98 150
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wr… 847920 … 3.07ms 3.1ms 322. 0B 0 161
#> 2 yaml::writ… 847920 … 6.01ms 6.14ms 152. 193KB 5.99 76
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <objct_> <bch:t> <bch:t> <dbl> <bch:byt> <dbl>
#> 1 yaml12::write_… 1695792… 6.08ms 6.12ms 162. 0B 0
#> 2 yaml::write_ya… 1695792… 12ms 12.12ms 47.2 386KB 3.94
#> # ℹ 7 more variables: n_itr <int>, n_gc <dbl>, total_time <bch:tm>,
#> # result <list>, memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wr… 3391536… 12.3ms 12.4ms 80.2 0B 0 41
#> 2 yaml::writ… 3391536… 23.6ms 24ms 40.8 772KB 3.89 21
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wr… 6783024… 24.8ms 25.6ms 38.3 0B 0 20
#> 2 yaml::writ… 6783024… 47.4ms 48.4ms 20.4 1.51MB 5.56 11
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wr… 1356600… 54.6ms 55ms 18.1 0B 0 10
#> 2 yaml::writ… 1356600… 95.3ms 98.8ms 10.1 3.02MB 5.07 6
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wri… 2713195… 108ms 109ms 9.16 0B 0 5
#> 2 yaml::write… 2713195… 192ms 194ms 5.16 6.03MB 3.44 3
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <objct_> <bch> <bch:> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::wri… 5426385… 211ms 211ms 4.73 0B 0 3
#> 2 yaml::write… 5426385… 390ms 478ms 2.09 12.1MB 4.18 2
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>