Implement entropy hashing with AC (optimised) (#15)

Extending my previous proof of concept work, I parametrized the inner
and outer models.
Then I implemented search for best CR and speed (when caching).
Used rayon to parallelize some of the work (compression still happens
single threaded, we just search on multiple threads).

I have also included logs for some of the searches (run on my MacBook
Air M1 with 16GB RAM).
I should probably make some cool graphs out of it.

Cargo.toml

@@ -13,6 +13,7 @@ default-run = "weath3rb0i"
 
 [dependencies]
 debug_unreachable = "0.1"
+rayon = "1.9.0"
 
 [profile.dev]
 opt-level = 1
@@ -34,12 +35,13 @@ unsafe_conversions = []
 
 [[bin]]
 name = "order0"
-
 [[bin]]
 name = "ac-over-huffman"
-
 [[bin]]
 name = "cmp"
-
 [[bin]]
 name = "ordern"
+[[bin]]
+name = "entropy-hashing-ac"
+[[bin]]
+name = "entropy-hashing-ac-cached"

src/bin/ac-over-huffman/main.rs

@@ -66,7 +66,7 @@ fn exec(buf: &[u8], huffman_size: u8, ctx_bits: u8, alignment_bits: u8) -> Resul
 
     let time = timer.elapsed();
     println!(
-        "[ac-over-huff] [hsize: {:2} ctx: {:2} align: {}] csize: {} (ratio: {:.3}), ctime: {:?} ({:?} per bit)",
+        "[ac-over-huff] [hsize: {:2}, ctx: {:2}, align: {}] csize: {} (ratio: {:.3}), ctime: {:?} ({:?} per bit)",
         huffman_size,
         ctx_bits,
         alignment_bits,

src/bin/entropy-hashing-ac-cached/enwik7.mc.3.log

@@ -0,0 +1,153 @@
+[eh-ac] [ctx:  8, align: 3 cache:  8] csize: 6316077 (ratio 0.632), ctime: 7.794461s (97ns per bit)
+[eh-ac] [ctx:  8, align: 3 cache:  0] csize: 6316077 (ratio 0.632), ctime: 8.669372167s (108ns per bit)
+[eh-ac] [ctx:  8, align: 3 cache: 12] csize: 6316077 (ratio 0.632), ctime: 8.851632541s (111ns per bit)
+[eh-ac] [ctx:  8, align: 3 cache:  4] csize: 6316077 (ratio 0.632), ctime: 9.252850375s (116ns per bit)
+[eh-ac] [ctx:  8, align: 3 cache: 16] csize: 6316077 (ratio 0.632), ctime: 9.53347325s (119ns per bit)
+[eh-ac] [ctx:  8, align: 3 cache: 20] csize: 6316077 (ratio 0.632), ctime: 10.125922333s (127ns per bit)
+[eh-ac] [ctx:  8, align: 3 cache: 24] csize: 6316077 (ratio 0.632), ctime: 10.284911s (129ns per bit)
+-> fastest: 6316077 in 7.794461s (97ns per bit) for [ctx: 8, align: 3, cache: 8]
+[eh-ac] [ctx:  9, align: 3 cache:  8] csize: 5967872 (ratio 0.597), ctime: 8.607036292s (108ns per bit)
+[eh-ac] [ctx:  9, align: 3 cache: 12] csize: 5967872 (ratio 0.597), ctime: 8.954403166s (112ns per bit)
+[eh-ac] [ctx:  9, align: 3 cache: 16] csize: 5967872 (ratio 0.597), ctime: 9.818092084s (123ns per bit)
+[eh-ac] [ctx:  9, align: 3 cache: 20] csize: 5967872 (ratio 0.597), ctime: 9.906964s (124ns per bit)
+[eh-ac] [ctx:  9, align: 3 cache:  0] csize: 5967872 (ratio 0.597), ctime: 9.945267584s (124ns per bit)
+[eh-ac] [ctx:  9, align: 3 cache:  4] csize: 5967872 (ratio 0.597), ctime: 10.447750416s (131ns per bit)
+[eh-ac] [ctx:  9, align: 3 cache: 24] csize: 5967872 (ratio 0.597), ctime: 10.683410916s (134ns per bit)
+-> fastest: 5967872 in 8.607036292s (108ns per bit) for [ctx: 9, align: 3, cache: 8]
+[eh-ac] [ctx: 10, align: 3 cache: 12] csize: 5604938 (ratio 0.560), ctime: 8.709490583s (109ns per bit)
+[eh-ac] [ctx: 10, align: 3 cache:  8] csize: 5604938 (ratio 0.560), ctime: 9.362230542s (117ns per bit)
+[eh-ac] [ctx: 10, align: 3 cache: 20] csize: 5604938 (ratio 0.560), ctime: 9.919285541s (124ns per bit)
+[eh-ac] [ctx: 10, align: 3 cache: 16] csize: 5604938 (ratio 0.560), ctime: 9.986881208s (125ns per bit)
+[eh-ac] [ctx: 10, align: 3 cache: 24] csize: 5604938 (ratio 0.560), ctime: 10.448618583s (131ns per bit)
+[eh-ac] [ctx: 10, align: 3 cache:  0] csize: 5604938 (ratio 0.560), ctime: 10.947612459s (137ns per bit)
+[eh-ac] [ctx: 10, align: 3 cache:  4] csize: 5604938 (ratio 0.560), ctime: 11.380833916s (142ns per bit)
+-> fastest: 5604938 in 8.709490583s (109ns per bit) for [ctx: 10, align: 3, cache: 12]
+[eh-ac] [ctx: 11, align: 3 cache: 12] csize: 5275208 (ratio 0.528), ctime: 9.0397715s (113ns per bit)
+[eh-ac] [ctx: 11, align: 3 cache: 16] csize: 5275208 (ratio 0.528), ctime: 10.586172917s (132ns per bit)
+[eh-ac] [ctx: 11, align: 3 cache:  8] csize: 5275208 (ratio 0.528), ctime: 10.588053125s (132ns per bit)
+[eh-ac] [ctx: 11, align: 3 cache: 20] csize: 5275208 (ratio 0.528), ctime: 11.267277375s (141ns per bit)
+[eh-ac] [ctx: 11, align: 3 cache: 24] csize: 5275208 (ratio 0.528), ctime: 11.267270709s (141ns per bit)
+[eh-ac] [ctx: 11, align: 3 cache:  0] csize: 5275208 (ratio 0.528), ctime: 12.404147875s (155ns per bit)
+[eh-ac] [ctx: 11, align: 3 cache:  4] csize: 5275208 (ratio 0.528), ctime: 12.735387584s (159ns per bit)
+-> fastest: 5275208 in 9.0397715s (113ns per bit) for [ctx: 11, align: 3, cache: 12]
+[eh-ac] [ctx: 12, align: 3 cache: 12] csize: 4969311 (ratio 0.497), ctime: 9.868507416s (123ns per bit)
+[eh-ac] [ctx: 12, align: 3 cache: 16] csize: 4969311 (ratio 0.497), ctime: 11.256680459s (141ns per bit)
+[eh-ac] [ctx: 12, align: 3 cache: 24] csize: 4969311 (ratio 0.497), ctime: 12.035754417s (150ns per bit)
+[eh-ac] [ctx: 12, align: 3 cache:  8] csize: 4969311 (ratio 0.497), ctime: 12.193016542s (152ns per bit)
+[eh-ac] [ctx: 12, align: 3 cache: 20] csize: 4969311 (ratio 0.497), ctime: 12.24817475s (153ns per bit)
+[eh-ac] [ctx: 12, align: 3 cache:  0] csize: 4969311 (ratio 0.497), ctime: 14.15030075s (177ns per bit)
+[eh-ac] [ctx: 12, align: 3 cache:  4] csize: 4969311 (ratio 0.497), ctime: 14.460046084s (181ns per bit)
+-> fastest: 4969311 in 9.868507416s (123ns per bit) for [ctx: 12, align: 3, cache: 12]
+[eh-ac] [ctx: 13, align: 3 cache: 16] csize: 4684310 (ratio 0.468), ctime: 10.676325708s (133ns per bit)
+[eh-ac] [ctx: 13, align: 3 cache: 12] csize: 4684310 (ratio 0.468), ctime: 11.195165291s (140ns per bit)
+[eh-ac] [ctx: 13, align: 3 cache: 24] csize: 4684310 (ratio 0.468), ctime: 12.804063458s (160ns per bit)
+[eh-ac] [ctx: 13, align: 3 cache: 20] csize: 4684310 (ratio 0.468), ctime: 12.806164791s (160ns per bit)
+[eh-ac] [ctx: 13, align: 3 cache:  8] csize: 4684310 (ratio 0.468), ctime: 13.790006625s (172ns per bit)
+[eh-ac] [ctx: 13, align: 3 cache:  0] csize: 4684310 (ratio 0.468), ctime: 15.956356208s (199ns per bit)
+[eh-ac] [ctx: 13, align: 3 cache:  4] csize: 4684310 (ratio 0.468), ctime: 16.300937333s (204ns per bit)
+-> fastest: 4684310 in 10.676325708s (133ns per bit) for [ctx: 13, align: 3, cache: 16]
+[eh-ac] [ctx: 14, align: 3 cache: 16] csize: 4457876 (ratio 0.446), ctime: 11.029500542s (138ns per bit)
+[eh-ac] [ctx: 14, align: 3 cache: 12] csize: 4457876 (ratio 0.446), ctime: 12.7368895s (159ns per bit)
+[eh-ac] [ctx: 14, align: 3 cache: 20] csize: 4457876 (ratio 0.446), ctime: 13.320213208s (167ns per bit)
+[eh-ac] [ctx: 14, align: 3 cache: 24] csize: 4457876 (ratio 0.446), ctime: 14.6594735s (183ns per bit)
+[eh-ac] [ctx: 14, align: 3 cache:  8] csize: 4457876 (ratio 0.446), ctime: 15.533021s (194ns per bit)
+[eh-ac] [ctx: 14, align: 3 cache:  0] csize: 4457876 (ratio 0.446), ctime: 17.588352292s (220ns per bit)
+[eh-ac] [ctx: 14, align: 3 cache:  4] csize: 4457876 (ratio 0.446), ctime: 17.897660542s (224ns per bit)
+-> fastest: 4457876 in 11.029500542s (138ns per bit) for [ctx: 14, align: 3, cache: 16]
+[eh-ac] [ctx: 15, align: 3 cache: 16] csize: 4241788 (ratio 0.424), ctime: 12.082824167s (151ns per bit)
+[eh-ac] [ctx: 15, align: 3 cache: 20] csize: 4241788 (ratio 0.424), ctime: 13.444948625s (168ns per bit)
+[eh-ac] [ctx: 15, align: 3 cache: 12] csize: 4241788 (ratio 0.424), ctime: 14.299350416s (179ns per bit)
+[eh-ac] [ctx: 15, align: 3 cache: 24] csize: 4241788 (ratio 0.424), ctime: 15.382663416s (192ns per bit)
+[eh-ac] [ctx: 15, align: 3 cache:  8] csize: 4241788 (ratio 0.424), ctime: 17.200661167s (215ns per bit)
+[eh-ac] [ctx: 15, align: 3 cache:  0] csize: 4241788 (ratio 0.424), ctime: 19.228871167s (240ns per bit)
+[eh-ac] [ctx: 15, align: 3 cache:  4] csize: 4241788 (ratio 0.424), ctime: 19.571118917s (245ns per bit)
+-> fastest: 4241788 in 12.082824167s (151ns per bit) for [ctx: 15, align: 3, cache: 16]
+[eh-ac] [ctx: 16, align: 3 cache: 20] csize: 4041879 (ratio 0.404), ctime: 13.240143875s (166ns per bit)
+[eh-ac] [ctx: 16, align: 3 cache: 16] csize: 4041879 (ratio 0.404), ctime: 13.315919208s (166ns per bit)
+[eh-ac] [ctx: 16, align: 3 cache: 12] csize: 4041879 (ratio 0.404), ctime: 15.942645667s (199ns per bit)
+[eh-ac] [ctx: 16, align: 3 cache: 24] csize: 4041879 (ratio 0.404), ctime: 16.107549875s (201ns per bit)
+[eh-ac] [ctx: 16, align: 3 cache:  8] csize: 4041879 (ratio 0.404), ctime: 18.916207084s (236ns per bit)
+[eh-ac] [ctx: 16, align: 3 cache:  0] csize: 4041879 (ratio 0.404), ctime: 20.988361167s (262ns per bit)
+[eh-ac] [ctx: 16, align: 3 cache:  4] csize: 4041879 (ratio 0.404), ctime: 21.372886542s (267ns per bit)
+-> fastest: 4041879 in 13.240143875s (166ns per bit) for [ctx: 16, align: 3, cache: 20]
+[eh-ac] [ctx: 17, align: 3 cache: 20] csize: 3851248 (ratio 0.385), ctime: 15.055197958s (188ns per bit)
+[eh-ac] [ctx: 17, align: 3 cache: 16] csize: 3851248 (ratio 0.385), ctime: 16.136045375s (202ns per bit)
+[eh-ac] [ctx: 17, align: 3 cache: 24] csize: 3851248 (ratio 0.385), ctime: 17.876767292s (223ns per bit)
+[eh-ac] [ctx: 17, align: 3 cache: 12] csize: 3851248 (ratio 0.385), ctime: 18.828006125s (235ns per bit)
+[eh-ac] [ctx: 17, align: 3 cache:  8] csize: 3851248 (ratio 0.385), ctime: 22.095326125s (276ns per bit)
+[eh-ac] [ctx: 17, align: 3 cache:  0] csize: 3851248 (ratio 0.385), ctime: 24.162549625s (302ns per bit)
+[eh-ac] [ctx: 17, align: 3 cache:  4] csize: 3851248 (ratio 0.385), ctime: 24.607925333s (308ns per bit)
+-> fastest: 3851248 in 15.055197958s (188ns per bit) for [ctx: 17, align: 3, cache: 20]
+[eh-ac] [ctx: 18, align: 3 cache: 20] csize: 3690564 (ratio 0.369), ctime: 16.331666667s (204ns per bit)
+[eh-ac] [ctx: 18, align: 3 cache: 16] csize: 3690564 (ratio 0.369), ctime: 18.320335s (229ns per bit)
+[eh-ac] [ctx: 18, align: 3 cache: 24] csize: 3690564 (ratio 0.369), ctime: 18.458653875s (231ns per bit)
+[eh-ac] [ctx: 18, align: 3 cache: 12] csize: 3690564 (ratio 0.369), ctime: 21.12637175s (264ns per bit)
+[eh-ac] [ctx: 18, align: 3 cache:  8] csize: 3690564 (ratio 0.369), ctime: 24.372665542s (305ns per bit)
+[eh-ac] [ctx: 18, align: 3 cache:  0] csize: 3690564 (ratio 0.369), ctime: 26.616466542s (333ns per bit)
+[eh-ac] [ctx: 18, align: 3 cache:  4] csize: 3690564 (ratio 0.369), ctime: 26.976785542s (337ns per bit)
+-> fastest: 3690564 in 16.331666667s (204ns per bit) for [ctx: 18, align: 3, cache: 20]
+[eh-ac] [ctx: 19, align: 3 cache: 20] csize: 3551890 (ratio 0.355), ctime: 20.963092625s (262ns per bit)
+[eh-ac] [ctx: 19, align: 3 cache: 24] csize: 3551890 (ratio 0.355), ctime: 21.101592542s (264ns per bit)
+[eh-ac] [ctx: 19, align: 3 cache: 16] csize: 3551890 (ratio 0.355), ctime: 22.844908666s (286ns per bit)
+[eh-ac] [ctx: 19, align: 3 cache: 12] csize: 3551890 (ratio 0.355), ctime: 25.828194208s (323ns per bit)
+[eh-ac] [ctx: 19, align: 3 cache:  8] csize: 3551890 (ratio 0.355), ctime: 28.80937125s (360ns per bit)
+[eh-ac] [ctx: 19, align: 3 cache:  0] csize: 3551890 (ratio 0.355), ctime: 30.873713417s (386ns per bit)
+[eh-ac] [ctx: 19, align: 3 cache:  4] csize: 3551890 (ratio 0.355), ctime: 31.308410792s (391ns per bit)
+-> fastest: 3551890 in 20.963092625s (262ns per bit) for [ctx: 19, align: 3, cache: 20]
+[eh-ac] [ctx: 20, align: 3 cache: 24] csize: 3425886 (ratio 0.343), ctime: 19.301181792s (241ns per bit)
+[eh-ac] [ctx: 20, align: 3 cache: 20] csize: 3425886 (ratio 0.343), ctime: 20.160817334s (252ns per bit)
+[eh-ac] [ctx: 20, align: 3 cache: 16] csize: 3425886 (ratio 0.343), ctime: 22.388790041s (280ns per bit)
+[eh-ac] [ctx: 20, align: 3 cache: 12] csize: 3425886 (ratio 0.343), ctime: 25.317496459s (316ns per bit)
+[eh-ac] [ctx: 20, align: 3 cache:  8] csize: 3425886 (ratio 0.343), ctime: 28.597980375s (357ns per bit)
+[eh-ac] [ctx: 20, align: 3 cache:  0] csize: 3425886 (ratio 0.343), ctime: 30.868952834s (386ns per bit)
+[eh-ac] [ctx: 20, align: 3 cache:  4] csize: 3425886 (ratio 0.343), ctime: 31.476671708s (393ns per bit)
+-> fastest: 3425886 in 19.301181792s (241ns per bit) for [ctx: 20, align: 3, cache: 24]
+[eh-ac] [ctx: 21, align: 3 cache: 24] csize: 3317291 (ratio 0.332), ctime: 19.827359666s (248ns per bit)
+[eh-ac] [ctx: 21, align: 3 cache: 20] csize: 3317291 (ratio 0.332), ctime: 20.903921875s (261ns per bit)
+[eh-ac] [ctx: 21, align: 3 cache: 16] csize: 3317291 (ratio 0.332), ctime: 23.513011666s (294ns per bit)
+[eh-ac] [ctx: 21, align: 3 cache: 12] csize: 3317291 (ratio 0.332), ctime: 26.47607875s (331ns per bit)
+[eh-ac] [ctx: 21, align: 3 cache:  8] csize: 3317291 (ratio 0.332), ctime: 29.624014125s (370ns per bit)
+[eh-ac] [ctx: 21, align: 3 cache:  0] csize: 3317291 (ratio 0.332), ctime: 31.735556584s (397ns per bit)
+[eh-ac] [ctx: 21, align: 3 cache:  4] csize: 3317291 (ratio 0.332), ctime: 32.38819925s (405ns per bit)
+-> fastest: 3317291 in 19.827359666s (248ns per bit) for [ctx: 21, align: 3, cache: 24]
+[eh-ac] [ctx: 22, align: 3 cache: 24] csize: 3229545 (ratio 0.323), ctime: 21.612879208s (270ns per bit)
+[eh-ac] [ctx: 22, align: 3 cache: 20] csize: 3229545 (ratio 0.323), ctime: 22.925507333s (287ns per bit)
+[eh-ac] [ctx: 22, align: 3 cache: 16] csize: 3229545 (ratio 0.323), ctime: 25.533208541s (319ns per bit)
+[eh-ac] [ctx: 22, align: 3 cache: 12] csize: 3229545 (ratio 0.323), ctime: 29.049849875s (363ns per bit)
+[eh-ac] [ctx: 22, align: 3 cache:  8] csize: 3229545 (ratio 0.323), ctime: 32.052884125s (401ns per bit)
+[eh-ac] [ctx: 22, align: 3 cache:  0] csize: 3229545 (ratio 0.323), ctime: 34.139155542s (427ns per bit)
+[eh-ac] [ctx: 22, align: 3 cache:  4] csize: 3229545 (ratio 0.323), ctime: 34.894869666s (436ns per bit)
+-> fastest: 3229545 in 21.612879208s (270ns per bit) for [ctx: 22, align: 3, cache: 24]
+[eh-ac] [ctx: 23, align: 3 cache: 24] csize: 3166717 (ratio 0.317), ctime: 26.562287209s (332ns per bit)
+[eh-ac] [ctx: 23, align: 3 cache: 20] csize: 3166717 (ratio 0.317), ctime: 27.772297917s (347ns per bit)
+[eh-ac] [ctx: 23, align: 3 cache: 16] csize: 3166717 (ratio 0.317), ctime: 30.12419625s (377ns per bit)
+[eh-ac] [ctx: 23, align: 3 cache: 12] csize: 3166717 (ratio 0.317), ctime: 32.923485917s (412ns per bit)
+[eh-ac] [ctx: 23, align: 3 cache:  8] csize: 3166717 (ratio 0.317), ctime: 36.021847333s (450ns per bit)
+[eh-ac] [ctx: 23, align: 3 cache:  0] csize: 3166717 (ratio 0.317), ctime: 37.987833458s (475ns per bit)
+[eh-ac] [ctx: 23, align: 3 cache:  4] csize: 3166717 (ratio 0.317), ctime: 38.718837583s (484ns per bit)
+-> fastest: 3166717 in 26.562287209s (332ns per bit) for [ctx: 23, align: 3, cache: 24]
+[eh-ac] [ctx: 24, align: 3 cache: 24] csize: 3125938 (ratio 0.313), ctime: 26.020431375s (325ns per bit)
+[eh-ac] [ctx: 24, align: 3 cache: 20] csize: 3125938 (ratio 0.313), ctime: 27.28847175s (341ns per bit)
+[eh-ac] [ctx: 24, align: 3 cache: 16] csize: 3125938 (ratio 0.313), ctime: 30.135662875s (377ns per bit)
+[eh-ac] [ctx: 24, align: 3 cache: 12] csize: 3125938 (ratio 0.313), ctime: 33.147139791s (414ns per bit)
+[eh-ac] [ctx: 24, align: 3 cache:  8] csize: 3125938 (ratio 0.313), ctime: 36.1792225s (452ns per bit)
+[eh-ac] [ctx: 24, align: 3 cache:  0] csize: 3125938 (ratio 0.313), ctime: 38.198100875s (477ns per bit)
+[eh-ac] [ctx: 24, align: 3 cache:  4] csize: 3125938 (ratio 0.313), ctime: 38.954118458s (487ns per bit)
+-> fastest: 3125938 in 26.020431375s (325ns per bit) for [ctx: 24, align: 3, cache: 24]
+[eh-ac] [ctx: 25, align: 3 cache: 24] csize: 3103899 (ratio 0.310), ctime: 28.037754958s (350ns per bit)
+[eh-ac] [ctx: 25, align: 3 cache: 20] csize: 3103899 (ratio 0.310), ctime: 29.15457925s (364ns per bit)
+[eh-ac] [ctx: 25, align: 3 cache: 16] csize: 3103899 (ratio 0.310), ctime: 32.255832292s (403ns per bit)
+[eh-ac] [ctx: 25, align: 3 cache: 12] csize: 3103899 (ratio 0.310), ctime: 35.274425792s (441ns per bit)
+[eh-ac] [ctx: 25, align: 3 cache:  8] csize: 3103899 (ratio 0.310), ctime: 38.278871041s (478ns per bit)
+[eh-ac] [ctx: 25, align: 3 cache:  0] csize: 3103899 (ratio 0.310), ctime: 40.362667042s (505ns per bit)
+[eh-ac] [ctx: 25, align: 3 cache:  4] csize: 3103899 (ratio 0.310), ctime: 41.023316208s (513ns per bit)
+-> fastest: 3103899 in 28.037754958s (350ns per bit) for [ctx: 25, align: 3, cache: 24]
+[eh-ac] [ctx: 26, align: 3 cache: 24] csize: 3103438 (ratio 0.310), ctime: 30.950021209s (387ns per bit)
+[eh-ac] [ctx: 26, align: 3 cache: 20] csize: 3103438 (ratio 0.310), ctime: 32.385497792s (405ns per bit)
+[eh-ac] [ctx: 26, align: 3 cache: 16] csize: 3103438 (ratio 0.310), ctime: 35.788329s (447ns per bit)
+[eh-ac] [ctx: 26, align: 3 cache: 12] csize: 3103438 (ratio 0.310), ctime: 38.528996166s (482ns per bit)
+[eh-ac] [ctx: 26, align: 3 cache:  8] csize: 3103438 (ratio 0.310), ctime: 42.077111s (526ns per bit)
+[eh-ac] [ctx: 26, align: 3 cache:  0] csize: 3103438 (ratio 0.310), ctime: 44.018570375s (550ns per bit)
+[eh-ac] [ctx: 26, align: 3 cache:  4] csize: 3103438 (ratio 0.310), ctime: 44.98528475s (562ns per bit)
+-> fastest: 3103438 in 30.950021209s (387ns per bit) for [ctx: 26, align: 3, cache: 24]
+--> gloabl best: 3103438 in 30.950021209s (387ns per bit) for [ctx: 26, align: 3, cache: 24]