[COLLECTIONS-843] Implement Layered Bloom filter

src/main/java/org/apache/commons/collections4/bloomfilter/BloomFilter.java

@@ -209,6 +209,21 @@ default boolean isFull() {
      */
     int cardinality();
 
+    /**
+     * Determines if all the bits are off. This is equivalent to
+     * {@code cardinality() == 0}.
+     *
+     * <p>
+     * <em>Note: This method is optimised for non-sparse filters.</em> Implementers
+     * are encouraged to implement faster checks if possible.
+     * </p>
+     *
+     * @return {@code true} if no bites are enabled, {@code false} otherwise.
+     */
+    default boolean isEmpty() {
+        return forEachBitMap(y -> y == 0);
+    }
+
     /**
      * Estimates the number of items in the Bloom filter.
      *

src/main/java/org/apache/commons/collections4/bloomfilter/BloomFilterProducer.java

@@ -16,20 +16,22 @@
  */
 package org.apache.commons.collections4.bloomfilter;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.function.BiPredicate;
 import java.util.function.Predicate;
 
 /**
- * Produces Bloom filters that are copies of Bloom filters in a collection (e.g.
- * LayerBloomFilter).
+ * Produces Bloom filters from a collection (e.g. LayeredBloomFilter).
  *
  * @since 4.5
  */
 public interface BloomFilterProducer {
+
     /**
-     * Executes a Bloom filter Predicate on each Bloom filter in the manager in
-     * depth order. Oldest filter first.
+     * Executes a Bloom filter Predicate on each Bloom filter in the collection. The
+     * ordering of the Bloom filters is not specified by this interface.
      *
      * @param bloomFilterPredicate the predicate to evaluate each Bloom filter with.
      * @return {@code false} when the first filter fails the predicate test. Returns
@@ -38,54 +40,102 @@ public interface BloomFilterProducer {
     boolean forEachBloomFilter(Predicate<BloomFilter> bloomFilterPredicate);
 
     /**
-     * Return a deep copy of the BloomFilterProducer data as a Bloom filter array.
-     * <p>
-     * The default implementation of this method is slow. It is recommended that
-     * implementing classes reimplement this method.
-     * </p>
+     * Return an array of the Bloom filters in the collection.
+     * <p><em>Implementations should specify if the array contains deep copies, immutable instances,
+     * or references to the filters in the collection.</em></p>
      *
      * @return An array of Bloom filters.
      */
     default BloomFilter[] asBloomFilterArray() {
-        class Filters {
-            private BloomFilter[] data = new BloomFilter[16];
-            private int size;
-
-            boolean add(final BloomFilter filter) {
-                if (size == data.length) {
-                    // This will throw an out-of-memory error if there are too many Bloom filters.
-                    data = Arrays.copyOf(data, size * 2);
-                }
-                data[size++] = filter.copy();
-                return true;
-            }
-
-            BloomFilter[] toArray() {
-                // Edge case to avoid a large array copy
-                return size == data.length ? data : Arrays.copyOf(data, size);
-            }
-        }
-        final Filters filters = new Filters();
-        forEachBloomFilter(filters::add);
-        return filters.toArray();
+        final List<BloomFilter> filters = new ArrayList<>();
+        forEachBloomFilter(f -> filters.add(f.copy()));
+        return filters.toArray(new BloomFilter[filters.size()]);
     }
 
     /**
      * Applies the {@code func} to each Bloom filter pair in order. Will apply all
      * of the Bloom filters from the other BloomFilterProducer to this producer. If
-     * this producer does not have as many BloomFilters it will provide
-     * {@code null} for all excess calls to the BiPredicate.
+     * either {@code this} producer or {@code other} producer has fewer BloomFilters
+     * ths method will provide {@code null} for all excess calls to the {@code func}.
+     *
+     * <p><em>This implementation returns references to the Bloom filter.  Other implementations
+     * should specify if the array contains deep copies, immutable instances,
+     * or references to the filters in the collection.</em></p>
      *
      * @param other The other BloomFilterProducer that provides the y values in the
      *              (x,y) pair.
      * @param func  The function to apply.
-     * @return A LongPredicate that tests this BitMapProducers bitmap values in
-     *         order.
+     * @return {@code true} if the {@code func} returned {@code true} for every pair,
+     *         {@code false} otherwise.
      */
     default boolean forEachBloomFilterPair(final BloomFilterProducer other,
             final BiPredicate<BloomFilter, BloomFilter> func) {
         final CountingPredicate<BloomFilter> p = new CountingPredicate<>(asBloomFilterArray(), func);
         return other.forEachBloomFilter(p) && p.forEachRemaining();
     }
 
+    /**
+     * Create a standard (non-layered) Bloom filter by merging all of the layers. If
+     * the filter is empty this method will return an empty Bloom filter.
+     *
+     * @return the merged bloom filter.
+     */
+    default BloomFilter flatten() {
+        BloomFilter bf[] = {null};
+        forEachBloomFilter( x -> {
+            if (bf[0]==null) {
+                bf[0] = new SimpleBloomFilter( x.getShape());
+            }
+            return bf[0].merge( x );
+        });
+        return bf[0];
+    }
+
+    /**
+     * Creates a BloomFilterProducer from an array of Bloom filters.
+     *
+     * <ul>
+     * <li>The asBloomFilterArray() method returns a copy of the original array
+     * with references to the original filters.</li>
+     * <li>The forEachBloomFilterPair() method uses references to the original filters.</li>
+     * </ul>
+     * <p><em>All modifications to the Bloom filters are reflected in the original filters</em></p>
+     *
+     * @param filters The filters to be returned by the producer.
+     * @return THe BloomFilterProducer containing the filters.
+     */
+    static BloomFilterProducer fromBloomFilterArray(BloomFilter... filters) {
+        return new BloomFilterProducer() {
+            @Override
+            public boolean forEachBloomFilter(final Predicate<BloomFilter> predicate) {
+                for (final BloomFilter filter : filters) {
+                    if (!predicate.test(filter)) {
+                        return false;
+                    }
+                }
+                return true;
+            }
+
+            /**
+             * This implementation returns a copy the original array, the contained Bloom filters
+             * are references to the originals, any modifications to them are reflected in the original
+             * filters.
+             */
+            @Override
+            public BloomFilter[] asBloomFilterArray() {
+                return Arrays.copyOf(filters, filters.length);
+            }
+
+            /**
+             * This implementation uses references to the original filters.  Any modifications to the
+             * filters are reflected in the originals.
+             */
+            @Override
+            public boolean forEachBloomFilterPair(final BloomFilterProducer other,
+                    final BiPredicate<BloomFilter, BloomFilter> func) {
+                final CountingPredicate<BloomFilter> p = new CountingPredicate<>(filters, func);
+                return other.forEachBloomFilter(p) && p.forEachRemaining();
+            }
+        };
+    }
 }

src/main/java/org/apache/commons/collections4/bloomfilter/CellProducer.java

@@ -67,6 +67,11 @@ default boolean forEachIndex(final IntPredicate predicate) {
         return forEachCell((i, v) -> predicate.test(i));
     }
 
+    @Override
+    default IndexProducer uniqueIndices() {
+        return this;
+    }
+
     /**
      * Creates a CellProducer from an IndexProducer.
      *

src/main/java/org/apache/commons/collections4/bloomfilter/CountingLongPredicate.java

@@ -19,20 +19,21 @@
 import java.util.function.LongPredicate;
 
 /**
- * A long predicate that applies the test func to each member of the @{code ary} in sequence for each call to @{code test()}.
- * if the @{code ary} is exhausted, the subsequent calls to @{code test} are executed with a zero value.
- * If the calls to @{code test} do not exhaust the @{code ary} the @{code forEachRemaining} method can be called to
- * execute the @code{text} with a zero value for each remaining @{code idx} value.
+ * A long predicate that applies the test func to each member of the {@code ary} in sequence for each call to {@code test()}.
+ * if the {@code ary} is exhausted, the subsequent calls to {@code test} are executed with a zero value.
+ * If the calls to {@code test} do not exhaust the {@code ary} the {@code forEachRemaining} method can be called to
+ * execute the @code{text} with a zero value for each remaining {@code idx} value.
+ * @since 4.5
  */
 class CountingLongPredicate implements LongPredicate {
     private int idx = 0;
     private final long[] ary;
     private final LongBiPredicate func;
 
     /**
-     * Constructs an instance that will compare the elements in @{code ary} with the elements returned by @{code func}.
-     * function is called as @{code func.test( idxValue, otherValue )}. If there are more @{code otherValue} values than
-     * @{code idxValues} then @{code func} is called as @{code func.test( 0, otherValue )}.
+     * Constructs an instance that will compare the elements in {@code ary} with the elements returned by {@code func}.
+     * function is called as {@code func.test( idxValue, otherValue )}. If there are more {@code otherValue} values than
+     * {@code idxValues} then {@code func} is called as {@code func.test( 0, otherValue )}.
      * @param ary The array of long values to compare.
      * @param func The function to apply to the pairs of long values.
      */

src/main/java/org/apache/commons/collections4/bloomfilter/CountingPredicate.java

@@ -20,29 +20,27 @@
 import java.util.function.Predicate;
 
 /**
- * A predicate that applies the test func to each member of the @{code ary} in
- * sequence for each call to @{code test()}. if the @{code ary} is exhausted,
- * the subsequent calls to @{code test} are executed with a {@code null} value.
- * If the calls to @{code test} do not exhaust the @{code ary} the @{code
+ * A predicate that applies the test {@code func} to each member of the {@code ary} in
+ * sequence for each call to {@code test()}. if the {@code ary} is exhausted,
+ * the subsequent calls to {@code test} are executed with a {@code null} value.
+ * If the calls to {@code test} do not exhaust the {@code ary} the {@code
  * forEachRemaining} method can be called to execute the @code{text} with a
- * {@code null} value for each remaining @{code idx} value.
+ * {@code null} value for each remaining {@code idx} value.
  *
  * @param <T> the type of object being compared.
- *
- * @Since 4.5
+ * @since 4.5
  */
 class CountingPredicate<T> implements Predicate<T> {
     private int idx = 0;
     private final T[] ary;
     private final BiPredicate<T, T> func;
 
     /**
-     * Constructs an instance that will compare the elements in @{code ary} with the
-     * elements returned by @{code func}. function is called as @{code func.test(
-     * idxValue, otherValue )}. If there are more @{code otherValue} values than
+     * Constructs an instance that will compare the elements in {@code ary} with the
+     * elements returned by {@code func}. function is called as {@code func.test(
+     * idxValue, otherValue )}. If there are more {@code otherValue} values than
+     * {@code idxValues} then {@code func} is called as {@code func.test(null, otherValue)}.
      *
-     * @{code idxValues} then @{code func} is called as @{code func.test( null,
-     *        otherValue )}.
      * @param ary  The array of long values to compare.
      * @param func The function to apply to the pairs of long values.
      */
@@ -57,10 +55,10 @@ public boolean test(final T other) {
     }
 
     /**
-     * Call the T-T consuming bi-predicate for each remaining unpaired T in the
+     * Call BiPredicate&lt;T,T&gt; for each remaining unpaired &lt;T&gt; in the
      * input array. This method should be invoked after the predicate has been
-     * passed to a @{code TProducer#forEachT(BiPredicate(T,T))} to consume any
-     * unpaired Ts. The second argument to the bi-predicate will be @{code null}.
+     * passed to a &lt;T&gt;Producer#forEach&lt;T&gt;(BiPredicate&lt;T,T&gt;) to consume any
+     * unpaired &lt;T&gt;s. The second argument to the BiPredicate will be {@code null}.
      *
      * @return true if all calls the predicate were successful
      */

src/main/java/org/apache/commons/collections4/bloomfilter/Hasher.java

@@ -27,7 +27,7 @@ public interface Hasher {
     /**
      * Creates an IndexProducer for this hasher based on the Shape.
      *
-     * <p>The @{code IndexProducer} will create indices within the range defined by the number of bits in
+     * <p>The {@code IndexProducer} will create indices within the range defined by the number of bits in
      * the shape. The total number of indices will respect the number of hash functions per item
      * defined by the shape. However the count of indices may not be a multiple of the number of
      * hash functions if the implementation has removed duplicates.</p>