From fc6c471943745e4b0549fdf408528947904e2106 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 19 Jun 2024 14:49:34 -0700 Subject: [PATCH 01/11] Java8-11 changes consistent with 17 --- pom.xml | 4 +- .../org/apache/datasketches/common/Util.java | 3 +- .../org/apache/datasketches/theta/Sketch.java | 9 +- .../apache/datasketches/theta/Sketches.java | 10 +- .../filters/bloomfilter/BloomFilterTest.java | 4 +- .../bloomfilter/DirectBitArrayTest.java | 2 +- .../hll/DirectAuxHashMapTest.java | 83 +- .../hll/DirectCouponListTest.java | 17 +- .../quantiles/DebugUnionTest.java | 6 +- .../DirectQuantilesMemoryRequestTest.java | 147 ++- .../quantiles/DoublesSketchTest.java | 51 +- .../quantiles/PreambleUtilTest.java | 6 +- .../datasketches/theta/CompactSketchTest.java | 6 +- .../theta/DirectQuickSelectSketchTest.java | 203 ++--- .../theta/HeapifyWrapSerVer1and2Test.java | 840 +++++++++--------- .../datasketches/theta/UnionImplTest.java | 37 +- .../datasketches/theta/UpdateSketchTest.java | 8 +- 17 files changed, 630 insertions(+), 806 deletions(-) diff --git a/pom.xml b/pom.xml index 35107f24c..1c1107c6c 100644 --- a/pom.xml +++ b/pom.xml @@ -134,13 +134,13 @@ under the License. - + + --> org.testng diff --git a/src/main/java/org/apache/datasketches/common/Util.java b/src/main/java/org/apache/datasketches/common/Util.java index 729b92f3f..19a8ee614 100644 --- a/src/main/java/org/apache/datasketches/common/Util.java +++ b/src/main/java/org/apache/datasketches/common/Util.java @@ -245,7 +245,8 @@ public static String zeroPad(final String s, final int fieldLength) { /** * Prepend or postpend the given string with the given character to fill the given field length. - * If the given string is equal to or greater than the given field length, it will be returned without modification. + * If the given string is equal to or greater than the given field length, it will be returned + * without modification. * @param s the given string * @param fieldLength the desired field length * @param padChar the desired pad character diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java index cc1fd4d23..6c59e4aa6 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketch.java +++ b/src/main/java/org/apache/datasketches/theta/Sketch.java @@ -292,14 +292,11 @@ public double getLowerBound(final int numStdDev) { /** * Returns the maximum number of storage bytes required for a CompactSketch with the given - * number of actual entries. Note that this assumes the worse case of the sketch in - * estimation mode, which requires storing theta and count. - * @param numberOfEntries the actual number of entries stored with the CompactSketch. + * number of actual entries. + * @param numberOfEntries the actual number of retained entries stored in the sketch. * @return the maximum number of storage bytes required for a CompactSketch with the given number - * of entries. - * @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) instead} + * of retained entries. */ - @Deprecated public static int getMaxCompactSketchBytes(final int numberOfEntries) { if (numberOfEntries == 0) { return 8; } if (numberOfEntries == 1) { return 16; } diff --git a/src/main/java/org/apache/datasketches/theta/Sketches.java b/src/main/java/org/apache/datasketches/theta/Sketches.java index 4b1461876..dc20072dc 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketches.java +++ b/src/main/java/org/apache/datasketches/theta/Sketches.java @@ -80,15 +80,11 @@ public static int getMaxAnotBResultBytes(final int maxNomEntries) { /** * Returns the maximum number of storage bytes required for a CompactSketch with the given - * number of actual entries. Note that this assumes the worse case of the sketch in - * estimation mode, which requires storing theta and count. - * @param numberOfEntries the actual number of entries stored with the CompactSketch. + * number of actual entries. + * @param numberOfEntries the actual number of retained entries stored in the sketch. * @return the maximum number of storage bytes required for a CompactSketch with the given number - * of entries. - * @see Sketch#getMaxCompactSketchBytes(int) - * @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) instead} + * of retained entries. */ - @Deprecated public static int getMaxCompactSketchBytes(final int numberOfEntries) { return Sketch.getMaxCompactSketchBytes(numberOfEntries); } diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java index dedaf9db3..fba761a94 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java @@ -28,7 +28,6 @@ import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesReadOnlyException; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; @@ -51,8 +50,7 @@ public void createNewFilterTest() throws Exception { assertFalse(bf1.isDirect()); assertFalse(bf1.isReadOnly()); - try (WritableHandle wh = WritableMemory.allocateDirect(sizeBytes)) { - final WritableMemory wmem = wh.getWritable(); + try (WritableMemory wmem = WritableMemory.allocateDirect(sizeBytes)) { final BloomFilter bf2 = new BloomFilter(numBits, numHashes, seed, wmem); assertTrue(bf2.isEmpty()); assertTrue(bf2.hasMemory()); diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java index a45bcbb82..46d98f558 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java @@ -139,7 +139,7 @@ public void basicWritableWrapTest() { @Test public void countWritableWrappedBitsWhenDirty() { // like basicOperationTest but with setBit which does - // not neecssarily track numBitsSet_ + // not necessarily track numBitsSet_ final HeapBitArray hba = new HeapBitArray(128); assertFalse(hba.getAndSetBit(1)); assertFalse(hba.getAndSetBit(2)); diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java index 78b18c1cc..71774783c 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java @@ -34,7 +34,6 @@ import org.testng.annotations.Test; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; @@ -50,51 +49,45 @@ public void checkGrow() { int n = 8; //put lgConfigK == 4 into HLL mode int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType); HllSketch hllSketch; - try (WritableHandle handle = WritableMemory.allocateDirect(bytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - WritableMemory wmem = handle.getWritable(); - hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); - for (int i = 0; i < n; i++) { - hllSketch.update(i); - } - hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values - hllSketch.couponUpdate(HllUtil.pair(8, 15)); - hllSketch.couponUpdate(HllUtil.pair(9, 15)); - //println(hllSketch.toString(true, true, true, true)); - DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl; - assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2); - assertTrue(hllSketch.isMemory()); - assertTrue(hllSketch.isOffHeap()); - assertTrue(hllSketch.isSameResource(wmem)); - - //Check heapify - byte[] byteArray = hllSketch.toCompactByteArray(); - HllSketch hllSketch2 = HllSketch.heapify(byteArray); - HllArray ha = (HllArray) hllSketch2.hllSketchImpl; - assertEquals(ha.getAuxHashMap().getLgAuxArrInts(), 2); - assertEquals(ha.getAuxHashMap().getAuxCount(), 3); - - //Check wrap - byteArray = hllSketch.toUpdatableByteArray(); - WritableMemory wmem2 = WritableMemory.writableWrap(byteArray); - hllSketch2 = HllSketch.writableWrap(wmem2); - //println(hllSketch2.toString(true, true, true, true)); - DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl; - assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2); - assertEquals(dha2.getAuxHashMap().getAuxCount(), 3); - - //Check grow to on-heap - hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow - //println(hllSketch.toString(true, true, true, true)); - dha = (DirectHllArray) hllSketch.hllSketchImpl; - assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3); - assertEquals(dha.getAuxHashMap().getAuxCount(), 4); - assertTrue(hllSketch.isMemory()); - assertFalse(hllSketch.isOffHeap()); - assertFalse(hllSketch.isSameResource(wmem)); - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(bytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); + for (int i = 0; i < n; i++) { + hllSketch.update(i); } + hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values + hllSketch.couponUpdate(HllUtil.pair(8, 15)); + hllSketch.couponUpdate(HllUtil.pair(9, 15)); + //println(hllSketch.toString(true, true, true, true)); + DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl; + assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2); + assertTrue(hllSketch.isMemory()); + assertTrue(hllSketch.isOffHeap()); + assertTrue(hllSketch.isSameResource(wmem)); + + //Check heapify + byte[] byteArray = hllSketch.toCompactByteArray(); + HllSketch hllSketch2 = HllSketch.heapify(byteArray); + HllArray ha = (HllArray) hllSketch2.hllSketchImpl; + assertEquals(ha.getAuxHashMap().getLgAuxArrInts(), 2); + assertEquals(ha.getAuxHashMap().getAuxCount(), 3); + + //Check wrap + byteArray = hllSketch.toUpdatableByteArray(); + WritableMemory wmem2 = WritableMemory.writableWrap(byteArray); + hllSketch2 = HllSketch.writableWrap(wmem2); + //println(hllSketch2.toString(true, true, true, true)); + DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl; + assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2); + assertEquals(dha2.getAuxHashMap().getAuxCount(), 3); + + //Check grow to on-heap + hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow + //println(hllSketch.toString(true, true, true, true)); + dha = (DirectHllArray) hllSketch.hllSketchImpl; + assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3); + assertEquals(dha.getAuxHashMap().getAuxCount(), 4); + assertTrue(hllSketch.isMemory()); + assertFalse(hllSketch.isOffHeap()); } @Test diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java index 38cbc4977..b19e5b361 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java @@ -28,7 +28,6 @@ import org.testng.annotations.Test; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; /** @@ -69,11 +68,8 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool //println("DIRECT"); byte[] barr1; - WritableMemory wmem = null; - try (WritableHandle hand = WritableMemory.allocateDirect(bytes)) { - wmem = hand.getWritable(); - //byte[] byteArr = new byte[bytes]; - //WritableMemory wmem = WritableMemory.wrap(byteArr); + WritableMemory wmem = WritableMemory.allocateDirect(bytes); //direct? + hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); assertTrue(hllSketch.isEmpty()); @@ -84,7 +80,7 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool assertFalse(hllSketch.isEmpty()); assertEquals(hllSketch.getCurMode(), tgtMode); assertTrue(hllSketch.isMemory()); - assertTrue(hllSketch.isOffHeap()); + assertTrue(hllSketch.isOffHeap()); // assertTrue(hllSketch.isSameResource(wmem)); //convert direct sketch to byte[] @@ -92,9 +88,7 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool //println(PreambleUtil.toString(barr1)); hllSketch.reset(); assertTrue(hllSketch.isEmpty()); - } catch (final Exception e) { - throw new RuntimeException(e); - } + //println("HEAP"); HllSketch hllSketch2 = new HllSketch(lgConfigK, tgtHllType); @@ -111,6 +105,9 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool assertEquals(barr1.length, barr2.length, barr1.length + ", " + barr2.length); //printDiffs(barr1, barr2); assertEquals(barr1, barr2); + if (wmem.isCloseable()) { + wmem.close(); + } } @SuppressWarnings("unused") //only used when above printlns are enabled. diff --git a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java index bbf89f338..039eefa0f 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java @@ -26,7 +26,6 @@ import java.util.HashSet; import org.testng.annotations.Test; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIterator; @@ -61,13 +60,10 @@ public void test() { DoublesSketch.setRandom(1); //make deterministic for test DoublesUnion dUnion; DoublesSketch dSketch; - try ( WritableHandle wdh = WritableMemory.allocateDirect(10_000_000) ) { - WritableMemory wmem = wdh.getWritable(); + try ( WritableMemory wmem = WritableMemory.allocateDirect(10_000_000) ) { dUnion = DoublesUnion.builder().setMaxK(8).build(wmem); for (int s = 0; s < numSketches; s++) { dUnion.union(sketchArr[s]); } dSketch = dUnion.getResult(); //result is on heap - } catch (final Exception e) { - throw new RuntimeException(e); } //iterates and counts errors diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java index 5c2882c5e..c8771d80a 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java @@ -30,7 +30,6 @@ import org.testng.annotations.Test; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; /** @@ -47,35 +46,30 @@ public void checkLimitedMemoryScenarios() { //Requesting application final int initBytes = ((2 * k) + 4) << 3; //just the BB //########## Owning Implementation - // This part would actually be part of the Memory owning implemention so it is faked here - try (WritableHandle wdh = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - final WritableMemory wmem = wdh.getWritable(); - println("Initial mem size: " + wmem.getCapacity()); - - //########## Receiving Application - // The receiving application has been given wmem to use for a sketch, - // but alas, it is not ultimately large enough. - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); - assertTrue(usk1.isEmpty()); - - //Load the sketch - for (int i = 0; i < u; i++) { - // The sketch uses The MemoryRequest, acquired from wmem, to acquire more memory as - // needed, and requests via the MemoryRequest to free the old allocations. - usk1.update(i); - } - final double result = usk1.getQuantile(0.5); - println("Result: " + result); - assertEquals(result, u / 2.0, 0.05 * u); //Success - - //########## Owning Implementation - //The actual Memory has been re-allocated several times, - // so the above wmem reference is invalid. - println("\nFinal mem size: " + wmem.getCapacity()); - } catch (Exception e) { - throw new RuntimeException(e); + // This part would actually be part of the Memory owning implementation so it is faked here + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + println("Initial mem size: " + wmem.getCapacity()); + + //########## Receiving Application + // The receiving application has been given wmem to use for a sketch, + // but alas, it is not ultimately large enough. + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); + assertTrue(usk1.isEmpty()); + + //Load the sketch + for (int i = 0; i < u; i++) { + // The sketch uses The MemoryRequest, acquired from wmem, to acquire more memory as + // needed, and requests via the MemoryRequest to free the old allocations. + usk1.update(i); } + final double result = usk1.getQuantile(0.5); + println("Result: " + result); + assertEquals(result, u / 2.0, 0.05 * u); //Success + + //The actual Memory has been re-allocated several times, + // so the the wmem reference is invalid. Use the sketch to get the last memory reference. + WritableMemory lastMem = usk1.getMemory(); + println("Final mem size: " + usk1.getMemory().getCapacity()); } @Test @@ -84,22 +78,16 @@ public void checkGrowBaseBuf() { final int u = 32; // don't need the BB to fill here final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything - try (WritableHandle memHandler = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - //final MemoryManager memMgr = new MemoryManager(); - //final WritableMemory mem1 = memMgr.request(initBytes); - final WritableMemory mem1 = memHandler.getWritable(); - println("Initial mem size: " + mem1.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1); - for (int i = 1; i <= u; i++) { - usk1.update(i); - } - final int currentSpace = usk1.getCombinedBufferItemCapacity(); - println("curCombBufItemCap: " + currentSpace); - assertEquals(currentSpace, 2 * k); - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory mem1 = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + println("Initial mem size: " + mem1.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1); + for (int i = 1; i <= u; i++) { + usk1.update(i); } + final int currentSpace = usk1.getCombinedBufferItemCapacity(); + println("curCombBufItemCap: " + currentSpace); + assertEquals(currentSpace, 2 * k); + println("last Mem Cap: " + usk1.getMemory().getCapacity()); } @Test @@ -108,26 +96,18 @@ public void checkGrowCombBuf() { final int u = (2 * k) - 1; //just to fill the BB final int initBytes = ((2 * k) + 4) << 3; //just room for BB - try (WritableHandle memHandler = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - //final MemoryManager memMgr = new MemoryManager(); - //final WritableMemory mem1 = memMgr.request(initBytes); - final WritableMemory mem1 = memHandler.getWritable(); - println("Initial mem size: " + mem1.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1); - for (int i = 1; i <= u; i++) { - usk1.update(i); - } - final int currentSpace = usk1.getCombinedBufferItemCapacity(); - println("curCombBufItemCap: " + currentSpace); - final double[] newCB = usk1.growCombinedBuffer(currentSpace, 3 * k); - final int newSpace = usk1.getCombinedBufferItemCapacity(); - println("newCombBurItemCap: " + newSpace); - assertEquals(newCB.length, 3 * k); - //memMgr.free(mem1); - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory mem1 = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + println("Initial mem size: " + mem1.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1); + for (int i = 1; i <= u; i++) { + usk1.update(i); } + final int currentSpace = usk1.getCombinedBufferItemCapacity(); + println("curCombBufItemCap: " + currentSpace); + final double[] newCB = usk1.growCombinedBuffer(currentSpace, 3 * k); + final int newSpace = usk1.getCombinedBufferItemCapacity(); + println("newCombBurItemCap: " + newSpace); + assertEquals(newCB.length, 3 * k); } @Test @@ -138,28 +118,23 @@ public void checkGrowFromWrappedEmptySketch() { final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); - try (WritableHandle memHandle = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - WritableMemory mem = memHandle.getWritable(); - origSketchMem.copyTo(0, mem, 0, initBytes); - UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(mem); - assertTrue(mem.isSameResource(usk2.getMemory())); - assertEquals(mem.getCapacity(), initBytes); - assertTrue(mem.isDirect()); - assertTrue(usk2.isEmpty()); - - //update the sketch forcing it to grow on-heap - for (int i = 1; i <= 5; i++) { usk2.update(i); } - assertEquals(usk2.getN(), 5); - WritableMemory mem2 = usk2.getMemory(); - assertFalse(mem.isSameResource(mem2)); - assertFalse(mem2.isDirect()); //should now be on-heap - - final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); - assertEquals(mem2.getCapacity(), expectedSize); - } catch (final Exception e) { - throw new RuntimeException(e); - } + WritableMemory mem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + origSketchMem.copyTo(0, mem, 0, initBytes); + UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(mem); + assertTrue(mem.isSameResource(usk2.getMemory())); + assertEquals(mem.getCapacity(), initBytes); + assertTrue(mem.isDirect()); + assertTrue(usk2.isEmpty()); + + //update the sketch forcing it to grow on-heap + for (int i = 1; i <= 5; i++) { usk2.update(i); } + assertEquals(usk2.getN(), 5); + WritableMemory mem2 = usk2.getMemory(); + assertFalse(mem.isAlive()); // + assertFalse(mem2.isDirect()); //should now be on-heap + + final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); + assertEquals(mem2.getCapacity(), expectedSize); } @Test @@ -171,7 +146,7 @@ public void printlnTest() { * @param s value to print */ static void println(final String s) { - //System.out.println(s); //disable here + System.out.println(s); //disable here } } diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java index 5bc5b4eaa..3f64db65e 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java @@ -28,7 +28,6 @@ import java.nio.ByteOrder; import org.apache.datasketches.memory.DefaultMemoryRequestServer; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.DoublesSortedView; import org.apache.datasketches.quantilescommon.DoublesSortedViewIterator; @@ -140,50 +139,36 @@ public void checkEmptyExceptions() { @Test public void directSketchShouldMoveOntoHeapEventually() { - try (WritableHandle wdh = WritableMemory.allocateDirect(1000, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - WritableMemory mem = wdh.getWritable(); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); - Assert.assertTrue(sketch.isSameResource(mem)); - for (int i = 0; i < 1000; i++) { - sketch.update(i); - } - Assert.assertFalse(sketch.isSameResource(mem)); - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory mem = WritableMemory.allocateDirect(1000, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); + Assert.assertTrue(sketch.isSameResource(mem)); + for (int i = 0; i < 1000; i++) { + sketch.update(i); } + println(sketch.toString()); } @Test public void directSketchShouldMoveOntoHeapEventually2() { int i = 0; - try (WritableHandle wdh = - WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer())) { - WritableMemory mem = wdh.getWritable(); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); - Assert.assertTrue(sketch.isSameResource(mem)); - for (; i < 1000; i++) { - if (sketch.isSameResource(mem)) { - sketch.update(i); - } else { - //println("MOVED OUT at i = " + i); - break; - } + WritableMemory mem = WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer()); + UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); + Assert.assertTrue(sketch.isSameResource(mem)); + for (; i < 1000; i++) { + if (mem.isAlive()) { + sketch.update(i); + } else { + println("Sketch Move to Heap at i = " + i); + break; } - } catch (final Exception e) { - throw new RuntimeException(e); } } @Test public void checkEmptyDirect() { - try (WritableHandle wdh = WritableMemory.allocateDirect(1000)) { - WritableMemory mem = wdh.getWritable(); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); - sketch.toByteArray(); //exercises a specific path - } catch (final Exception e) { - throw new RuntimeException(e); - } + WritableMemory mem = WritableMemory.allocateDirect(1000); + UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); + sketch.toByteArray(); //exercises a specific path } @Test diff --git a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java index be2f328b2..07f680630 100644 --- a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java @@ -40,7 +40,6 @@ import org.testng.annotations.Test; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; public class PreambleUtilTest { @@ -48,8 +47,7 @@ public class PreambleUtilTest { @Test public void checkInsertsAndExtracts() { final int bytes = 32; - try (WritableHandle offHeapMemHandler = WritableMemory.allocateDirect(bytes)) { - final WritableMemory offHeapMem = offHeapMemHandler.getWritable(); + try (WritableMemory offHeapMem = WritableMemory.allocateDirect(bytes)) { final WritableMemory onHeapMem = WritableMemory.writableWrap(new byte[bytes]); onHeapMem.clear(); @@ -156,8 +154,6 @@ public void checkInsertsAndExtracts() { assertEquals(offHD, vD); onHeapMem.clear(); offHeapMem.clear(); - } catch (final Exception e) { - throw new RuntimeException(e); } } diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java index 42b6069da..70db8c783 100644 --- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java @@ -29,7 +29,6 @@ import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; @@ -78,8 +77,7 @@ public void checkHeapifyWrap(int k, int u, boolean ordered) { //Prepare Memory for direct int bytes = usk.getCompactBytes(); //for Compact - try (WritableHandle wdh = WritableMemory.allocateDirect(bytes)) { - WritableMemory directMem = wdh.getWritable(); + try (WritableMemory directMem = WritableMemory.allocateDirect(bytes)) { /**Via CompactSketch.compact**/ refSk = usk.compact(ordered, directMem); @@ -90,8 +88,6 @@ public void checkHeapifyWrap(int k, int u, boolean ordered) { /**Via CompactSketch.compact**/ testSk = (CompactSketch)Sketch.wrap(directMem); checkByRange(refSk, testSk, u, ordered); - } catch (final Exception e) { - throw new RuntimeException(e); } } diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index 5191c7c68..4830f9576 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -45,7 +45,6 @@ import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesReadOnlyException; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.HashOperations; import org.apache.datasketches.thetacommon.ThetaUtil; @@ -56,19 +55,15 @@ */ public class DirectQuickSelectSketchTest { - @Test//(expectedExceptions = SketchesArgumentException.class) + @Test(expectedExceptions = SketchesArgumentException.class) public void checkBadSerVer() { int k = 512; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks - assertTrue(usk.isEmpty()); for (int i = 0; i< k; i++) { usk.update(i); } - assertFalse(usk.isEmpty()); assertEquals(usk.getEstimate(), k, 0.0); assertEquals(sk1.getRetainedEntries(false), k); @@ -76,17 +71,13 @@ public void checkBadSerVer() { mem.putByte(SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte Sketch.wrap(mem); - } catch (final Exception e) { - if (e instanceof SketchesArgumentException) {} - else { throw new RuntimeException(e); } } } @Test//(expectedExceptions = SketchesArgumentException.class) public void checkConstructorKtooSmall() { int k = 8; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch.builder().setNominalEntries(k).build(mem); } catch (final Exception e) { if (e instanceof SketchesArgumentException) {} @@ -97,8 +88,7 @@ public void checkConstructorKtooSmall() { @Test//(expectedExceptions = SketchesArgumentException.class) public void checkConstructorMemTooSmall() { int k = 16; - try (WritableHandle h = makeNativeMemory(k/2)) { - WritableMemory mem = h.getWritable(); + try (WritableMemory mem = makeNativeMemory(k/2)) { UpdateSketch.builder().setNominalEntries(k).build(mem); } catch (final Exception e) { if (e instanceof SketchesArgumentException) {} @@ -124,9 +114,7 @@ public void checkHeapifyMemoryEstimating() { int k = 512; int u = 2*k; //thus estimating - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch sk1 = UpdateSketch.builder().setNominalEntries(k).build(mem); for (int i=0; i k); - } catch (final Exception e) { - throw new RuntimeException(e); } } @@ -452,9 +403,7 @@ public void checkSamplingMode() { int k = 4096; float p = (float)0.5; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch usk = UpdateSketch.builder().setP(p).setNominalEntries(k).build(mem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks @@ -470,17 +419,13 @@ public void checkSamplingMode() { assertTrue(ub > est); double lb = usk.getLowerBound(1); assertTrue(lb < est); - } catch (final Exception e) { - throw new RuntimeException(e); } } @Test public void checkErrorBounds() { int k = 512; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); //Exact mode @@ -503,8 +448,6 @@ public void checkErrorBounds() { ub = usk.getUpperBound(2); assertTrue(est <= ub); assertTrue(est >= lb); - } catch (final Exception e) { - throw new RuntimeException(e); } } @@ -514,9 +457,7 @@ public void checkEmptyAndP() { //virgin, p = 1.0 int k = 1024; float p = (float)1.0; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch usk = UpdateSketch.builder().setP(p).setNominalEntries(k).build(mem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks @@ -545,8 +486,6 @@ public void checkEmptyAndP() { double lb = usk2.getLowerBound(2); assertTrue(lb <= est); //println("LB: "+lb); - } catch (final Exception e) { - throw new RuntimeException(e); } } @@ -554,9 +493,7 @@ public void checkEmptyAndP() { public void checkUpperAndLowerBounds() { int k = 512; int u = 2*k; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); for (int i = 0; i < u; i++ ) { usk.update(i); } @@ -566,8 +503,6 @@ public void checkUpperAndLowerBounds() { double lb = usk.getLowerBound(1); assertTrue(ub > est); assertTrue(lb < est); - } catch (final Exception e) { - throw new RuntimeException(e); } } @@ -575,9 +510,7 @@ public void checkUpperAndLowerBounds() { public void checkRebuild() { int k = 512; int u = 4*k; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks @@ -595,8 +528,6 @@ public void checkRebuild() { sk1.rebuild(); assertEquals(sk1.getRetainedEntries(false), k); assertEquals(sk1.getRetainedEntries(true), k); - } catch (final Exception e) { - throw new RuntimeException(e); } } @@ -604,9 +535,7 @@ public void checkRebuild() { public void checkResetAndStartingSubMultiple() { int k = 512; int u = 4*k; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks @@ -626,8 +555,6 @@ public void checkResetAndStartingSubMultiple() { assertNotNull(sk1.getMemory()); assertFalse(sk1.isOrdered()); - } catch (final Exception e) { - throw new RuntimeException(e); } } @@ -635,9 +562,7 @@ public void checkResetAndStartingSubMultiple() { public void checkExactModeMemoryArr() { int k = 4096; int u = 4096; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks assertTrue(usk.isEmpty()); @@ -646,8 +571,6 @@ public void checkExactModeMemoryArr() { assertEquals(usk.getEstimate(), u, 0.0); assertEquals(sk1.getRetainedEntries(false), u); - } catch (final Exception e) { - throw new RuntimeException(e); } } @@ -656,9 +579,7 @@ public void checkEstModeMemoryArr() { int k = 4096; int u = 2*k; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - + try (WritableMemory mem = makeNativeMemory(k)) { UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks assertTrue(usk.isEmpty()); @@ -667,8 +588,6 @@ public void checkEstModeMemoryArr() { assertEquals(usk.getEstimate(), u, u*.05); assertTrue(sk1.getRetainedEntries(false) > k); - } catch (final Exception e) { - throw new RuntimeException(e); } } @@ -678,9 +597,8 @@ public void checkEstModeNativeMemory() { int u = 2*k; int memCapacity = (k << 4) + (Family.QUICKSELECT.getMinPreLongs() << 3); - try(WritableHandle memHandler = WritableMemory.allocateDirect(memCapacity)) { - - UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(memHandler.getWritable()); + try(WritableMemory mem = WritableMemory.allocateDirect(memCapacity)) { + UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks assertTrue(usk.isEmpty()); @@ -689,8 +607,6 @@ public void checkEstModeNativeMemory() { println(""+est); assertEquals(usk.getEstimate(), u, u*.05); assertTrue(sk1.getRetainedEntries(false) > k); - } catch (final Exception e) { - throw new RuntimeException(e); } } @@ -699,8 +615,8 @@ public void checkConstructReconstructFromMemory() { int k = 4096; int u = 2*k; - try (WritableHandle h = makeNativeMemory(k)) { - UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(h.getWritable()); + try (WritableMemory mem = makeNativeMemory(k)) { + UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); assertTrue(usk.isEmpty()); for (int i = 0; i< u; i++) { usk.update(i); } //force estimation @@ -726,8 +642,6 @@ public void checkConstructReconstructFromMemory() { assertEquals(count2, count1); assertEquals(est2, est1, 0.0); - } catch (final Exception e) { - throw new RuntimeException(e); } } @@ -868,15 +782,12 @@ public void checkMoveAndResize() { int k = 1 << 12; int u = 2 * k; int bytes = Sketches.getMaxUpdateSketchBytes(k); - try (WritableHandle wdh = WritableMemory.allocateDirect(bytes/2)) { //will request - WritableMemory wmem = wdh.getWritable(); - UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); - assertTrue(sketch.isSameResource(wmem)); - for (int i = 0; i < u; i++) { sketch.update(i); } - assertFalse(sketch.isSameResource(wmem)); - } catch (final Exception e) { - throw new RuntimeException(e); - } + WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request + UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); + assertTrue(sketch.isSameResource(wmem)); + for (int i = 0; i < u; i++) { sketch.update(i); } + assertTrue(sketch.getMemory().isAlive()); + assertFalse(wmem.isAlive()); } @Test @@ -884,35 +795,29 @@ public void checkReadOnlyRebuildResize() { int k = 1 << 12; int u = 2 * k; int bytes = Sketches.getMaxUpdateSketchBytes(k); - try (WritableHandle wdh = WritableMemory.allocateDirect(bytes/2)) { //will request - WritableMemory wmem = wdh.getWritable(); - UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); - for (int i = 0; i < u; i++) { sketch.update(i); } - double est1 = sketch.getEstimate(); - byte[] ser = sketch.toByteArray(); - Memory mem = Memory.wrap(ser); - UpdateSketch roSketch = (UpdateSketch) Sketches.wrapSketch(mem); - double est2 = roSketch.getEstimate(); - assertEquals(est2, est1); - try { - roSketch.rebuild(); - fail(); - } catch (SketchesReadOnlyException e) { - //expected - } - try { - roSketch.reset(); - fail(); - } catch (SketchesReadOnlyException e) { - //expected - } - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request + UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); + for (int i = 0; i < u; i++) { sketch.update(i); } + double est1 = sketch.getEstimate(); + byte[] ser = sketch.toByteArray(); + Memory mem = Memory.wrap(ser); + UpdateSketch roSketch = (UpdateSketch) Sketches.wrapSketch(mem); + double est2 = roSketch.getEstimate(); + assertEquals(est2, est1); + try { + roSketch.rebuild(); + fail(); + } catch (SketchesReadOnlyException e) { + //expected + } + try { + roSketch.reset(); + fail(); + } catch (SketchesReadOnlyException e) { + //expected } - } - @Test public void printlnTest() { println("PRINTING: "+this.getClass().getName()); @@ -929,7 +834,7 @@ private static final int getMaxBytes(int k) { return (k << 4) + (Family.QUICKSELECT.getMinPreLongs() << 3); } - private static WritableHandle makeNativeMemory(int k) { + private static WritableMemory makeNativeMemory(int k) { return WritableMemory.allocateDirect(getMaxBytes(k)); } diff --git a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java index bd7652bdc..0e88c4555 100644 --- a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java +++ b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java @@ -24,7 +24,6 @@ import static org.testng.Assert.assertTrue; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.ThetaUtil; import org.apache.datasketches.tuple.Util; @@ -39,29 +38,29 @@ public void checkHeapifyCompactSketchAssumedDefaultSeed() { final int k = 64; final long seed = ThetaUtil.DEFAULT_UPDATE_SEED; final short seedHash = Util.computeSeedHash(seed); - UpdateSketch sv3usk = UpdateSketch.builder().setNominalEntries(k).setSeed(seed).build(); - for (int i=0; i Date: Wed, 19 Jun 2024 17:44:58 -0700 Subject: [PATCH 02/11] More updates consistent with 17 --- .../bloomfilter/DirectBitArrayRTest.java | 5 ++-- .../bloomfilter/DirectBitArrayTest.java | 13 +++++---- .../DirectQuantilesMemoryRequestTest.java | 29 ++++++++++--------- .../quantiles/DoublesSketchTest.java | 19 ++++++------ .../datasketches/theta/UnionImplTest.java | 1 + 5 files changed, 36 insertions(+), 31 deletions(-) diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java index 521019e62..2f06b5cb8 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java @@ -27,6 +27,7 @@ import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesReadOnlyException; import org.apache.datasketches.memory.Memory; +import org.apache.datasketches.memory.MemoryBoundsException; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; @@ -142,8 +143,8 @@ public void bitAddressOutOfBoundsNonEmptyTest() { final Memory mem = bitArrayToMemory(hba); DirectBitArrayR dba = DirectBitArrayR.wrap(mem, hba.isEmpty()); - assertThrows(AssertionError.class, () -> dba.getBit(-10)); - assertThrows(AssertionError.class, () -> dba.getBit(2048)); + assertThrows(MemoryBoundsException.class, () -> dba.getBit(-10)); + assertThrows(MemoryBoundsException.class, () -> dba.getBit(2048)); } @Test diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java index 46d98f558..813b0449b 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java @@ -25,6 +25,7 @@ import static org.testng.Assert.assertTrue; import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.memory.MemoryBoundsException; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; @@ -172,12 +173,12 @@ public void bitAddresOutOfBoundsNonEmptyTest() { dba.getAndSetBit(i); } - assertThrows(AssertionError.class, () -> dba.getBit(-10)); - assertThrows(AssertionError.class, () -> dba.getBit(2048)); - assertThrows(AssertionError.class, () -> dba.setBit(-20)); - assertThrows(AssertionError.class, () -> dba.setBit(4096)); - assertThrows(AssertionError.class, () -> dba.getAndSetBit(-30)); - assertThrows(AssertionError.class, () -> dba.getAndSetBit(8192)); + assertThrows(MemoryBoundsException.class, () -> dba.getBit(-10)); + assertThrows(MemoryBoundsException.class, () -> dba.getBit(2048)); + assertThrows(MemoryBoundsException.class, () -> dba.setBit(-20)); + assertThrows(MemoryBoundsException.class, () -> dba.setBit(4096)); + assertThrows(MemoryBoundsException.class, () -> dba.getAndSetBit(-30)); + assertThrows(MemoryBoundsException.class, () -> dba.getAndSetBit(8192)); } @Test diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java index c8771d80a..f477a2c52 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java @@ -70,6 +70,7 @@ public void checkLimitedMemoryScenarios() { //Requesting application // so the the wmem reference is invalid. Use the sketch to get the last memory reference. WritableMemory lastMem = usk1.getMemory(); println("Final mem size: " + usk1.getMemory().getCapacity()); + if (wmem.isAlive()) { System.out.println("Here"); } } @Test @@ -78,9 +79,9 @@ public void checkGrowBaseBuf() { final int u = 32; // don't need the BB to fill here final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything - WritableMemory mem1 = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - println("Initial mem size: " + mem1.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + println("Initial mem size: " + wmem.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); for (int i = 1; i <= u; i++) { usk1.update(i); } @@ -96,9 +97,9 @@ public void checkGrowCombBuf() { final int u = (2 * k) - 1; //just to fill the BB final int initBytes = ((2 * k) + 4) << 3; //just room for BB - WritableMemory mem1 = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - println("Initial mem size: " + mem1.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + println("Initial mem size: " + wmem.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); for (int i = 1; i <= u; i++) { usk1.update(i); } @@ -118,19 +119,19 @@ public void checkGrowFromWrappedEmptySketch() { final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); - WritableMemory mem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - origSketchMem.copyTo(0, mem, 0, initBytes); - UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(mem); - assertTrue(mem.isSameResource(usk2.getMemory())); - assertEquals(mem.getCapacity(), initBytes); - assertTrue(mem.isDirect()); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + origSketchMem.copyTo(0, wmem, 0, initBytes); + UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem); + assertTrue(wmem.isSameResource(usk2.getMemory())); + assertEquals(wmem.getCapacity(), initBytes); + assertTrue(wmem.isDirect()); assertTrue(usk2.isEmpty()); //update the sketch forcing it to grow on-heap for (int i = 1; i <= 5; i++) { usk2.update(i); } assertEquals(usk2.getN(), 5); WritableMemory mem2 = usk2.getMemory(); - assertFalse(mem.isAlive()); // + assertFalse(wmem.isAlive()); // assertFalse(mem2.isDirect()); //should now be on-heap final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); @@ -146,7 +147,7 @@ public void printlnTest() { * @param s value to print */ static void println(final String s) { - System.out.println(s); //disable here + //System.out.println(s); //disable here } } diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java index 3f64db65e..1bf6ad589 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java @@ -139,9 +139,9 @@ public void checkEmptyExceptions() { @Test public void directSketchShouldMoveOntoHeapEventually() { - WritableMemory mem = WritableMemory.allocateDirect(1000, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); - Assert.assertTrue(sketch.isSameResource(mem)); + WritableMemory wmem = WritableMemory.allocateDirect(1000, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); + Assert.assertTrue(sketch.isSameResource(wmem)); for (int i = 0; i < 1000; i++) { sketch.update(i); } @@ -151,11 +151,11 @@ public void directSketchShouldMoveOntoHeapEventually() { @Test public void directSketchShouldMoveOntoHeapEventually2() { int i = 0; - WritableMemory mem = WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer()); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); - Assert.assertTrue(sketch.isSameResource(mem)); + WritableMemory wmem = WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer()); + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); + Assert.assertTrue(sketch.isSameResource(wmem)); for (; i < 1000; i++) { - if (mem.isAlive()) { + if (wmem.isAlive()) { sketch.update(i); } else { println("Sketch Move to Heap at i = " + i); @@ -166,9 +166,10 @@ public void directSketchShouldMoveOntoHeapEventually2() { @Test public void checkEmptyDirect() { - WritableMemory mem = WritableMemory.allocateDirect(1000); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); + WritableMemory wmem = WritableMemory.allocateDirect(1000); + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); sketch.toByteArray(); //exercises a specific path + wmem.close(); } @Test diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java index 2883b4e13..b4ff9a245 100644 --- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java +++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java @@ -204,6 +204,7 @@ public void checkMoveAndResize() { final Union union2 = SetOperation.builder().buildUnion(); //on-heap union assertFalse(union2.isSameResource(wmem2)); //obviously not + wmem.close(); } @Test From 756dd4b02f1cb909f5d5c89433c802bfbd5c17ee Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 19 Jun 2024 18:57:51 -0700 Subject: [PATCH 03/11] Fixing small things. --- .../hll/DirectCouponListTest.java | 42 +++++++++---------- .../DirectQuantilesMemoryRequestTest.java | 1 - 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java index b19e5b361..1bb9310d0 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java @@ -68,27 +68,25 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool //println("DIRECT"); byte[] barr1; - WritableMemory wmem = WritableMemory.allocateDirect(bytes); //direct? - - hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); - assertTrue(hllSketch.isEmpty()); - - for (int i = 0; i < n; i++) { - hllSketch.update(i); - } - //println(hllSketch.toString(true, true, false, false)); - assertFalse(hllSketch.isEmpty()); - assertEquals(hllSketch.getCurMode(), tgtMode); - assertTrue(hllSketch.isMemory()); - assertTrue(hllSketch.isOffHeap()); // - assertTrue(hllSketch.isSameResource(wmem)); - - //convert direct sketch to byte[] - barr1 = (compact) ? hllSketch.toCompactByteArray() : hllSketch.toUpdatableByteArray(); - //println(PreambleUtil.toString(barr1)); - hllSketch.reset(); - assertTrue(hllSketch.isEmpty()); + WritableMemory wmem = WritableMemory.allocateDirect(bytes); + hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); + assertTrue(hllSketch.isEmpty()); + for (int i = 0; i < n; i++) { + hllSketch.update(i); + } + //println(hllSketch.toString(true, true, false, false)); + assertFalse(hllSketch.isEmpty()); + assertEquals(hllSketch.getCurMode(), tgtMode); + assertTrue(hllSketch.isMemory()); + assertTrue(hllSketch.isOffHeap()); + assertTrue(hllSketch.isSameResource(wmem)); + + //convert direct sketch to byte[] + barr1 = (compact) ? hllSketch.toCompactByteArray() : hllSketch.toUpdatableByteArray(); + //println(PreambleUtil.toString(barr1)); + hllSketch.reset(); + assertTrue(hllSketch.isEmpty()); //println("HEAP"); HllSketch hllSketch2 = new HllSketch(lgConfigK, tgtHllType); @@ -105,9 +103,7 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool assertEquals(barr1.length, barr2.length, barr1.length + ", " + barr2.length); //printDiffs(barr1, barr2); assertEquals(barr1, barr2); - if (wmem.isCloseable()) { - wmem.close(); - } + wmem.close(); } @SuppressWarnings("unused") //only used when above printlns are enabled. diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java index f477a2c52..2ec2fbe27 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java @@ -70,7 +70,6 @@ public void checkLimitedMemoryScenarios() { //Requesting application // so the the wmem reference is invalid. Use the sketch to get the last memory reference. WritableMemory lastMem = usk1.getMemory(); println("Final mem size: " + usk1.getMemory().getCapacity()); - if (wmem.isAlive()) { System.out.println("Here"); } } @Test From e61d7b7405fc156357c20ecf0c873dfda5b1d9c4 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 19 Jun 2024 19:57:48 -0700 Subject: [PATCH 04/11] fix getCompactSketchMaxBytes & javadocs --- src/main/java/org/apache/datasketches/theta/Sketch.java | 4 ++-- src/test/java/org/apache/datasketches/theta/SketchesTest.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java index 6c59e4aa6..92768c8e2 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketch.java +++ b/src/main/java/org/apache/datasketches/theta/Sketch.java @@ -311,8 +311,8 @@ public static int getMaxCompactSketchBytes(final int numberOfEntries) { * nomEntries. */ public static int getCompactSketchMaxBytes(final int lgNomEntries) { - return (int)((2 << lgNomEntries) * ThetaUtil.REBUILD_THRESHOLD) - + Family.QUICKSELECT.getMaxPreLongs() * Long.BYTES; + return (int)((2 << lgNomEntries) * ThetaUtil.REBUILD_THRESHOLD + + Family.QUICKSELECT.getMaxPreLongs()) * Long.BYTES; } /** diff --git a/src/test/java/org/apache/datasketches/theta/SketchesTest.java b/src/test/java/org/apache/datasketches/theta/SketchesTest.java index 84942fe21..e2a60cb89 100644 --- a/src/test/java/org/apache/datasketches/theta/SketchesTest.java +++ b/src/test/java/org/apache/datasketches/theta/SketchesTest.java @@ -145,7 +145,8 @@ public void checkUtilMethods() { assertEquals(24+(k+1)*8, maxCompSkBytes); final int compSkMaxBytes = getCompactSketchMaxBytes(lgK); { - assertEquals(compSkMaxBytes, ((2 << lgK) * 15) / 16 + (Family.QUICKSELECT.getMaxPreLongs() << 3)); + int bytes = (int)((2 << lgK) * ThetaUtil.REBUILD_THRESHOLD + Family.QUICKSELECT.getMaxPreLongs()) * Long.BYTES; + assertEquals(compSkMaxBytes, bytes); } final int maxSkBytes = getMaxUpdateSketchBytes(k); From 51ce69405ea2a4835af14076b651092f26fe6ba8 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 20 Jun 2024 08:36:20 -0700 Subject: [PATCH 05/11] Remove dependency on Apache Parent POM --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index 1c1107c6c..001c6ab8d 100644 --- a/pom.xml +++ b/pom.xml @@ -25,12 +25,6 @@ under the License. 4.0.0 - - org.apache - apache - 32 - - org.apache.datasketches datasketches-java 6.1.0-SNAPSHOT From 61864f08e685c87c31385420962d4ebbabf0297b Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 27 Jun 2024 17:41:53 -0700 Subject: [PATCH 06/11] Remove code comment. --- .../org/apache/datasketches/kll/KllDoublesValidationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/apache/datasketches/kll/KllDoublesValidationTest.java b/src/test/java/org/apache/datasketches/kll/KllDoublesValidationTest.java index f647b43f4..42802ba65 100644 --- a/src/test/java/org/apache/datasketches/kll/KllDoublesValidationTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllDoublesValidationTest.java @@ -160,7 +160,7 @@ public class KllDoublesValidationTest { private static int[] makeInputArray(int n, int stride) { assert isOdd(stride); - int mask = (1 << 23) - 1; // because library items are single-precision floats //TODO ? + int mask = (1 << 23) - 1; // because library items are single-precision floats int cur = 0; int[] arr = new int[n]; for (int i = 0; i < n; i++) { From 80631a27ed42691a1aa03884da7b67fe93b6101b Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 24 Jul 2024 16:24:30 -0700 Subject: [PATCH 07/11] replace parent pom. --- pom.xml | 58 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/pom.xml b/pom.xml index 001c6ab8d..cd9415604 100644 --- a/pom.xml +++ b/pom.xml @@ -25,6 +25,12 @@ under the License. 4.0.0 + + org.apache + apache + 32 + + org.apache.datasketches datasketches-java 6.1.0-SNAPSHOT @@ -77,9 +83,7 @@ under the License. - - 2.2.0 - + 3.0.0-SNAPSHOT 7.5.1 @@ -89,8 +93,8 @@ under the License. check_cpp_historical_files - 3.6.3 - 1.8 + 3.6.3 + 8 ${java.version} ${java.version} -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 @@ -100,31 +104,31 @@ under the License. ${charset.encoding} yyyy-MM-dd'T'HH-mm-ss'Z' - - 2.16.2 - - - 3.7.1 - 3.13.0 - 3.1.1 - 3.4.1 - 3.2.3 - 3.4.0 - 3.6.3 - 3.0.1 - 3.2.0 - 3.3.1 - 3.2.5 - - 0.16.1 - - 0.8.12 - + + 3.7.1 + 3.13.0 + 3.1.2 + 3.4.1 + 3.2.3 + 3.4.1 + 3.6.3 + 3.0.1 + 3.2.0 + 3.3.1 + 3.2.5 + + 4.9.10 + + 0.16.1 + 4.3.0 + + 0.8.12 + + 2.16.2 - 1.0.0 - 4.9.10 + 1.0.0 From 0e1d83dbd3dec85c8e6f45da9100cb3291e5c053 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 31 Jul 2024 13:00:33 -0700 Subject: [PATCH 08/11] Update Memory dependency to 3.0.0. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cd9415604..6332427ff 100644 --- a/pom.xml +++ b/pom.xml @@ -83,7 +83,7 @@ under the License. - 3.0.0-SNAPSHOT + 3.0.0 7.5.1 From 999f89a127a8de6d81694bda4ca721a1bc7c04dd Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 6 Aug 2024 13:02:52 -0700 Subject: [PATCH 09/11] Update POM with Memory 3.0.0 dependency --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 6332427ff..cc5de8f90 100644 --- a/pom.xml +++ b/pom.xml @@ -132,13 +132,12 @@ under the License. - + org.testng From 93ebd40fc4038d51ad38a2fd0a95b2990270d857 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 6 Aug 2024 14:31:37 -0700 Subject: [PATCH 10/11] Remove references to -P strict --- README.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/README.md b/README.md index 2248217aa..bdc186445 100644 --- a/README.md +++ b/README.md @@ -46,22 +46,15 @@ However, if your application also directly relies on the APIs of the *datasketch you may need additional JVM arguments. Please refer to the [datasketches-memory README](https://github.com/apache/datasketches-memory/blob/master/README.md) for details. -If your application uses Maven, you can also use the *pom.xml* of this component as an example of how to automatically -configure the JVM arguments for compilation and testing based on the version of the JDK. +If your application uses Maven, you can also use the *pom.xml* of this component as an example of how to automatically configure the JVM arguments for compilation and testing based on the version of the JDK. ### Recommended Build Tool This DataSketches component is structured as a Maven project and Maven is the recommended Build Tool. -There are two types of tests: normal unit tests and tests run by the strict profile. - To run normal unit tests: $ mvn clean test -To run the strict profile tests (only supported in Java 8): - - $ mvn clean test -P strict - To install jars built from the downloaded source: $ mvn clean install -DskipTests=true From 8a5e05f992df4322af17e1be591da9cc31e1f422 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 15 Aug 2024 14:34:09 -0700 Subject: [PATCH 11/11] Changes requested from reviews of PR#576 --- .../quantiles/DoublesByteArrayImpl.java | 24 +++++++++---------- .../quantiles/DoublesSketchTest.java | 5 +++- .../theta/DirectQuickSelectSketchTest.java | 20 +++++----------- .../datasketches/theta/UnionImplTest.java | 7 +++--- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/apache/datasketches/quantiles/DoublesByteArrayImpl.java b/src/main/java/org/apache/datasketches/quantiles/DoublesByteArrayImpl.java index f4df5aa8b..8451bad33 100644 --- a/src/main/java/org/apache/datasketches/quantiles/DoublesByteArrayImpl.java +++ b/src/main/java/org/apache/datasketches/quantiles/DoublesByteArrayImpl.java @@ -58,7 +58,7 @@ static byte[] toByteArray(final DoublesSketch sketch, final boolean ordered, fin | (ordered ? ORDERED_FLAG_MASK : 0) | (compact ? (COMPACT_FLAG_MASK | READ_ONLY_FLAG_MASK) : 0); - if (empty && !sketch.hasMemory()) { //empty & has Memory + if (empty && !sketch.hasMemory()) { //empty & !has Memory final byte[] outByteArr = new byte[Long.BYTES]; final WritableMemory memOut = WritableMemory.writableWrap(outByteArr); final int preLongs = 1; @@ -79,15 +79,7 @@ static byte[] toByteArray(final DoublesSketch sketch, final boolean ordered, fin */ private static byte[] convertToByteArray(final DoublesSketch sketch, final int flags, final boolean ordered, final boolean compact) { - final int preLongs = 2; - final int extra = 2; // extra space for min and max quantiles - final int prePlusExtraBytes = (preLongs + extra) << 3; - final int k = sketch.getK(); - final long n = sketch.getN(); - - // If not-compact, have accessor always report full levels. Then use level size to determine - // whether to copy data out. - final DoublesSketchAccessor dsa = DoublesSketchAccessor.wrap(sketch, !compact); + final int preLongs = sketch.isEmpty() ? 1 : 2; final int outBytes = (compact ? sketch.getCurrentCompactSerializedSizeBytes() : sketch.getCurrentUpdatableSerializedSizeBytes()); @@ -95,15 +87,23 @@ private static byte[] convertToByteArray(final DoublesSketch sketch, final int f final byte[] outByteArr = new byte[outBytes]; final WritableMemory memOut = WritableMemory.writableWrap(outByteArr); - //insert preamble-0, N, min, max + //insert pre0 + final int k = sketch.getK(); insertPre0(memOut, preLongs, flags, k); if (sketch.isEmpty()) { return outByteArr; } + //insert N, min, max + final long n = sketch.getN(); insertN(memOut, n); insertMinDouble(memOut, sketch.isEmpty() ? Double.NaN : sketch.getMinItem()); insertMaxDouble(memOut, sketch.isEmpty() ? Double.NaN : sketch.getMaxItem()); - long memOffsetBytes = prePlusExtraBytes; + // If not-compact, have accessor always report full levels. Then use level size to determine + // whether to copy data out. + final DoublesSketchAccessor dsa = DoublesSketchAccessor.wrap(sketch, !compact); + + final int minAndMax = 2; // extra space for min and max quantiles + long memOffsetBytes = (preLongs + minAndMax) << 3; // might need to sort base buffer but don't want to change input sketch final int bbCnt = computeBaseBufferItems(k, n); diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java index 1bf6ad589..cf8e142b7 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java @@ -162,13 +162,16 @@ public void directSketchShouldMoveOntoHeapEventually2() { break; } } + assertFalse(wmem.isAlive()); } @Test public void checkEmptyDirect() { WritableMemory wmem = WritableMemory.allocateDirect(1000); UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); - sketch.toByteArray(); //exercises a specific path + byte[] bytes = sketch.toByteArray(); //exercises a specific path + byte[] result = {1,3,8,4,-128,0,0,0}; + assertEquals(bytes, result); wmem.close(); } diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index 4830f9576..ccfbf704f 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -74,26 +74,18 @@ public void checkBadSerVer() { } } - @Test//(expectedExceptions = SketchesArgumentException.class) + @Test(expectedExceptions = SketchesArgumentException.class) public void checkConstructorKtooSmall() { int k = 8; - try (WritableMemory mem = makeNativeMemory(k)) { - UpdateSketch.builder().setNominalEntries(k).build(mem); - } catch (final Exception e) { - if (e instanceof SketchesArgumentException) {} - else { throw new RuntimeException(e); } - } + WritableMemory mem = makeNativeMemory(k); + UpdateSketch.builder().setNominalEntries(k).build(mem); } - @Test//(expectedExceptions = SketchesArgumentException.class) + @Test(expectedExceptions = SketchesArgumentException.class) public void checkConstructorMemTooSmall() { int k = 16; - try (WritableMemory mem = makeNativeMemory(k/2)) { - UpdateSketch.builder().setNominalEntries(k).build(mem); - } catch (final Exception e) { - if (e instanceof SketchesArgumentException) {} - else { throw new RuntimeException(e); } - } + WritableMemory mem = makeNativeMemory(k/2); + UpdateSketch.builder().setNominalEntries(k).build(mem); } @Test(expectedExceptions = SketchesArgumentException.class) diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java index b4ff9a245..31c26e4ec 100644 --- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java +++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java @@ -187,14 +187,14 @@ public void checkVer2EmptyHandling() { } @Test - public void checkMoveAndResize() { + public void checkMoveAndResizeOffHeap() { final int k = 1 << 12; final int u = 2 * k; final int bytes = Sketches.getMaxUpdateSketchBytes(k); - WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2); + WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2); //too small, forces new allocation on heap WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2); final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); - assertTrue(sketch.isSameResource(wmem)); + assertTrue(sketch.isSameResource(wmem)); //also testing the isSameResource function final Union union = SetOperation.builder().buildUnion(wmem2); assertTrue(union.isSameResource(wmem2)); @@ -205,6 +205,7 @@ public void checkMoveAndResize() { final Union union2 = SetOperation.builder().buildUnion(); //on-heap union assertFalse(union2.isSameResource(wmem2)); //obviously not wmem.close(); + //note wmem2 has already been closed by the DefaultMemoryRequestServer } @Test