From c192326944e2c7dd852033f80c782fe7b9c14d83 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Thu, 16 Jan 2025 16:26:35 +0800 Subject: [PATCH 1/3] The time range of device in resource file may be larger than the actual time range in tsfile --- .../repair/RepairDataFileScanUtil.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index 2d268761ecab..ddf75236b263 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@ -187,9 +187,8 @@ private void checkAlignedDeviceSeries( new TimeRange( timeseriesMetadata.getStatistics().getStartTime(), timeseriesMetadata.getStatistics().getEndTime()); - if (checkTsFileResource && !timeseriesTimeRange.equals(deviceTimeRangeInResource)) { - throw new CompactionStatisticsCheckFailedException( - device, deviceTimeRangeInResource, timeseriesTimeRange); + if (checkTsFileResource) { + compareDeviceTimeRange(device, deviceTimeRangeInResource, timeseriesTimeRange); } long actualTimeseriesStartTime = Long.MAX_VALUE; @@ -283,10 +282,22 @@ private void checkNonAlignedDeviceSeries( if (!checkTsFileResource || actualDeviceStartTime > actualDeviceEndTime) { return; } - TimeRange actualDeviceTimeRange = new TimeRange(actualDeviceStartTime, actualDeviceEndTime); - if (!actualDeviceTimeRange.equals(deviceTimeRangeInResource)) { - throw new CompactionStatisticsCheckFailedException( - device, deviceTimeRangeInResource, actualDeviceTimeRange); + compareDeviceTimeRange(device, deviceTimeRangeInResource, new TimeRange(actualDeviceStartTime, actualDeviceEndTime)); + } + + private void compareDeviceTimeRange(IDeviceID device, TimeRange deviceTimeRangeInResource, TimeRange actualDeviceTimeRange) { + long innerCompactionCount = resource.getTsFileID().getInnerCompactionCount(); + if (innerCompactionCount == 0) { + // for the files generate by flush with deletions, the statistics may be larger than the actual + if (!deviceTimeRangeInResource.contains(actualDeviceTimeRange)) { + throw new CompactionStatisticsCheckFailedException( + device, deviceTimeRangeInResource, actualDeviceTimeRange); + } + } else { + if (!actualDeviceTimeRange.equals(deviceTimeRangeInResource)) { + throw new CompactionStatisticsCheckFailedException( + device, deviceTimeRangeInResource, actualDeviceTimeRange); + } } } From d9eacb73699cc8156435ce1f95b1302914facd4f Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 17 Jan 2025 09:37:15 +0800 Subject: [PATCH 2/3] spotless --- .../compaction/repair/RepairDataFileScanUtil.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index ddf75236b263..45e7f7f43405 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@ -282,13 +282,18 @@ private void checkNonAlignedDeviceSeries( if (!checkTsFileResource || actualDeviceStartTime > actualDeviceEndTime) { return; } - compareDeviceTimeRange(device, deviceTimeRangeInResource, new TimeRange(actualDeviceStartTime, actualDeviceEndTime)); + compareDeviceTimeRange( + device, + deviceTimeRangeInResource, + new TimeRange(actualDeviceStartTime, actualDeviceEndTime)); } - private void compareDeviceTimeRange(IDeviceID device, TimeRange deviceTimeRangeInResource, TimeRange actualDeviceTimeRange) { + private void compareDeviceTimeRange( + IDeviceID device, TimeRange deviceTimeRangeInResource, TimeRange actualDeviceTimeRange) { long innerCompactionCount = resource.getTsFileID().getInnerCompactionCount(); if (innerCompactionCount == 0) { - // for the files generate by flush with deletions, the statistics may be larger than the actual + // for the files generate by flush with deletions, the statistics may be larger than the + // actual if (!deviceTimeRangeInResource.contains(actualDeviceTimeRange)) { throw new CompactionStatisticsCheckFailedException( device, deviceTimeRangeInResource, actualDeviceTimeRange); From 16bc64991e09a936163ce3f96bd3a5d089471053 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 17 Jan 2025 11:29:20 +0800 Subject: [PATCH 3/3] fix ut --- .../compaction/repair/RepairDataFileScanUtilTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtilTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtilTest.java index a4fe8e20e13e..90ae850afa11 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtilTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtilTest.java @@ -134,7 +134,7 @@ public void testWrongChunkStatisticsWithAlignedSeries() throws IOException { @Test public void testWrongResourceStatistics() throws IOException { - TsFileResource resource = createEmptyFileAndResource(true); + TsFileResource resource = createEmptyFileAndResourceWithName("1-1-1-0", 0, true); try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) { writer.startChunkGroup("d1"); writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue(