diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index 2157623e1c67..89fe9facc71b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -118,7 +118,15 @@ public void sortTvLists() { AlignedTVList alignedTvList = (AlignedTVList) entry.getKey(); int queryRowCount = entry.getValue(); if (!alignedTvList.isSorted() && queryRowCount > alignedTvList.seqRowCount()) { - alignedTvList.sort(); + // sort() returns the current row count + // TVList may grow between prepareTvListMapForQuery and actual query execution(now). + // The queryRowCount recorded here is only a snapshot taken during prepareTvListMapForQuery + // phase. + // Additional written rows that are not covered by the original queryRowCount can be + // involved in current sort operation. + // We must update queryRowCount here, otherwise, it may be used later to build + // BitMaps, causing bitmap array size mismatch and possible out of bound. + entry.setValue(alignedTvList.sort()); long alignedTvListRamSize = alignedTvList.calculateRamSize(); alignedTvList.lockQueryList(); try { @@ -375,7 +383,7 @@ public IPointReader getPointReader() { AlignedTVList alignedTvList = (AlignedTVList) entry.getKey(); int queryLength = entry.getValue(); if (!alignedTvList.isSorted() && queryLength > alignedTvList.seqRowCount()) { - alignedTvList.sort(); + entry.setValue(alignedTvList.sort()); long alignedTvListRamSize = alignedTvList.calculateRamSize(); alignedTvList.lockQueryList(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java index 7304eab6fb84..ec8a74480634 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java @@ -136,7 +136,7 @@ public void sortTvLists() { TVList tvList = entry.getKey(); int queryRowCount = entry.getValue(); if (!tvList.isSorted() && queryRowCount > tvList.seqRowCount()) { - tvList.sort(); + entry.setValue(tvList.sort()); long tvListRamSize = tvList.calculateRamSize(); tvList.lockQueryList(); try { @@ -289,7 +289,7 @@ public IPointReader getPointReader() { TVList tvList = entry.getKey(); int queryLength = entry.getValue(); if (!tvList.isSorted() && queryLength > tvList.seqRowCount()) { - tvList.sort(); + entry.setValue(tvList.sort()); long tvListRamSize = tvList.calculateRamSize(); tvList.lockQueryList(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 237d6d28a0f1..8c732d4f08e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -1599,6 +1599,10 @@ public boolean isTimeDeleted(int index, boolean needConvertIndex) { } public BitMap getAllValueColDeletedMap() { + return getAllValueColDeletedMap(rowCount); + } + + public BitMap getAllValueColDeletedMap(int rowCount) { // row exists when any column value exists if (bitMaps == null) { return null; @@ -1768,7 +1772,7 @@ public AlignedTVListIterator( (columnIndexList == null) ? IntStream.range(0, dataTypes.size()).boxed().collect(Collectors.toList()) : columnIndexList; - this.allValueColDeletedMap = ignoreAllNullRows ? getAllValueColDeletedMap() : null; + this.allValueColDeletedMap = ignoreAllNullRows ? getAllValueColDeletedMap(this.rows) : null; this.floatPrecision = floatPrecision != null ? floatPrecision : 0; this.encodingList = encodingList; this.timeColumnDeletion = timeColumnDeletion; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java index 2ad728a1a264..2603790bd668 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java @@ -32,12 +32,13 @@ public class BackAlignedTVList extends QuickAlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java index cd85976a58ef..9faedb162a15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java @@ -26,12 +26,13 @@ public class BackBinaryTVList extends QuickBinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java index 969d39812857..2186c1fa66e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java @@ -27,12 +27,13 @@ public class BackBooleanTVList extends QuickBooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java index 6d79b9d0ab29..0cb7c438db49 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java @@ -27,12 +27,13 @@ public class BackDoubleTVList extends QuickDoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java index 75b09507cc49..a3208b96f6be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java @@ -27,12 +27,13 @@ public class BackFloatTVList extends QuickFloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java index 403314f58179..6994b8e76eb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java @@ -34,12 +34,13 @@ public class BackIntTVList extends QuickIntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java index 58d4a4e2fc8e..580af939c7f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java @@ -26,12 +26,13 @@ public class BackLongTVList extends QuickLongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java index 69f24527d679..c5bd55500567 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java @@ -31,11 +31,12 @@ public class QuickAlignedTVList extends AlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java index 645d0dbce5a2..b8abddaee5df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java @@ -26,11 +26,12 @@ public class QuickBinaryTVList extends BinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java index 779159ca7197..50df94c772af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java @@ -26,11 +26,12 @@ public class QuickBooleanTVList extends BooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java index 58f3cc1ce2f8..44120672fce3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java @@ -26,11 +26,12 @@ public class QuickDoubleTVList extends DoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java index 2fc22ab29902..6ad9b1d72577 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java @@ -26,11 +26,12 @@ public class QuickFloatTVList extends FloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java index 035dbc4f3936..984b322872a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java @@ -33,11 +33,12 @@ public class QuickIntTVList extends IntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java index 5f629720c5d6..98134984c0c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java @@ -26,11 +26,12 @@ public class QuickLongTVList extends LongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index cfeeac11968a..d61611c4cc5f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -185,7 +185,7 @@ public long getReservedMemoryBytes() { return reservedMemoryBytes; } - public abstract void sort(); + public abstract int sort(); public void increaseReferenceCount() { referenceCount.incrementAndGet(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java index 479e0a969a39..61d5c75096eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java @@ -31,7 +31,7 @@ public class TimAlignedTVList extends AlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -40,6 +40,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java index c78d67034da0..df0b17953718 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java @@ -26,7 +26,7 @@ public class TimBinaryTVList extends BinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java index 0e7dc48faaf3..e5051ec0ede8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java @@ -26,7 +26,7 @@ public class TimBooleanTVList extends BooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java index 29fc62d91c6e..8b4b40c2da53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java @@ -26,7 +26,7 @@ public class TimDoubleTVList extends DoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java index 83ad6aca1f67..522b8bf7d61c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java @@ -26,7 +26,7 @@ public class TimFloatTVList extends FloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java index 33ccb1b0e5d3..e1165faa12f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java @@ -33,7 +33,7 @@ public class TimIntTVList extends IntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -42,6 +42,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java index bcd5c3022f9c..e741ca51504a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java @@ -26,7 +26,7 @@ public class TimLongTVList extends LongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java index d64e7c7e5cd7..b16e20d4f857 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java @@ -45,17 +45,21 @@ import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; import org.apache.iotdb.db.utils.MathUtils; +import org.apache.iotdb.db.utils.datastructure.MemPointIterator; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; @@ -148,6 +152,59 @@ public void memSeriesSortIteratorTest() throws IOException, QueryProcessExceptio Assert.assertEquals(count, i); } + /** + * Regression test for concurrent writes between prepare and query execution. The test ensures + * that when new rows are written after the prepare phase but before TVList sorting, the query + * refreshes the rowCount after sort and avoids using a stale queryRowCount for bitmap + * construction, which would otherwise cause ArrayIndexOutOfBoundsException. + */ + @Test + public void testWriteDuringPrepareTVListAndActualQueryExecution() + throws QueryProcessException, IOException, IllegalPathException { + + PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0"); + List measurementSchemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32)); + for (int i = 1000; i < 2000; i++) { + memTable.writeAlignedRow( + new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + for (int i = 100; i < 200; i++) { + memTable.writeAlignedRow( + new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + memTable.delete( + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 150, 160)); + memTable.delete( + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 150, 160)); + memTable.delete( + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 150, 160)); + ResourceByPathUtils resourcesByPathUtils = + ResourceByPathUtils.getResourceInstance( + new AlignedFullPath( + new StringArrayDeviceID("root.test.d1"), + Arrays.asList("s1", "s2", "s3"), + measurementSchemas)); + ReadOnlyMemChunk readOnlyMemChunk = + resourcesByPathUtils.getReadOnlyMemChunkFromMemTable( + new QueryContext(1), memTable, null, Long.MAX_VALUE, null); + + for (int i = 1; i <= 50; i++) { + memTable.writeAlignedRow( + new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + + readOnlyMemChunk.sortTvLists(); + + MemPointIterator memPointIterator = readOnlyMemChunk.createMemPointIterator(Ordering.ASC, null); + while (memPointIterator.hasNextBatch()) { + memPointIterator.nextBatch(); + } + } + @Test public void memSeriesToStringTest() throws IOException { TSDataType dataType = TSDataType.INT32;