From eb26915a1517543830a8dada93b82b3c4e1ce74d Mon Sep 17 00:00:00 2001 From: shuwenwei <55970239+shuwenwei@users.noreply.github.com> Date: Wed, 28 Jan 2026 17:38:53 +0800 Subject: [PATCH] Fix allValueColDeletedMap index out of bound caused by stale rowCount after TVList sort (#17098) --- .../memtable/AlignedReadOnlyMemChunk.java | 12 ++++- .../dataregion/memtable/ReadOnlyMemChunk.java | 4 +- .../db/utils/datastructure/AlignedTVList.java | 6 ++- .../datastructure/BackAlignedTVList.java | 3 +- .../utils/datastructure/BackBinaryTVList.java | 3 +- .../datastructure/BackBooleanTVList.java | 3 +- .../utils/datastructure/BackDoubleTVList.java | 3 +- .../utils/datastructure/BackFloatTVList.java | 3 +- .../db/utils/datastructure/BackIntTVList.java | 3 +- .../utils/datastructure/BackLongTVList.java | 3 +- .../datastructure/QuickAlignedTVList.java | 3 +- .../datastructure/QuickBinaryTVList.java | 3 +- .../datastructure/QuickBooleanTVList.java | 3 +- .../datastructure/QuickDoubleTVList.java | 3 +- .../utils/datastructure/QuickFloatTVList.java | 3 +- .../utils/datastructure/QuickIntTVList.java | 3 +- .../utils/datastructure/QuickLongTVList.java | 3 +- .../iotdb/db/utils/datastructure/TVList.java | 2 +- .../utils/datastructure/TimAlignedTVList.java | 3 +- .../utils/datastructure/TimBinaryTVList.java | 3 +- .../utils/datastructure/TimBooleanTVList.java | 3 +- .../utils/datastructure/TimDoubleTVList.java | 3 +- .../utils/datastructure/TimFloatTVList.java | 3 +- .../db/utils/datastructure/TimIntTVList.java | 3 +- .../db/utils/datastructure/TimLongTVList.java | 3 +- .../memtable/PrimitiveMemTableTest.java | 54 +++++++++++++++++++ 26 files changed, 114 insertions(+), 27 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index d00424856bae2..bbab08d444284 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -113,7 +113,15 @@ public void sortTvLists() { AlignedTVList alignedTvList = (AlignedTVList) entry.getKey(); int queryRowCount = entry.getValue(); if (!alignedTvList.isSorted() && queryRowCount > alignedTvList.seqRowCount()) { - alignedTvList.sort(); + // sort() returns the current row count + // TVList may grow between prepareTvListMapForQuery and actual query execution(now). + // The queryRowCount recorded here is only a snapshot taken during prepareTvListMapForQuery + // phase. + // Additional written rows that are not covered by the original queryRowCount can be + // involved in current sort operation. + // We must update queryRowCount here, otherwise, it may be used later to build + // BitMaps, causing bitmap array size mismatch and possible out of bound. + entry.setValue(alignedTvList.sort()); long alignedTvListRamSize = alignedTvList.calculateRamSize(); alignedTvList.lockQueryList(); try { @@ -358,7 +366,7 @@ public IPointReader getPointReader() { AlignedTVList alignedTvList = (AlignedTVList) entry.getKey(); int queryLength = entry.getValue(); if (!alignedTvList.isSorted() && queryLength > alignedTvList.seqRowCount()) { - alignedTvList.sort(); + entry.setValue(alignedTvList.sort()); long alignedTvListRamSize = alignedTvList.calculateRamSize(); alignedTvList.lockQueryList(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java index 3438759cfa902..38c8e9e2d13cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java @@ -135,7 +135,7 @@ public void sortTvLists() { TVList tvList = entry.getKey(); int queryRowCount = entry.getValue(); if (!tvList.isSorted() && queryRowCount > tvList.seqRowCount()) { - tvList.sort(); + entry.setValue(tvList.sort()); long tvListRamSize = tvList.calculateRamSize(); tvList.lockQueryList(); try { @@ -287,7 +287,7 @@ public IPointReader getPointReader() { TVList tvList = entry.getKey(); int queryLength = entry.getValue(); if (!tvList.isSorted() && queryLength > tvList.seqRowCount()) { - tvList.sort(); + entry.setValue(tvList.sort()); long tvListRamSize = tvList.calculateRamSize(); tvList.lockQueryList(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index e4230787a7611..090b057498748 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -1259,6 +1259,10 @@ public static AlignedTVList deserialize(DataInputStream stream) throws IOExcepti } public BitMap getAllValueColDeletedMap() { + return getAllValueColDeletedMap(rowCount); + } + + public BitMap getAllValueColDeletedMap(int rowCount) { // row exists when any column value exists if (bitMaps == null) { return null; @@ -1415,7 +1419,7 @@ public AlignedTVListIterator( (columnIndexList == null) ? IntStream.range(0, dataTypes.size()).boxed().collect(Collectors.toList()) : columnIndexList; - this.allValueColDeletedMap = getAllValueColDeletedMap(); + this.allValueColDeletedMap = getAllValueColDeletedMap(this.rows); this.valueColumnsDeletionList = valueColumnsDeletionList; this.floatPrecision = floatPrecision != null ? floatPrecision : 0; this.encodingList = encodingList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java index 2ad728a1a2643..2603790bd668d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java @@ -32,12 +32,13 @@ public class BackAlignedTVList extends QuickAlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java index cd85976a58ef4..9faedb162a15a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java @@ -26,12 +26,13 @@ public class BackBinaryTVList extends QuickBinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java index 969d398128572..2186c1fa66e5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java @@ -27,12 +27,13 @@ public class BackBooleanTVList extends QuickBooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java index 6d79b9d0ab294..0cb7c438db49a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java @@ -27,12 +27,13 @@ public class BackDoubleTVList extends QuickDoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java index 75b09507cc49a..a3208b96f6be1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java @@ -27,12 +27,13 @@ public class BackFloatTVList extends QuickFloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java index 3ccc5cbc946f5..3a2af2841c96d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java @@ -27,12 +27,13 @@ public class BackIntTVList extends QuickIntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java index 58d4a4e2fc8e0..580af939c7f7f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java @@ -26,12 +26,13 @@ public class BackLongTVList extends QuickLongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java index 69f24527d6799..c5bd55500567e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java @@ -31,11 +31,12 @@ public class QuickAlignedTVList extends AlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java index 645d0dbce5a26..b8abddaee5dfc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java @@ -26,11 +26,12 @@ public class QuickBinaryTVList extends BinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java index 779159ca71979..50df94c772af3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java @@ -26,11 +26,12 @@ public class QuickBooleanTVList extends BooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java index 58f3cc1ce2f87..44120672fce32 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java @@ -26,11 +26,12 @@ public class QuickDoubleTVList extends DoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java index 2fc22ab29902d..6ad9b1d72577f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java @@ -26,11 +26,12 @@ public class QuickFloatTVList extends FloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java index e83646e32024b..af7e35eb65591 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java @@ -26,11 +26,12 @@ public class QuickIntTVList extends IntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java index 5f629720c5d64..98134984c0c0f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java @@ -26,11 +26,12 @@ public class QuickLongTVList extends LongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index 8695182712f42..0f4ab88441aa6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -176,7 +176,7 @@ public long getReservedMemoryBytes() { return reservedMemoryBytes; } - public abstract void sort(); + public abstract int sort(); public void increaseReferenceCount() { referenceCount.incrementAndGet(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java index 479e0a969a39f..61d5c75096ebb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java @@ -31,7 +31,7 @@ public class TimAlignedTVList extends AlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -40,6 +40,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java index c78d67034da0b..df0b179537184 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java @@ -26,7 +26,7 @@ public class TimBinaryTVList extends BinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java index 0e7dc48faaf35..e5051ec0ede87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java @@ -26,7 +26,7 @@ public class TimBooleanTVList extends BooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java index 29fc62d91c6e3..8b4b40c2da53f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java @@ -26,7 +26,7 @@ public class TimDoubleTVList extends DoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java index 83ad6aca1f676..522b8bf7d61cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java @@ -26,7 +26,7 @@ public class TimFloatTVList extends FloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java index 8216f2eb2961d..594980d45464d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java @@ -26,7 +26,7 @@ public class TimIntTVList extends IntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java index bcd5c3022f9c7..e741ca51504ab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java @@ -26,7 +26,7 @@ public class TimLongTVList extends LongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public synchronized void sort() { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java index 572e6cd7ef881..f9186b774e1f3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java @@ -44,16 +44,20 @@ import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; import org.apache.iotdb.db.storageengine.dataregion.modification.Modification; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; import org.apache.iotdb.db.utils.MathUtils; +import org.apache.iotdb.db.utils.datastructure.MemPointIterator; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; @@ -123,6 +127,56 @@ public void memSeriesSortIteratorTest() throws IOException, QueryProcessExceptio Assert.assertEquals(count, i); } + /** + * Regression test for concurrent writes between prepare and query execution. The test ensures + * that when new rows are written after the prepare phase but before TVList sorting, the query + * refreshes the rowCount after sort and avoids using a stale queryRowCount for bitmap + * construction, which would otherwise cause ArrayIndexOutOfBoundsException. + */ + @Test + public void testWriteDuringPrepareTVListAndActualQueryExecution() + throws QueryProcessException, IOException, IllegalPathException { + + PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0"); + List measurementSchemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32)); + for (int i = 1000; i < 2000; i++) { + memTable.writeAlignedRow( + new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + for (int i = 100; i < 200; i++) { + memTable.writeAlignedRow( + new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + MeasurementPath path = new MeasurementPath("root.test.d1.s1", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 150, 160); + path = new MeasurementPath("root.test.d1.s2", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 150, 160); + path = new MeasurementPath("root.test.d1.s3", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 150, 160); + ResourceByPathUtils resourcesByPathUtils = + ResourceByPathUtils.getResourceInstance( + new AlignedPath("root.test.d1", Arrays.asList("s1", "s2", "s3"), measurementSchemas)); + ReadOnlyMemChunk readOnlyMemChunk = + resourcesByPathUtils.getReadOnlyMemChunkFromMemTable( + new QueryContext(1), memTable, null, Long.MAX_VALUE, null); + + for (int i = 1; i <= 50; i++) { + memTable.writeAlignedRow( + new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + + readOnlyMemChunk.sortTvLists(); + + MemPointIterator memPointIterator = readOnlyMemChunk.createMemPointIterator(Ordering.ASC, null); + while (memPointIterator.hasNextBatch()) { + memPointIterator.nextBatch(); + } + } + @Test public void memSeriesToStringTest() throws IOException { TSDataType dataType = TSDataType.INT32;